[[PATCH] sparc64: Add 16GB hugepage support] On 24/05/2017 (Wed 17:29) Nitin Gupta wrote:
> Orabug: 25362942 > > Signed-off-by: Nitin Gupta <[email protected]> If this wasn't an accidental git send-email misfire, then there should be a long log indicating the use case, the perforamnce increase, the testing that was done, etc. etc. Normally I'd not notice but since I was Cc'd I figured it was worth a mention -- for example the vendor ID above doesn't mean a thing to all the rest of us, hence why I suspect it was a git send-email misfire; sadly, I think we've all accidentally done that at least once.... Paul. -- > --- > arch/sparc/include/asm/page_64.h | 3 +- > arch/sparc/include/asm/pgtable_64.h | 5 +++ > arch/sparc/include/asm/tsb.h | 35 +++++++++++++++++- > arch/sparc/kernel/tsb.S | 2 +- > arch/sparc/mm/hugetlbpage.c | 74 > ++++++++++++++++++++++++++----------- > arch/sparc/mm/init_64.c | 41 ++++++++++++++++---- > 6 files changed, 128 insertions(+), 32 deletions(-) > > diff --git a/arch/sparc/include/asm/page_64.h > b/arch/sparc/include/asm/page_64.h > index 5961b2d..8ee1f97 100644 > --- a/arch/sparc/include/asm/page_64.h > +++ b/arch/sparc/include/asm/page_64.h > @@ -17,6 +17,7 @@ > > #define HPAGE_SHIFT 23 > #define REAL_HPAGE_SHIFT 22 > +#define HPAGE_16GB_SHIFT 34 > #define HPAGE_2GB_SHIFT 31 > #define HPAGE_256MB_SHIFT 28 > #define HPAGE_64K_SHIFT 16 > @@ -28,7 +29,7 @@ > #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) > #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA > #define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) > -#define HUGE_MAX_HSTATE 4 > +#define HUGE_MAX_HSTATE 5 > #endif > > #ifndef __ASSEMBLY__ > diff --git a/arch/sparc/include/asm/pgtable_64.h > b/arch/sparc/include/asm/pgtable_64.h > index 6fbd931..2444b02 100644 > --- a/arch/sparc/include/asm/pgtable_64.h > +++ b/arch/sparc/include/asm/pgtable_64.h > @@ -414,6 +414,11 @@ static inline bool is_hugetlb_pmd(pmd_t pmd) > return !!(pmd_val(pmd) & _PAGE_PMD_HUGE); > } > > +static inline bool is_hugetlb_pud(pud_t pud) > +{ > + return !!(pud_val(pud) & _PAGE_PUD_HUGE); > +} > + > #ifdef CONFIG_TRANSPARENT_HUGEPAGE > static inline pmd_t pmd_mkhuge(pmd_t pmd) > { > diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h > index 32258e0..fbd8da7 100644 > --- a/arch/sparc/include/asm/tsb.h > +++ b/arch/sparc/include/asm/tsb.h > @@ -195,6 +195,36 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, > __tsb_phys_patch_end; > nop; \ > 699: > > + /* PUD has been loaded into REG1, interpret the value, seeing > + * if it is a HUGE PUD or a normal one. If it is not valid > + * then jump to FAIL_LABEL. If it is a HUGE PUD, and it > + * translates to a valid PTE, branch to PTE_LABEL. > + * > + * We have to propagate bits [32:22] from the virtual address > + * to resolve at 4M granularity. > + */ > +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) > +#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, > PTE_LABEL) \ > + brz,pn REG1, FAIL_LABEL; \ > + sethi %uhi(_PAGE_PUD_HUGE), REG2; \ > + sllx REG2, 32, REG2; \ > + andcc REG1, REG2, %g0; \ > + be,pt %xcc, 700f; \ > + sethi %hi(0x1ffc0000), REG2; \ > + brgez,pn REG1, FAIL_LABEL; \ > + sllx REG2, 1, REG2; \ > + brgez,pn REG1, FAIL_LABEL; \ > + andn REG1, REG2, REG1; \ > + and VADDR, REG2, REG2; \ > + brlz,pt REG1, PTE_LABEL; \ > + or REG1, REG2, REG1; \ > +700: > +#else > +#define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, > PTE_LABEL) \ > + brz,pn REG1, FAIL_LABEL; \ > + nop; > +#endif > + > /* PMD has been loaded into REG1, interpret the value, seeing > * if it is a HUGE PMD or a normal one. If it is not valid > * then jump to FAIL_LABEL. If it is a HUGE PMD, and it > @@ -209,14 +239,14 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, > __tsb_phys_patch_end; > sethi %uhi(_PAGE_PMD_HUGE), REG2; \ > sllx REG2, 32, REG2; \ > andcc REG1, REG2, %g0; \ > - be,pt %xcc, 700f; \ > + be,pt %xcc, 701f; \ > sethi %hi(4 * 1024 * 1024), REG2; \ > brgez,pn REG1, FAIL_LABEL; \ > andn REG1, REG2, REG1; \ > and VADDR, REG2, REG2; \ > brlz,pt REG1, PTE_LABEL; \ > or REG1, REG2, REG1; \ > -700: > +701: > #else > #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, > PTE_LABEL) \ > brz,pn REG1, FAIL_LABEL; \ > @@ -242,6 +272,7 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, > __tsb_phys_patch_end; > srlx REG2, 64 - PAGE_SHIFT, REG2; \ > andn REG2, 0x7, REG2; \ > ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \ > + USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \ > brz,pn REG1, FAIL_LABEL; \ > sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \ > srlx REG2, 64 - PAGE_SHIFT, REG2; \ > diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S > index 10689cf..a0a5a13 100644 > --- a/arch/sparc/kernel/tsb.S > +++ b/arch/sparc/kernel/tsb.S > @@ -117,7 +117,7 @@ tsb_miss_page_table_walk_sun4v_fastpath: > /* Valid PTE is now in %g5. */ > > #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) > - sethi %uhi(_PAGE_PMD_HUGE), %g7 > + sethi %uhi(_PAGE_PMD_HUGE | _PAGE_PUD_HUGE), %g7 > sllx %g7, 32, %g7 > > andcc %g5, %g7, %g0 > diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c > index 7c29d38..62c1e62 100644 > --- a/arch/sparc/mm/hugetlbpage.c > +++ b/arch/sparc/mm/hugetlbpage.c > @@ -143,6 +143,10 @@ static pte_t sun4v_hugepage_shift_to_tte(pte_t entry, > unsigned int shift) > pte_val(entry) = pte_val(entry) & ~_PAGE_SZALL_4V; > > switch (shift) { > + case HPAGE_16GB_SHIFT: > + hugepage_size = _PAGE_SZ16GB_4V; > + pte_val(entry) |= _PAGE_PUD_HUGE; > + break; > case HPAGE_2GB_SHIFT: > hugepage_size = _PAGE_SZ2GB_4V; > pte_val(entry) |= _PAGE_PMD_HUGE; > @@ -187,6 +191,9 @@ static unsigned int sun4v_huge_tte_to_shift(pte_t entry) > unsigned int shift; > > switch (tte_szbits) { > + case _PAGE_SZ16GB_4V: > + shift = HPAGE_16GB_SHIFT; > + break; > case _PAGE_SZ2GB_4V: > shift = HPAGE_2GB_SHIFT; > break; > @@ -263,7 +270,12 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, > > pgd = pgd_offset(mm, addr); > pud = pud_alloc(mm, pgd, addr); > - if (pud) { > + if (!pud) > + return NULL; > + > + if (sz >= PUD_SIZE) > + pte = (pte_t *)pud; > + else { > pmd = pmd_alloc(mm, pud, addr); > if (!pmd) > return NULL; > @@ -288,12 +300,16 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned > long addr) > if (!pgd_none(*pgd)) { > pud = pud_offset(pgd, addr); > if (!pud_none(*pud)) { > - pmd = pmd_offset(pud, addr); > - if (!pmd_none(*pmd)) { > - if (is_hugetlb_pmd(*pmd)) > - pte = (pte_t *)pmd; > - else > - pte = pte_offset_map(pmd, addr); > + if (is_hugetlb_pud(*pud)) > + pte = (pte_t *)pud; > + else { > + pmd = pmd_offset(pud, addr); > + if (!pmd_none(*pmd)) { > + if (is_hugetlb_pmd(*pmd)) > + pte = (pte_t *)pmd; > + else > + pte = pte_offset_map(pmd, addr); > + } > } > } > } > @@ -304,12 +320,20 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned > long addr) > void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, > pte_t *ptep, pte_t entry) > { > - unsigned int i, nptes, orig_shift, shift; > - unsigned long size; > + unsigned int nptes, orig_shift, shift; > + unsigned long i, size; > pte_t orig; > > size = huge_tte_to_size(entry); > - shift = size >= HPAGE_SIZE ? PMD_SHIFT : PAGE_SHIFT; > + > + shift = PAGE_SHIFT; > + if (size >= PUD_SIZE) > + shift = PUD_SHIFT; > + else if (size >= PMD_SIZE) > + shift = PMD_SHIFT; > + else > + shift = PAGE_SHIFT; > + > nptes = size >> shift; > > if (!pte_present(*ptep) && pte_present(entry)) > @@ -332,19 +356,23 @@ void set_huge_pte_at(struct mm_struct *mm, unsigned > long addr, > pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, > pte_t *ptep) > { > - unsigned int i, nptes, hugepage_shift; > + unsigned int i, nptes, orig_shift, shift; > unsigned long size; > pte_t entry; > > entry = *ptep; > size = huge_tte_to_size(entry); > - if (size >= HPAGE_SIZE) > - nptes = size >> PMD_SHIFT; > + > + shift = PAGE_SHIFT; > + if (size >= PUD_SIZE) > + shift = PUD_SHIFT; > + else if (size >= PMD_SIZE) > + shift = PMD_SHIFT; > else > - nptes = size >> PAGE_SHIFT; > + shift = PAGE_SHIFT; > > - hugepage_shift = pte_none(entry) ? PAGE_SHIFT : > - huge_tte_to_shift(entry); > + nptes = size >> shift; > + orig_shift = pte_none(entry) ? PAGE_SHIFT : huge_tte_to_shift(entry); > > if (pte_present(entry)) > mm->context.hugetlb_pte_count -= nptes; > @@ -353,11 +381,11 @@ pte_t huge_ptep_get_and_clear(struct mm_struct *mm, > unsigned long addr, > for (i = 0; i < nptes; i++) > ptep[i] = __pte(0UL); > > - maybe_tlb_batch_add(mm, addr, ptep, entry, 0, hugepage_shift); > + maybe_tlb_batch_add(mm, addr, ptep, entry, 0, orig_shift); > /* An HPAGE_SIZE'ed page is composed of two REAL_HPAGE_SIZE'ed pages */ > if (size == HPAGE_SIZE) > maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0, > - hugepage_shift); > + orig_shift); > > return entry; > } > @@ -370,7 +398,8 @@ int pmd_huge(pmd_t pmd) > > int pud_huge(pud_t pud) > { > - return 0; > + return !pud_none(pud) && > + (pud_val(pud) & (_PAGE_VALID|_PAGE_PUD_HUGE)) != _PAGE_VALID; > } > > static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, > @@ -434,8 +463,11 @@ static void hugetlb_free_pud_range(struct mmu_gather > *tlb, pgd_t *pgd, > next = pud_addr_end(addr, end); > if (pud_none_or_clear_bad(pud)) > continue; > - hugetlb_free_pmd_range(tlb, pud, addr, next, floor, > - ceiling); > + if (is_hugetlb_pud(*pud)) > + pud_clear(pud); > + else > + hugetlb_free_pmd_range(tlb, pud, addr, next, floor, > + ceiling); > } while (pud++, addr = next, addr != end); > > start &= PGDIR_MASK; > diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c > index 0cda653..7c0fe73 100644 > --- a/arch/sparc/mm/init_64.c > +++ b/arch/sparc/mm/init_64.c > @@ -337,6 +337,10 @@ static int __init setup_hugepagesz(char *string) > hugepage_shift = ilog2(hugepage_size); > > switch (hugepage_shift) { > + case HPAGE_16GB_SHIFT: > + hv_pgsz_mask = HV_PGSZ_MASK_16GB; > + hv_pgsz_idx = HV_PGSZ_IDX_16GB; > + break; > case HPAGE_2GB_SHIFT: > hv_pgsz_mask = HV_PGSZ_MASK_2GB; > hv_pgsz_idx = HV_PGSZ_IDX_2GB; > @@ -376,6 +380,7 @@ void update_mmu_cache(struct vm_area_struct *vma, > unsigned long address, pte_t * > { > struct mm_struct *mm; > unsigned long flags; > + bool is_huge_tsb; > pte_t pte = *ptep; > > if (tlb_type != hypervisor) { > @@ -393,15 +398,37 @@ void update_mmu_cache(struct vm_area_struct *vma, > unsigned long address, pte_t * > > spin_lock_irqsave(&mm->context.lock, flags); > > + is_huge_tsb = false; > #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) > - if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) && > - is_hugetlb_pmd(__pmd(pte_val(pte)))) { > - /* We are fabricating 8MB pages using 4MB real hw pages. */ > - pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); > - __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, > - address, pte_val(pte)); > - } else > + if (mm->context.hugetlb_pte_count || mm->context.thp_pte_count) { > + unsigned long hugepage_size = PAGE_SIZE; > + > + if (is_vm_hugetlb_page(vma)) > + hugepage_size = huge_page_size(hstate_vma(vma)); > + > + if (hugepage_size >= PUD_SIZE) { > + unsigned long mask = 0x1ffc00000UL; > + > + /* Transfer bits [32:22] from address to resolve > + * at 4M granularity. > + */ > + pte_val(pte) &= ~mask; > + pte_val(pte) |= (address & mask); > + } else if (hugepage_size >= PMD_SIZE) { > + /* We are fabricating 8MB pages using 4MB > + * real hw pages. > + */ > + pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); > + } > + > + if (hugepage_size >= PMD_SIZE) { > + __update_mmu_tsb_insert(mm, MM_TSB_HUGE, > + REAL_HPAGE_SHIFT, address, pte_val(pte)); > + is_huge_tsb = true; > + } > + } > #endif > + if (!is_huge_tsb) > __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, > address, pte_val(pte)); > > -- > 2.9.2 >
