On Wed, Nov 11, 2020 at 12:22:46PM +0100, Peter Zijlstra wrote:
> Trying to match the Code: to PageHuge as generate here makes this the
> PageCompound() test burning, not even compound_head() going bad.
>
> must ponder more...
Oooh.. does this help?
---
kernel/events/core.c | 81 +++++++++++++++++++++++++++++++---------------------
1 file changed, 49 insertions(+), 32 deletions(-)
diff --git a/kernel/events/core.c b/kernel/events/core.c
index d2f3ca792936..3b42576c99f1 100644
--- a/kernel/events/core.c
+++ b/kernel/events/core.c
@@ -7015,65 +7015,82 @@ static u64 perf_virt_to_phys(u64 virt)
*/
__weak u64 arch_perf_get_page_size(struct mm_struct *mm, unsigned long addr)
{
+#ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
struct page *page;
- pgd_t *pgd;
- p4d_t *p4d;
- pud_t *pud;
- pmd_t *pmd;
- pte_t *pte;
-
- pgd = pgd_offset(mm, addr);
- if (pgd_none(*pgd))
+ pgd_t *pgdp, pgd;
+ p4d_t *p4dp, p4d;
+ pud_t *pudp, pud;
+ pmd_t *pmdp, pmd;
+ pte_t *ptep, pte;
+
+ pgdp = pgd_offset(mm, addr);
+ pgd = READ_ONCE(*pgdp);
+ if (pgd_none(pgd))
return 0;
- p4d = p4d_offset(pgd, addr);
- if (!p4d_present(*p4d))
+ p4dp = p4d_offset(&pgd, addr);
+ p4d = READ_ONCE(*p4dp);
+ if (!p4d_present(p4d))
return 0;
- if (p4d_leaf(*p4d))
+ if (p4d_leaf(p4d))
return 1ULL << P4D_SHIFT;
- pud = pud_offset(p4d, addr);
- if (!pud_present(*pud))
+ pudp = pud_offset(&p4d, addr);
+ pud = READ_ONCE(*pudp);
+ if (!pud_present(pud))
return 0;
- if (pud_leaf(*pud)) {
+ if (pud_leaf(pud)) {
#ifdef pud_page
- page = pud_page(*pud);
- if (PageHuge(page))
- return page_size(compound_head(page));
+ if (!pud_devmap(pud)) {
+ page = pud_page(pud);
+ if (PageHuge(page))
+ return page_size(compound_head(page));
+ }
#endif
return 1ULL << PUD_SHIFT;
}
- pmd = pmd_offset(pud, addr);
- if (!pmd_present(*pmd))
+ pmdp = pmd_offset(&pud, addr);
+ pmd = READ_ONCE(*pmdp);
+ if (!pmd_present(pmd))
return 0;
- if (pmd_leaf(*pmd)) {
+ if (pmd_leaf(pmd)) {
#ifdef pmd_page
- page = pmd_page(*pmd);
- if (PageHuge(page))
- return page_size(compound_head(page));
+ if (!pmd_devmap(pmd)) {
+ page = pmd_page(pmd);
+ if (PageHuge(page))
+ return page_size(compound_head(page));
+ }
#endif
return 1ULL << PMD_SHIFT;
}
- pte = pte_offset_map(pmd, addr);
- if (!pte_present(*pte)) {
- pte_unmap(pte);
+ ptep = pte_offset_map(&pmd, addr);
+ pte = READ_ONCE(*ptep); // gup_get_pte()
+ if (!pte_present(pte)) {
+ pte_unmap(ptep);
return 0;
}
- page = pte_page(*pte);
- if (PageHuge(page)) {
- u64 size = page_size(compound_head(page));
- pte_unmap(pte);
- return size;
+ if (!pte_devmap(pte) && !pte_special(pte)) {
+ page = pte_page(pte);
+ if (PageHuge(page)) {
+ u64 size = page_size(compound_head(page));
+ pte_unmap(ptep);
+ return size;
+ }
}
- pte_unmap(pte);
+ pte_unmap(ptep);
return PAGE_SIZE;
+
+#else /* CONFIG_ARCH_HAS_PTE_SPECIAL */
+
+ return 0;
+#endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
}
#else
