From: "David Hildenbrand (Arm)" <[email protected]>
The default helpers fallback to pud_offset(), passing a pointer to the
stack value.
Let's provide variants that do exactly what (pgd|p4d|pud)_offset() do,
but ignore the passed pgd_t/p4d_t/pud_t, consequently not working on a
pointer to the stack value like the default handlers would.
With this patch, Drop the arm64 custom variant that did that. As it documents:
Note: reusing the original pointer means that it may be
dereferenced the same (live) page-table entry multiple times.
This is safe because it is still only loaded once in the
context of each level and the CPU guarantees same-address
read-after-read ordering.
Do not document this, as the double read will be removed in the next patch.
Signed-off-by: David Hildenbrand (Arm) <[email protected]>
---
arch/arm64/include/asm/pgtable.h | 23 -----------------------
include/asm-generic/pgtable-nop4d.h | 7 +++++++
include/asm-generic/pgtable-nopmd.h | 7 +++++++
include/asm-generic/pgtable-nopud.h | 7 +++++++
4 files changed, 21 insertions(+), 23 deletions(-)
diff --git a/arch/arm64/include/asm/pgtable.h b/arch/arm64/include/asm/pgtable.h
index e89ec5f4787b..a3c882e46777 100644
--- a/arch/arm64/include/asm/pgtable.h
+++ b/arch/arm64/include/asm/pgtable.h
@@ -1180,29 +1180,6 @@ static inline bool pgtable_l5_enabled(void) { return
false; }
#define p4d_clear_fixmap()
#define p4d_offset_kimg(dir,addr) ((p4d_t *)dir)
-
-static inline
-p4d_t *p4d_offset_lockless_folded(pgd_t *pgdp, pgd_t pgd, unsigned long addr)
-{
- /*
- * With runtime folding of the pud, pud_offset_lockless() passes
- * the 'pgd_t *' we return here to p4d_to_folded_pud(), which
- * will offset the pointer assuming that it points into
- * a page-table page. However, the fast GUP path passes us a
- * pgd_t allocated on the stack and so we must use the original
- * pointer in 'pgdp' to construct the p4d pointer instead of
- * using the generic p4d_offset_lockless() implementation.
- *
- * Note: reusing the original pointer means that we may
- * dereference the same (live) page-table entry multiple times.
- * This is safe because it is still only loaded once in the
- * context of each level and the CPU guarantees same-address
- * read-after-read ordering.
- */
- return p4d_offset(pgdp, addr);
-}
-#define p4d_offset_lockless p4d_offset_lockless_folded
-
#endif /* CONFIG_PGTABLE_LEVELS > 4 */
#define pgd_set_fixmap(addr) ((pgd_t *)set_fixmap_offset(FIX_PGD, addr))
diff --git a/include/asm-generic/pgtable-nop4d.h
b/include/asm-generic/pgtable-nop4d.h
index c6a5a43899b5..019c3f074b77 100644
--- a/include/asm-generic/pgtable-nop4d.h
+++ b/include/asm-generic/pgtable-nop4d.h
@@ -39,6 +39,13 @@ static inline p4d_t *p4d_offset(pgd_t *pgd, unsigned long
address)
return (p4d_t *)pgd;
}
+static inline p4d_t *p4d_offset_lockless(pgd_t *pgdp, pgd_t pgd,
+ unsigned long address)
+{
+ return (p4d_t *)pgdp;
+}
+#define p4d_offset_lockless p4d_offset_lockless
+
#define p4d_val(x) (pgd_val((x).pgd))
#define __p4d(x) ((p4d_t) { __pgd(x) })
diff --git a/include/asm-generic/pgtable-nopmd.h
b/include/asm-generic/pgtable-nopmd.h
index 4e5aaf253b6a..1dd5b165234b 100644
--- a/include/asm-generic/pgtable-nopmd.h
+++ b/include/asm-generic/pgtable-nopmd.h
@@ -49,6 +49,13 @@ static inline pmd_t * pmd_offset(pud_t * pud, unsigned long
address)
}
#define pmd_offset pmd_offset
+static inline pmd_t *pmd_offset_lockless(pud_t *pudp, pud_t pud,
+ unsigned long address)
+{
+ return (pmd_t *)pudp;
+}
+#define pmd_offset_lockless pmd_offset_lockless
+
#define pmd_val(x) (pud_val((x).pud))
#define __pmd(x) ((pmd_t) { __pud(x) } )
diff --git a/include/asm-generic/pgtable-nopud.h
b/include/asm-generic/pgtable-nopud.h
index 6c9bca78047c..5a2b0a81ae19 100644
--- a/include/asm-generic/pgtable-nopud.h
+++ b/include/asm-generic/pgtable-nopud.h
@@ -47,6 +47,13 @@ static inline pud_t *pud_offset(p4d_t *p4d, unsigned long
address)
}
#define pud_offset pud_offset
+static inline pud_t *pud_offset_lockless(p4d_t *p4dp, p4d_t p4d,
+ unsigned long address)
+{
+ return (pud_t *)p4dp;
+}
+#define pud_offset_lockless pud_offset_lockless
+
#define pud_val(x) (p4d_val((x).p4d))
#define __pud(x) ((pud_t) { __p4d(x) })
--
2.43.0
