On Wed, Aug 14, 2013 at 11:50:14AM +0200, Ingo Molnar wrote: > > > > Well, I'll have to check if it really doesn't generate additional > > instructions in generated code, since it's hotpath. I'll ping back once > > things are done. > > An __always_inline should never do that.
Here is the final one, please ping me if something looks not as clean as it wanted to be and i'll tune code up, thanks! To hpa@: I had to use explicit @mask because it allows to pass -1ul mask which is optimized off then by a compiler. --- From: Cyrill Gorcunov <[email protected]> Subject: [PATCH -mm] mm: Unify pte_to_pgoff and pgoff_to_pte helpers Use unified pte_bfop helper to manipulate bits in pte/pgoff bitfield, and convert pte_to_pgoff/pgoff_to_pte to inlines. Signed-off-by: Cyrill Gorcunov <[email protected]> Cc: Andrew Morton <[email protected]> Cc: Andy Lutomirski <[email protected]> Cc: "H. Peter Anvin" <[email protected]> Cc: Pavel Emelyanov <[email protected]> Cc: Matt Mackall <[email protected]> Cc: Xiao Guangrong <[email protected]> Cc: Marcelo Tosatti <[email protected]> Cc: KOSAKI Motohiro <[email protected]> Cc: Stephen Rothwell <[email protected]> Cc: Peter Zijlstra <[email protected]> Cc: "Aneesh Kumar K.V" <[email protected]> --- arch/x86/include/asm/pgtable-2level.h | 70 +++++++++++++++++++++------------- 1 file changed, 44 insertions(+), 26 deletions(-) Index: linux-2.6.git/arch/x86/include/asm/pgtable-2level.h =================================================================== --- linux-2.6.git.orig/arch/x86/include/asm/pgtable-2level.h +++ linux-2.6.git/arch/x86/include/asm/pgtable-2level.h @@ -55,8 +55,12 @@ static inline pmd_t native_pmdp_get_and_ #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp) #endif -#define _mfrob(v,r,m,l) ((((v) >> (r)) & (m)) << (l)) -#define __frob(v,r,l) (((v) >> (r)) << (l)) +/* Bit manipulation helper on pte/pgoff entry */ +static inline unsigned long pte_bfop(unsigned long value, unsigned int rightshift, + unsigned long mask, unsigned int leftshift) +{ + return ((value >> rightshift) & mask) << leftshift; +} #ifdef CONFIG_MEM_SOFT_DIRTY @@ -82,19 +86,26 @@ static inline pmd_t native_pmdp_get_and_ #define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2) #define PTE_FILE_LSHIFT4 (PTE_FILE_BITS1 + PTE_FILE_BITS2 + PTE_FILE_BITS3) -#define pte_to_pgoff(pte) \ - (_mfrob((pte).pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + \ - _mfrob((pte).pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + \ - _mfrob((pte).pte_low, PTE_FILE_SHIFT3, PTE_FILE_MASK3, PTE_FILE_LSHIFT3) + \ - __frob((pte).pte_low, PTE_FILE_SHIFT4, PTE_FILE_LSHIFT4)) - -#define pgoff_to_pte(off) \ - ((pte_t) { .pte_low = \ - _mfrob(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + \ - _mfrob(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + \ - _mfrob(off, PTE_FILE_LSHIFT3, PTE_FILE_MASK3, PTE_FILE_SHIFT3) + \ - __frob(off, PTE_FILE_LSHIFT4, PTE_FILE_SHIFT4) + \ - _PAGE_FILE }) +static __always_inline pgoff_t pte_to_pgoff(pte_t pte) +{ + return (pgoff_t) + (pte_bfop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + + pte_bfop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + + pte_bfop(pte.pte_low, PTE_FILE_SHIFT3, PTE_FILE_MASK3, PTE_FILE_LSHIFT3) + + pte_bfop(pte.pte_low, PTE_FILE_SHIFT4, -1UL, PTE_FILE_LSHIFT4)); +} + +static __always_inline pte_t pgoff_to_pte(pgoff_t off) +{ + return (pte_t){ + .pte_low = + pte_bfop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + + pte_bfop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + + pte_bfop(off, PTE_FILE_LSHIFT3, PTE_FILE_MASK3, PTE_FILE_SHIFT3) + + pte_bfop(off, PTE_FILE_LSHIFT4, -1UL, PTE_FILE_SHIFT4) + + _PAGE_FILE, + }; +} #else /* CONFIG_MEM_SOFT_DIRTY */ @@ -120,17 +131,24 @@ static inline pmd_t native_pmdp_get_and_ #define PTE_FILE_LSHIFT2 (PTE_FILE_BITS1) #define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2) -#define pte_to_pgoff(pte) \ - (_mfrob((pte).pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + \ - _mfrob((pte).pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + \ - __frob((pte).pte_low, PTE_FILE_SHIFT3, PTE_FILE_LSHIFT3)) - -#define pgoff_to_pte(off) \ - ((pte_t) { .pte_low = \ - _mfrob(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + \ - _mfrob(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + \ - __frob(off, PTE_FILE_LSHIFT3, PTE_FILE_SHIFT3) + \ - _PAGE_FILE }) +static __always_inline pgoff_t pte_to_pgoff(pte_t pte) +{ + return (pgoff_t) + (pte_bfop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + + pte_bfop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + + pte_bfop(pte.pte_low, PTE_FILE_SHIFT3, -1UL, PTE_FILE_LSHIFT3)); +} + +static __always_inline pte_t pgoff_to_pte(pgoff_t off) +{ + return (pte_t){ + .pte_low = + pte_bfop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + + pte_bfop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + + pte_bfop(off, PTE_FILE_LSHIFT3, -1UL, PTE_FILE_SHIFT3) + + _PAGE_FILE, + }; +} #endif /* CONFIG_MEM_SOFT_DIRTY */ -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
