CONFIG_LOCKDEP_PAGELOCK needs to keep lockdep_map_cross per page. Since
it's a debug feature, it's preferred to keep it in struct page_ext than
struct page. Move it to struct page_ext.
Signed-off-by: Byungchul Park
---
include/linux/mm_types.h | 4 ---
include/linux/page-flags.h | 19 +++--
include/linux/page_ext.h | 4 +++
include/linux/pagemap.h| 28 ---
lib/Kconfig.debug | 1 +
mm/filemap.c | 69 ++
mm/page_alloc.c| 3 --
mm/page_ext.c | 4 +++
8 files changed, 118 insertions(+), 14 deletions(-)
diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index f1e3dba..ac3121c 100644
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -220,10 +220,6 @@ struct page {
#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
int _last_cpupid;
#endif
-
-#ifdef CONFIG_LOCKDEP_PAGELOCK
- struct lockdep_map_cross map;
-#endif
}
/*
* The struct page can be forced to be double word aligned so that atomic ops
diff --git a/include/linux/page-flags.h b/include/linux/page-flags.h
index b793342..879dd0d 100644
--- a/include/linux/page-flags.h
+++ b/include/linux/page-flags.h
@@ -374,28 +374,41 @@ static __always_inline int PageSwapCache(struct page
*page)
#ifdef CONFIG_LOCKDEP_PAGELOCK
#include
+#include
TESTPAGEFLAG(Locked, locked, PF_NO_TAIL)
static __always_inline void __SetPageLocked(struct page *page)
{
+ struct page_ext *e;
+
__set_bit(PG_locked, &PF_NO_TAIL(page, 1)->flags);
page = compound_head(page);
- lock_acquire_exclusive((struct lockdep_map *)&page->map, 0, 1, NULL,
_RET_IP_);
+ e = lookup_page_ext(page);
+ if (unlikely(!e))
+ return;
+
+ lock_acquire_exclusive((struct lockdep_map *)&e->map, 0, 1, NULL,
_RET_IP_);
}
static __always_inline void __ClearPageLocked(struct page *page)
{
+ struct page_ext *e;
+
__clear_bit(PG_locked, &PF_NO_TAIL(page, 1)->flags);
page = compound_head(page);
+ e = lookup_page_ext(page);
+ if (unlikely(!e))
+ return;
+
/*
* lock_commit_crosslock() is necessary for crosslock
* when the lock is released, before lock_release().
*/
- lock_commit_crosslock((struct lockdep_map *)&page->map);
- lock_release((struct lockdep_map *)&page->map, 0, _RET_IP_);
+ lock_commit_crosslock((struct lockdep_map *)&e->map);
+ lock_release((struct lockdep_map *)&e->map, 0, _RET_IP_);
}
#else
__PAGEFLAG(Locked, locked, PF_NO_TAIL)
diff --git a/include/linux/page_ext.h b/include/linux/page_ext.h
index 9298c39..d1c52c8c 100644
--- a/include/linux/page_ext.h
+++ b/include/linux/page_ext.h
@@ -44,6 +44,10 @@ enum page_ext_flags {
*/
struct page_ext {
unsigned long flags;
+
+#ifdef CONFIG_LOCKDEP_PAGELOCK
+ struct lockdep_map_cross map;
+#endif
};
extern void pgdat_page_ext_init(struct pglist_data *pgdat);
diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
index 9f448c6..b75b8bc 100644
--- a/include/linux/pagemap.h
+++ b/include/linux/pagemap.h
@@ -16,6 +16,7 @@
#include
#ifdef CONFIG_LOCKDEP_PAGELOCK
#include
+#include
#endif
/*
@@ -454,28 +455,47 @@ static inline pgoff_t linear_page_index(struct
vm_area_struct *vma,
}
#ifdef CONFIG_LOCKDEP_PAGELOCK
+extern struct page_ext_operations lockdep_pagelock_ops;
+
#define lock_page_init(p) \
do { \
static struct lock_class_key __key; \
- lockdep_init_map_crosslock((struct lockdep_map *)&(p)->map, \
+ struct page_ext *e = lookup_page_ext(p);\
+ \
+ if (unlikely(!e)) \
+ break; \
+ \
+ lockdep_init_map_crosslock((struct lockdep_map *)&(e)->map, \
"(PG_locked)" #p, &__key, 0); \
} while (0)
static inline void lock_page_acquire(struct page *page, int try)
{
+ struct page_ext *e;
+
page = compound_head(page);
- lock_acquire_exclusive((struct lockdep_map *)&page->map, 0,
+ e = lookup_page_ext(page);
+ if (unlikely(!e))
+ return;
+
+ lock_acquire_exclusive((struct lockdep_map *)&e->map, 0,
try, NULL, _RET_IP_);
}
static inline void lock_page_release(struct page *page)
{
+ struct page_ext *e;
+
page = compound_head(page);
+ e = lookup_page_ext(page);
+ if (unlikely(!e))
+ return;
+
/*
* lock_commit_crosslock() is necessary for crosslocks.
*/
- lock_commit_crosslock((struct lo
