--- fs/fuse/dev.c | 12 ++++--- fs/fuse/file.c | 4 +- fs/fuse/inode.c | 3 + include/linux/freezer.h | 19 ++++++++++++ include/linux/gfp.h | 4 +- include/linux/mutex.h | 2 + include/linux/page-flags.h | 2 + include/linux/pagemap.h | 8 +++-- kernel/freezer.c | 2 - kernel/mutex.c | 10 +++++- mm/filemap.c | 70 +++++++++++++++++++++++++++++++++++++++------ mm/page_alloc.c | 6 +++ 12 files changed, 121 insertions(+), 21 deletions(-) --- a/fs/fuse/dev.c +++ b/fs/fuse/dev.c @@ -19,6 +19,7 @@ #include #include #include +#include MODULE_ALIAS_MISCDEV(FUSE_MINOR); MODULE_ALIAS("devname:fuse"); @@ -106,7 +107,7 @@ struct fuse_req *fuse_get_req(struct fus atomic_inc(&fc->num_waiting); block_sigs(&oldset); - intr = wait_event_interruptible(fc->blocked_waitq, !fc->blocked); + intr = wait_event_freeze_intr(fc->blocked_waitq, !fc->blocked); restore_sigs(&oldset); err = -EINTR; if (intr) @@ -143,7 +144,8 @@ static struct fuse_req *get_reserved_req struct fuse_file *ff = file->private_data; do { - wait_event(fc->reserved_req_waitq, ff->reserved_req); + wait_event_freeze_nonintr(fc->reserved_req_waitq, + ff->reserved_req); spin_lock(&fc->lock); if (ff->reserved_req) { req = ff->reserved_req; @@ -191,7 +193,7 @@ struct fuse_req *fuse_get_req_nofail(str struct fuse_req *req; atomic_inc(&fc->num_waiting); - wait_event(fc->blocked_waitq, !fc->blocked); + wait_event_freeze_nonintr(fc->blocked_waitq, !fc->blocked); req = fuse_request_alloc(); if (!req) req = get_reserved_req(fc, file); @@ -330,7 +332,7 @@ __acquires(fc->lock) return; spin_unlock(&fc->lock); - wait_event_interruptible(req->waitq, req->state == FUSE_REQ_FINISHED); + wait_event_freeze_intr(req->waitq, req->state == FUSE_REQ_FINISHED); spin_lock(&fc->lock); } @@ -386,7 +388,7 @@ __acquires(fc->lock) * Wait it out. */ spin_unlock(&fc->lock); - wait_event(req->waitq, req->state == FUSE_REQ_FINISHED); + wait_event_freeze_nonintr(req->waitq, req->state == FUSE_REQ_FINISHED); spin_lock(&fc->lock); if (!req->aborted) --- a/fs/fuse/file.c +++ b/fs/fuse/file.c @@ -15,6 +15,7 @@ #include #include #include +#include static const struct file_operations fuse_direct_io_file_operations; @@ -349,7 +350,8 @@ static int fuse_wait_on_page_writeback(s { struct fuse_inode *fi = get_fuse_inode(inode); - wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); + wait_event_freeze_nonintr(fi->page_waitq, + !fuse_page_is_writeback(inode, index)); return 0; } --- a/fs/fuse/inode.c +++ b/fs/fuse/inode.c @@ -290,6 +290,8 @@ struct inode *fuse_iget(struct super_blo inode->i_flags |= S_NOATIME|S_NOCMTIME; inode->i_generation = generation; inode->i_data.backing_dev_info = &fc->bdi; + inode->i_data.flags |= __GFP_FREEZABLE; + inode->i_mutex.freezable = true; fuse_init_inode(inode, attr); unlock_new_inode(inode); } else if ((inode->i_mode ^ attr->mode) & S_IFMT) { @@ -1055,6 +1057,7 @@ static int fuse_fill_super(struct super_ list_add_tail(&fc->entry, &fuse_conn_list); sb->s_root = root_dentry; + sb->s_vfs_rename_mutex.freezable = true; fc->connected = 1; file->private_data = fuse_conn_get(fc); mutex_unlock(&fuse_mutex); --- a/include/linux/freezer.h +++ b/include/linux/freezer.h @@ -163,6 +163,22 @@ static inline bool freezer_should_skip(s * defined in */ +#define wait_event_freeze_nonintr(wq, condition) \ +({ \ + freezer_do_not_count(); \ + wait_event(wq, (condition)); \ + freezer_count(); \ +}) + +#define wait_event_freeze_intr(wq, condition) \ +({ \ + int __retval; \ + freezer_do_not_count(); \ + __retval = wait_event_interruptible(wq, (condition)); \ + freezer_count(); \ + __retval; \ +}) + #define wait_event_freezekillable(wq, condition) \ ({ \ int __retval; \ @@ -232,6 +248,9 @@ static inline void set_freezable(void) { #define wait_event_freezekillable(wq, condition) \ wait_event_killable(wq, condition) +#define wait_event_freeze_nonintr(wq, condition) \ + wait_event(wq, condition) + #endif /* !CONFIG_FREEZER */ #endif /* FREEZER_H_INCLUDED */ --- a/include/linux/gfp.h +++ b/include/linux/gfp.h @@ -35,6 +35,7 @@ struct vm_area_struct; #define ___GFP_NO_KSWAPD 0x400000u #define ___GFP_OTHER_NODE 0x800000u #define ___GFP_WRITE 0x1000000u +#define ___GFP_FREEZABLE 0x2000000u /* If the above are modified, __GFP_BITS_SHIFT may need updating */ /* @@ -92,6 +93,7 @@ struct vm_area_struct; #define __GFP_OTHER_NODE ((__force gfp_t)___GFP_OTHER_NODE) /* On behalf of other node */ #define __GFP_KMEMCG ((__force gfp_t)___GFP_KMEMCG) /* Allocation comes from a memcg-accounted resource */ #define __GFP_WRITE ((__force gfp_t)___GFP_WRITE) /* Allocator intends to dirty page */ +#define __GFP_FREEZABLE ((__force gfp_t)___GFP_FREEZABLE) /* Allocate freezable page */ /* * This may seem redundant, but it's a way of annotating false positives vs. @@ -99,7 +101,7 @@ struct vm_area_struct; */ #define __GFP_NOTRACK_FALSE_POSITIVE (__GFP_NOTRACK) -#define __GFP_BITS_SHIFT 25 /* Room for N __GFP_FOO bits */ +#define __GFP_BITS_SHIFT 26 /* Room for N __GFP_FOO bits */ #define __GFP_BITS_MASK ((__force gfp_t)((1 << __GFP_BITS_SHIFT) - 1)) /* This equals 0, but use constants in case they ever change */ --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -50,6 +50,7 @@ struct mutex { atomic_t count; spinlock_t wait_lock; struct list_head wait_list; + bool freezable; #if defined(CONFIG_DEBUG_MUTEXES) || defined(CONFIG_SMP) struct task_struct *owner; #endif @@ -106,6 +107,7 @@ static inline void mutex_destroy(struct { .count = ATOMIC_INIT(1) \ , .wait_lock = __SPIN_LOCK_UNLOCKED(lockname.wait_lock) \ , .wait_list = LIST_HEAD_INIT(lockname.wait_list) \ + , .freezable = false \ __DEBUG_MUTEX_INITIALIZER(lockname) \ __DEP_MAP_MUTEX_INITIALIZER(lockname) } --- a/include/linux/page-flags.h +++ b/include/linux/page-flags.h @@ -109,6 +109,7 @@ enum pageflags { #ifdef CONFIG_TRANSPARENT_HUGEPAGE PG_compound_lock, #endif + PG_freezable, __NR_PAGEFLAGS, /* Filesystems */ @@ -208,6 +209,7 @@ PAGEFLAG(Pinned, pinned) TESTSCFLAG(Pinn PAGEFLAG(SavePinned, savepinned); /* Xen */ PAGEFLAG(Reserved, reserved) __CLEARPAGEFLAG(Reserved, reserved) PAGEFLAG(SwapBacked, swapbacked) __CLEARPAGEFLAG(SwapBacked, swapbacked) +PAGEFLAG(Freezable, freezable) __PAGEFLAG(SlobFree, slob_free) --- a/include/linux/pagemap.h +++ b/include/linux/pagemap.h @@ -382,12 +382,14 @@ static inline int lock_page_or_retry(str */ extern void wait_on_page_bit(struct page *page, int bit_nr); -extern int wait_on_page_bit_killable(struct page *page, int bit_nr); +extern void __wait_on_page_locked(struct page *page); + +extern int __wait_on_page_locked_killable(struct page *page); static inline int wait_on_page_locked_killable(struct page *page) { if (PageLocked(page)) - return wait_on_page_bit_killable(page, PG_locked); + return __wait_on_page_locked_killable(page); return 0; } @@ -401,7 +403,7 @@ static inline int wait_on_page_locked_ki static inline void wait_on_page_locked(struct page *page) { if (PageLocked(page)) - wait_on_page_bit(page, PG_locked); + __wait_on_page_locked(page); } /* --- a/kernel/freezer.c +++ b/kernel/freezer.c @@ -111,7 +111,7 @@ bool freeze_task(struct task_struct *p) unsigned long flags; spin_lock_irqsave(&freezer_lock, flags); - if (!freezing(p) || frozen(p)) { + if (!freezing(p) || frozen(p) || freezer_should_skip(p)) { spin_unlock_irqrestore(&freezer_lock, flags); return false; } --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ __mutex_init(struct mutex *lock, const c spin_lock_init(&lock->wait_lock); INIT_LIST_HEAD(&lock->wait_list); mutex_clear_owner(lock); + lock->freezable = false; debug_mutex_init(lock, name, key); } @@ -240,7 +242,13 @@ __mutex_lock_common(struct mutex *lock, /* didn't get the lock, go to sleep: */ spin_unlock_mutex(&lock->wait_lock, flags); - schedule_preempt_disabled(); + if (likely(!lock->freezable)) { + schedule_preempt_disabled(); + } else { + sched_preempt_enable_no_resched(); + freezable_schedule(); + preempt_disable(); + } spin_lock_mutex(&lock->wait_lock, flags); } --- a/mm/filemap.c +++ b/mm/filemap.c @@ -33,6 +33,7 @@ #include /* for BUG_ON(!in_atomic()) only */ #include #include +#include #include "internal.h" /* @@ -544,15 +545,46 @@ void wait_on_page_bit(struct page *page, } EXPORT_SYMBOL(wait_on_page_bit); -int wait_on_page_bit_killable(struct page *page, int bit_nr) +void __wait_on_page_locked(struct page *page) { - DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); + DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); + + if (PageLocked(page)) { + if (likely(!PageFreezable(page))) { + __wait_on_bit(page_waitqueue(page), &wait, + sleep_on_page, + TASK_UNINTERRUPTIBLE); + } else { + freezer_do_not_count(); + __wait_on_bit(page_waitqueue(page), &wait, + sleep_on_page, + TASK_UNINTERRUPTIBLE); + freezer_count(); + } + } +} +EXPORT_SYMBOL(__wait_on_page_locked); + +int __wait_on_page_locked_killable(struct page *page) +{ + DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); - if (!test_bit(bit_nr, &page->flags)) + if (!PageLocked(page)) return 0; - return __wait_on_bit(page_waitqueue(page), &wait, - sleep_on_page_killable, TASK_KILLABLE); + if (likely(!PageFreezable(page))) { + return __wait_on_bit(page_waitqueue(page), &wait, + sleep_on_page_killable, TASK_KILLABLE); + } else { + int ret; + + freezer_do_not_count(); + ret = __wait_on_bit(page_waitqueue(page), &wait, + sleep_on_page_killable, TASK_KILLABLE); + freezer_count(); + + return ret; + } } /** @@ -619,8 +651,15 @@ void __lock_page(struct page *page) { DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); - __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page, - TASK_UNINTERRUPTIBLE); + if (likely(!PageFreezable(page))) { + __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page, + TASK_UNINTERRUPTIBLE); + } else { + freezer_do_not_count(); + __wait_on_bit_lock(page_waitqueue(page), &wait, sleep_on_page, + TASK_UNINTERRUPTIBLE); + freezer_count(); + } } EXPORT_SYMBOL(__lock_page); @@ -628,8 +667,21 @@ int __lock_page_killable(struct page *pa { DEFINE_WAIT_BIT(wait, &page->flags, PG_locked); - return __wait_on_bit_lock(page_waitqueue(page), &wait, - sleep_on_page_killable, TASK_KILLABLE); + if (likely(!PageFreezable(page))) { + return __wait_on_bit_lock(page_waitqueue(page), &wait, + sleep_on_page_killable, + TASK_KILLABLE); + } else { + int ret; + + freezer_do_not_count(); + ret = __wait_on_bit_lock(page_waitqueue(page), &wait, + sleep_on_page_killable, + TASK_KILLABLE); + freezer_count(); + + return ret; + } } EXPORT_SYMBOL_GPL(__lock_page_killable); --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -532,6 +532,9 @@ static inline void __free_one_page(struc unsigned long uninitialized_var(buddy_idx); struct page *buddy; + if (unlikely(PageFreezable(page))) + ClearPageFreezable(page); + if (unlikely(PageCompound(page))) if (unlikely(destroy_compound_page(page, order))) return; @@ -2628,6 +2631,8 @@ __alloc_pages_nodemask(gfp_t gfp_mask, u goto retry_cpuset; memcg_kmem_commit_charge(page, memcg, order); + if (unlikely(gfp_mask & __GFP_FREEZABLE)) + SetPageFreezable(page); return page; } @@ -6107,6 +6112,7 @@ static const struct trace_print_flags pa #ifdef CONFIG_TRANSPARENT_HUGEPAGE {1UL << PG_compound_lock, "compound_lock" }, #endif + {1UL << PG_freezable, "freezable" }, }; static void dump_page_flags(unsigned long flags)