fs/dcache.c | 122 ++++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 95 insertions(+), 27 deletions(-) diff --git a/fs/dcache.c b/fs/dcache.c index 1bd4614ce93b..6443b07f8a3c 100644 --- a/fs/dcache.c +++ b/fs/dcache.c @@ -357,15 +357,74 @@ static void dentry_unlink_inode(struct dentry * dentry) } /* + * The DCACHE_LRU_LIST bit is set whenever the 'd_lru' entry + * is in use - which includes both the "real" per-superblock + * LRU list _and_ the DCACHE_SHRINK_LIST use. + * + * The DCACHE_SHRINK_LIST bit is set whenever the dentry is + * on the shrink list (ie not on the superblock LRU list). + * + * The per-cpu "nd_dentry_unused" counters are updated with + * the DCACHE_LRU_LIST bit. + * + * These helper functions make sure we always follow the + * rules. d_lock must be held by the caller. + */ +#define D_FLAG_VERIFY(dentry,x) WARN_ON_ONCE(((dentry)->d_flags & (DCACHE_LRU_LIST | DCACHE_SHRINK_LIST)) != (x)) +static void d_lru_add(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, 0); + dentry->d_flags |= DCACHE_LRU_LIST; + this_cpu_inc(nr_dentry_unused); + WARN_ON_ONCE(!list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); +} + +static void d_lru_del(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); + dentry->d_flags &= ~DCACHE_LRU_LIST; + this_cpu_dec(nr_dentry_unused); + WARN_ON_ONCE(!list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)); +} + +static void d_lru_isolate(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); + dentry->d_flags &= ~DCACHE_LRU_LIST; + this_cpu_dec(nr_dentry_unused); + list_del_init(&dentry->d_lru); +} + +static void d_shrink_del(struct dentry *dentry) +{ + D_FLAG_VERIFY(dentry, DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); + list_del_init(&dentry->d_lru); + dentry->d_flags &= ~(DCACHE_SHRINK_LIST | DCACHE_LRU_LIST); + this_cpu_dec(nr_dentry_unused); +} + +static void d_shrink_add(struct dentry *dentry, struct list_head *list) +{ + D_FLAG_VERIFY(dentry, 0); + list_add(&dentry->d_lru, list); + dentry->d_flags |= DCACHE_SHRINK_LIST | DCACHE_LRU_LIST; + this_cpu_inc(nr_dentry_unused); +} + +static void d_lru_shrink_move(struct dentry *dentry, struct list_head *list) +{ + D_FLAG_VERIFY(dentry, DCACHE_LRU_LIST); + dentry->d_flags |= DCACHE_SHRINK_LIST; + list_move_tail(&dentry->d_lru, list); +} + +/* * dentry_lru_(add|del)_list) must be called with d_lock held. */ static void dentry_lru_add(struct dentry *dentry) { - if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) { - if (list_lru_add(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)) - this_cpu_inc(nr_dentry_unused); - dentry->d_flags |= DCACHE_LRU_LIST; - } + if (unlikely(!(dentry->d_flags & DCACHE_LRU_LIST))) + d_lru_add(dentry); } /* @@ -377,15 +436,11 @@ static void dentry_lru_add(struct dentry *dentry) */ static void dentry_lru_del(struct dentry *dentry) { - if (dentry->d_flags & DCACHE_SHRINK_LIST) { - list_del_init(&dentry->d_lru); - dentry->d_flags &= ~DCACHE_SHRINK_LIST; - return; + if (dentry->d_flags & DCACHE_LRU_LIST) { + if (dentry->d_flags & DCACHE_SHRINK_LIST) + return d_shrink_del(dentry); + d_lru_del(dentry); } - - if (list_lru_del(&dentry->d_sb->s_dentry_lru, &dentry->d_lru)) - this_cpu_dec(nr_dentry_unused); - dentry->d_flags &= ~DCACHE_LRU_LIST; } /** @@ -837,6 +892,13 @@ static void shrink_dentry_list(struct list_head *list) dentry = list_entry_rcu(list->prev, struct dentry, d_lru); if (&dentry->d_lru == list) break; /* empty */ + + + /* + * Get the dentry lock, and re-verify that the dentry is + * this on the shrinking list. If it is, we know that + * DCACHE_SHRINK_LIST and DCACHE_LRU_LIST are set. + */ spin_lock(&dentry->d_lock); if (dentry != list_entry(list->prev, struct dentry, d_lru)) { spin_unlock(&dentry->d_lock); @@ -848,8 +910,7 @@ static void shrink_dentry_list(struct list_head *list) * to the LRU here, so we can simply remove it from the list * here regardless of whether it is referenced or not. */ - list_del_init(&dentry->d_lru); - dentry->d_flags &= ~DCACHE_SHRINK_LIST; + d_shrink_del(dentry); /* * We found an inuse dentry which was not removed from @@ -861,12 +922,20 @@ static void shrink_dentry_list(struct list_head *list) } rcu_read_unlock(); + /* + * If 'try_to_prune()' returns a dentry, it will + * be the same one we passed in, and d_lock will + * have been held the whole time, so it will not + * have been added to any other lists. We failed + * to get the inode lock. + * + * We just add it back to the shrink list. + */ dentry = try_prune_one_dentry(dentry); rcu_read_lock(); if (dentry) { - dentry->d_flags |= DCACHE_SHRINK_LIST; - list_add(&dentry->d_lru, list); + d_shrink_add(dentry, list); spin_unlock(&dentry->d_lock); } } @@ -894,7 +963,7 @@ dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg) * another pass through the LRU. */ if (dentry->d_lockref.count) { - list_del_init(&dentry->d_lru); + d_lru_isolate(dentry); spin_unlock(&dentry->d_lock); return LRU_REMOVED; } @@ -925,9 +994,7 @@ dentry_lru_isolate(struct list_head *item, spinlock_t *lru_lock, void *arg) return LRU_ROTATE; } - dentry->d_flags |= DCACHE_SHRINK_LIST; - list_move_tail(&dentry->d_lru, freeable); - this_cpu_dec(nr_dentry_unused); + d_lru_shrink_move(dentry, freeable); spin_unlock(&dentry->d_lock); return LRU_REMOVED; @@ -972,9 +1039,7 @@ static enum lru_status dentry_lru_isolate_shrink(struct list_head *item, if (!spin_trylock(&dentry->d_lock)) return LRU_SKIP; - dentry->d_flags |= DCACHE_SHRINK_LIST; - list_move_tail(&dentry->d_lru, freeable); - this_cpu_dec(nr_dentry_unused); + d_lru_shrink_move(dentry, freeable); spin_unlock(&dentry->d_lock); return LRU_REMOVED; @@ -1362,9 +1427,12 @@ static enum d_walk_ret select_collect(void *_data, struct dentry *dentry) if (dentry->d_lockref.count) { dentry_lru_del(dentry); } else if (!(dentry->d_flags & DCACHE_SHRINK_LIST)) { - dentry_lru_del(dentry); - list_add_tail(&dentry->d_lru, &data->dispose); - dentry->d_flags |= DCACHE_SHRINK_LIST; + /* + * If we get here, lockref.count is zero, and + * the dentry should be on the LRU list. So we + * can just use the "move" helper. + */ + d_lru_shrink_move(dentry, &data->dispose); data->found++; ret = D_WALK_NORETRY; }