Signed-off-by: Gao Xiang <[email protected]>
---
fs/erofs/internal.h | 61 ++++++++++++++++
fs/erofs/super.c | 79 ++++++++++++++++++++-
fs/erofs/unzip_vle.c | 197 +++++++++++++++++++++++++++++++++++++++++++++++----
fs/erofs/unzip_vle.h | 8 ---
4 files changed, 323 insertions(+), 22 deletions(-)
diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h
index 8d7b6ab..edba0da 100644
--- a/fs/erofs/internal.h
+++ b/fs/erofs/internal.h
@@ -59,6 +59,22 @@ struct erofs_fault_info {
};
#endif
+#ifdef CONFIG_EROFS_FS_ZIP_BIDIRECTIONAL
+#define EROFS_FS_ZIP_CACHE_LVL (2)
+#elif defined(CONFIG_EROFS_FS_ZIP_UNIDIRECTIONAL)
+#define EROFS_FS_ZIP_CACHE_LVL (1)
+#else
+#define EROFS_FS_ZIP_CACHE_LVL (0)
+#endif
+
+#if (!defined(EROFS_FS_HAS_MANAGED_CACHE) && (EROFS_FS_ZIP_CACHE_LVL > 0))
+#define EROFS_FS_HAS_MANAGED_CACHE
+
+#define EROFS_UNALLOCATED_CACHED_PAGE ((void *)0x5F0EF00D)
+
+extern int try_to_free_cached_page(struct address_space *, struct page *);
+#endif
+
/* EROFS_SUPER_MAGIC_V1 to represent the whole file system */
#define EROFS_SUPER_MAGIC EROFS_SUPER_MAGIC_V1
@@ -88,6 +104,11 @@ struct erofs_sb_info {
spinlock_t lock;
#endif
} workstn;
+
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ struct inode *managed_cache;
+#endif
+
#endif
u32 build_time_nsec;
@@ -176,13 +197,53 @@ struct erofs_workgroup {
atomic_t refcount;
};
+#define EROFS_LOCKED_MAGIC (INT_MIN | 0xE0F510CCL)
+
+static inline bool erofs_workgroup_try_to_freeze(
+ struct erofs_workgroup *grp, int v)
+{
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+ if (v != atomic_cmpxchg(&grp->refcount,
+ v, EROFS_LOCKED_MAGIC))
+ return false;
+ preempt_disable();
+#else
+ preempt_disable();
+ if (atomic_read(&grp->refcount) != v) {
+ preempt_enable();
+ return false;
+ }
+#endif
+ return true;
+}
+
+static inline void erofs_workgroup_unfreeze(
+ struct erofs_workgroup *grp, int v)
+{
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+ atomic_set(&grp->refcount, v);
+#endif
+ preempt_enable();
+}
+
static inline bool erofs_workgroup_get(struct erofs_workgroup *grp, int *ocnt)
{
+ const int locked = (int)EROFS_LOCKED_MAGIC;
int o;
repeat:
o = atomic_read(&grp->refcount);
+ /* spin if it is temporarily locked at the reclaim path */
+ if (unlikely(o == locked)) {
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK)
+ do
+ cpu_relax();
+ while (atomic_read(&grp->refcount) == locked);
+#endif
+ goto repeat;
+ }
+
if (unlikely(o <= 0))
return -1;
diff --git a/fs/erofs/super.c b/fs/erofs/super.c
index 546a308..fc4d750 100644
--- a/fs/erofs/super.c
+++ b/fs/erofs/super.c
@@ -245,6 +245,67 @@ static int parse_options(struct super_block *sb, char
*options)
return 0;
}
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+
+static const struct address_space_operations managed_cache_aops;
+
+static int managed_cache_releasepage(struct page *page, gfp_t gfp_mask)
+{
+ int ret = 1; /* 0 - busy */
+ struct address_space *const mapping = page->mapping;
+
+ BUG_ON(!PageLocked(page));
+ BUG_ON(mapping->a_ops != &managed_cache_aops);
+
+ if (PagePrivate(page))
+ ret = try_to_free_cached_page(mapping, page);
+
+ return ret;
+}
+
+static void managed_cache_invalidatepage(struct page *page,
+ unsigned int offset, unsigned int length)
+{
+ const unsigned int stop = length + offset;
+
+ BUG_ON(!PageLocked(page));
+
+ /* Check for overflow */
+ BUG_ON(stop > PAGE_SIZE || stop < length);
+
+ if (offset == 0 && stop == PAGE_SIZE)
+ while(!managed_cache_releasepage(page, GFP_NOFS))
+ cond_resched();
+}
+
+static const struct address_space_operations managed_cache_aops = {
+ .releasepage = managed_cache_releasepage,
+ .invalidatepage = managed_cache_invalidatepage,
+};
+
+struct inode *erofs_init_managed_cache(struct super_block *sb)
+{
+ struct inode *inode = new_inode(sb);
+
+ if (unlikely(inode == NULL))
+ return ERR_PTR(-ENOMEM);
+
+ set_nlink(inode, 1);
+ inode->i_size = OFFSET_MAX;
+
+ inode->i_mapping->a_ops = &managed_cache_aops;
+ mapping_set_gfp_mask(inode->i_mapping,
+ GFP_NOFS | __GFP_HIGHMEM |
+ __GFP_MOVABLE | __GFP_NOFAIL
+#if defined(CONFIG_CMA) && defined(___GFP_CMA)
+ | ___GFP_CMA
+#endif
+ );
+ return inode;
+}
+
+#endif
+
static int erofs_read_super(struct super_block *sb,
const char *dev_name, void *data, int silent)
{
@@ -299,11 +360,19 @@ static int erofs_read_super(struct super_block *sb,
#endif
#endif
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ sbi->managed_cache = erofs_init_managed_cache(sb);
+ if (sbi->managed_cache == NULL) {
+ err = -ENOMEM;
+ goto err_sbi;
+ }
+#endif
+
/* get the root inode */
inode = erofs_iget(sb, ROOT_NID(sbi), true);
if (IS_ERR(inode)) {
err = PTR_ERR(inode);
- goto err_sbi;
+ goto iget_err;
}
if (!S_ISDIR(inode->i_mode)) {
@@ -346,6 +415,10 @@ static int erofs_read_super(struct super_block *sb,
err_iput:
if (sb->s_root == NULL)
iput(inode);
+iget_err:
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ iput(sbi->managed_cache);
+#endif
err_sbi:
sb->s_fs_info = NULL;
kfree(sbi);
@@ -368,6 +441,10 @@ static void erofs_put_super(struct super_block *sb)
infoln("unmounted for %s", sbi->dev_name);
__putname(sbi->dev_name);
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ iput(sbi->managed_cache);
+#endif
+
mutex_lock(&sbi->umount_mutex);
#ifdef CONFIG_EROFS_FS_ZIP
diff --git a/fs/erofs/unzip_vle.c b/fs/erofs/unzip_vle.c
index 7ca1d5d..0920851 100644
--- a/fs/erofs/unzip_vle.c
+++ b/fs/erofs/unzip_vle.c
@@ -77,6 +77,107 @@ struct z_erofs_vle_work_handler {
#define VLE_WORK_HANDLER_INIT() \
{ .curr = NULL, .role = Z_EROFS_VLE_WORK_PRIMARY_OWNER }
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+int try_to_free_cached_page(struct address_space *mapping, struct page *page)
+{
+ struct inode *inode = mapping->host;
+ unsigned clusterpages = erofs_clusterpages(EROFS_SB(inode->i_sb));
+ struct z_erofs_vle_workgroup *grp = (void *)page_private(page);
+ int i, ret = 0;
+
+ /* prevent the workgroup from being freed */
+ rcu_read_lock();
+
+ if (!erofs_workgroup_try_to_freeze(&grp->obj, 1))
+ goto out;
+
+ for(i = 0; i < clusterpages; ++i) {
+ if (grp->compressed_pages[i] == page) {
+ WRITE_ONCE(grp->compressed_pages[i], NULL);
+ ret = 1;
+ break;
+ }
+ }
+ erofs_workgroup_unfreeze(&grp->obj, 1);
+out:
+ rcu_read_unlock();
+
+ if (ret) {
+ ClearPagePrivate(page);
+ put_page(page);
+ }
+ return ret;
+}
+
+static inline void grab_compressed_pages_in_managed_cache(
+ struct super_block *sb,
+ pgoff_t start,
+ struct page **compressed_pages,
+ int clusterpages, bool alloc_reserve)
+{
+ pgoff_t cur;
+#if 0
+ struct page *pages[clusterpages];
+#endif
+ unsigned i, j, found;
+
+ for (i = 0; i < clusterpages; ++i) {
+ if (READ_ONCE(compressed_pages[i]) == NULL) {
+ cur = (start += i);
+ compressed_pages += i;
+ clusterpages -= i;
+ goto hitted;
+ }
+ }
+
+ return;
+hitted:
+#if 0
+ found = find_get_pages_range(EROFS_SB(sb)->managed_cache->i_mapping,
+ &cur, start + clusterpages, clusterpages, pages);
+#endif
+
+ i = 0;
+ for (j = 0; i < clusterpages && j < found; ++i) {
+ struct page *page;
+
+#if 0
+ cur = pages[j]->index - start;
+ if (cur != i) {
+ if (!alloc_reserve)
+ continue;
+ page = EROFS_UNALLOCATED_CACHED_PAGE;
+ } else
+ page = pages[j++];
+#else
+ page = find_get_page(EROFS_SB(sb)->managed_cache->i_mapping,
+ start + j);
+ if (page == NULL) {
+ if (!alloc_reserve)
+ continue;
+ page = EROFS_UNALLOCATED_CACHED_PAGE;
+ }
+#endif
+
+ if (cmpxchg(compressed_pages + cur, NULL, page) == NULL)
+ continue;
+
+ if (cur == i)
+ put_page(page);
+ }
+
+ if (!alloc_reserve)
+ return;
+
+ while (i < clusterpages) {
+ cmpxchg(compressed_pages + i, NULL,
+ EROFS_UNALLOCATED_CACHED_PAGE);
+ ++i;
+ }
+}
+
+#endif
+
/* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */
static inline bool try_to_reuse_as_compressed_page(
struct z_erofs_vle_work_handler *w,
@@ -308,6 +409,13 @@ static inline void __update_workgrp_llen(struct
z_erofs_vle_workgroup *grp,
}
}
+#define handler_is_owner(w) ((w)->role >= Z_EROFS_VLE_WORK_PRIMARY_OWNER)
+
+struct z_erofs_vle_frontend {
+ bool initial;
+ erofs_off_t cachedzone_la;
+};
+
static int z_erofs_vle_work_iter_begin(struct z_erofs_vle_work_handler *w,
struct super_block *sb,
struct erofs_map_blocks *map,
@@ -386,8 +494,6 @@ void z_erofs_vle_work_release(struct z_erofs_vle_work *work)
erofs_workgroup_put(&grp->obj);
}
-#define handler_is_owner(w) ((w)->role >= Z_EROFS_VLE_WORK_PRIMARY_OWNER)
-
static inline void
z_erofs_vle_work_iter_end(struct z_erofs_vle_work_handler *w)
{
@@ -408,7 +514,8 @@ void z_erofs_vle_work_release(struct z_erofs_vle_work *work)
w->curr = NULL;
}
-static int z_erofs_do_read_page(struct page *page,
+static int z_erofs_do_read_page(struct z_erofs_vle_frontend *fe,
+ struct page *page,
struct z_erofs_vle_work_handler *h,
struct erofs_map_blocks_iter *m,
z_erofs_vle_owned_workgrp_t *owned_head,
@@ -458,6 +565,14 @@ static int z_erofs_do_read_page(struct page *page,
if (unlikely(err))
goto err_out;
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ grab_compressed_pages_in_managed_cache(sb,
+ m->map.m_pa / EROFS_BLKSIZ, h->compressed_pages,
+ h->compressed_deficit, fe->initial || m->map.m_la <=
fe->cachedzone_la);
+
+ fe->initial = false;
+#endif
+
owned &= handler_is_owner(h);
work = h->curr;
hitted:
@@ -542,6 +657,7 @@ static inline void z_erofs_vle_read_endio(struct bio *bio)
{
unsigned i;
struct bio_vec *bvec;
+
#if (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 13, 0))
const int err = bio->bi_status;
#elif (LINUX_VERSION_CODE >= KERNEL_VERSION(4, 3, 0))
@@ -550,16 +666,25 @@ static inline void z_erofs_vle_read_endio(struct bio *bio)
bio_for_each_segment_all(bvec, bio, i) {
struct page *page = bvec->bv_page;
+ bool cachedpage = false;
DBG_BUGON(PageUptodate(page));
+
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ if (page->mapping != NULL) {
+ struct inode *inode = page->mapping->host;
+
+ cachedpage = (inode ==
+ EROFS_SB(inode->i_sb)->managed_cache);
+ }
+#endif
+
if (unlikely(err))
SetPageError(page);
-
- /* FIXME: the following snippets are for cached work */
- else if (0)
+ else if (cachedpage)
SetPageUptodate(page);
- if (0)
+ if (cachedpage)
unlock_page(page);
}
@@ -574,7 +699,8 @@ static int z_erofs_vle_unzip(struct super_block *sb,
struct z_erofs_vle_workgroup *grp,
struct list_head *page_pool)
{
- unsigned clusterpages = erofs_clusterpages(EROFS_SB(sb));
+ struct erofs_sb_info *sbi = EROFS_SB(sb);
+ unsigned clusterpages = erofs_clusterpages(sbi);
struct z_erofs_pagevec_ctor ctor;
unsigned nr_pages;
#ifndef CONFIG_EROFS_FS_ZIP_MULTIREF
@@ -666,6 +792,10 @@ static int z_erofs_vle_unzip(struct super_block *sb,
if (page->mapping == NULL)
continue;
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ if (page->mapping->host == sbi->managed_cache)
+ continue;
+#endif
pagenr = z_erofs_onlinepage_index(page);
@@ -747,6 +877,10 @@ static int z_erofs_vle_unzip(struct super_block *sb,
if (page->mapping == NULL)
list_add(&page->lru, page_pool);
+#ifdef EROFS_FS_HAS_MANAGED_CACHE
+ else if (page->mapping->host == sbi->managed_cache)
+ continue;
+#endif
WRITE_ONCE(compressed_pages[i], NULL);
}
@@ -841,7 +975,8 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
struct z_erofs_vle_unzip_io *io)
{
struct bio *bio = NULL;
- unsigned clusterpages = erofs_clusterpages(EROFS_SB(sb));
+ struct erofs_sb_info *sbi = EROFS_SB(sb);
+ unsigned clusterpages = erofs_clusterpages(sbi);
pgoff_t last_page;
bool sync;
unsigned bios_submitted;
@@ -878,15 +1013,34 @@ static bool z_erofs_vle_submit_all(struct super_block
*sb,
/* fulfill all compressed pages */
for (i = 0; i < clusterpages; ++i) {
struct page *page;
+ struct page *old = READ_ONCE(compressed_pages[i]);
+ bool cached = (old == EROFS_UNALLOCATED_CACHED_PAGE);
- if (READ_ONCE(compressed_pages[i]) != NULL)
+ if (old != NULL && !cached)
continue;
page = erofs_allocpage(page_pool, GFP_KERNEL);
page->mapping = NULL;
+ if (cached) {
+ if (add_to_page_cache_lru(page,
+ sbi->managed_cache->i_mapping,
+ grp->obj.index + i, GFP_KERNEL))
+ cached = false;
+ else {
+ if (cmpxchg(compressed_pages + i,
+ old, page) == old) {
+ set_page_private(page,
(unsigned long)grp);
+ SetPagePrivate(page);
+ } else {
+ put_page(page);
+ }
+ continue;
+ }
+ }
- if (cmpxchg(compressed_pages + i, NULL, page) != NULL)
+ if (cmpxchg(compressed_pages + i, old, page) != old) {
list_add(&page->lru, page_pool);
+ }
}
current_page = grp->obj.index;
@@ -898,6 +1052,19 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
bio = NULL;
}
repeat:
+ if (compressed_pages[i]->mapping ==
sbi->managed_cache->i_mapping) {
+ if (PageUptodate(compressed_pages[i])) {
+ if (++i < clusterpages) {
+ if (bio != NULL)
+ goto submit_bio_retry;
+ else
+ goto repeat;
+ }
+ continue;
+ }
+ BUG_ON(!PageLocked(compressed_pages[i]));
+ }
+
if (bio == NULL) {
bio = prepare_bio(sb, current_page,
BIO_MAX_PAGES, z_erofs_vle_read_endio);
@@ -927,6 +1094,8 @@ static bool z_erofs_vle_submit_all(struct super_block *sb,
static int z_erofs_vle_normalaccess_readpage(struct file *file,
struct page *page)
{
+ struct z_erofs_vle_frontend fe = { .initial = true,
+ .cachedzone_la = page->index << PAGE_SHIFT };
struct erofs_map_blocks_iter m_iter = {
.map = { .m_llen = 0, .m_plen = 0 },
.mpage = NULL
@@ -937,7 +1106,7 @@ static int z_erofs_vle_normalaccess_readpage(struct file
*file,
struct z_erofs_vle_unzip_io io;
LIST_HEAD(pagepool);
- int err = z_erofs_do_read_page(page,
+ int err = z_erofs_do_read_page(&fe, page,
&h, &m_iter, &owned_head, &pagepool);
z_erofs_vle_work_iter_end(&h);
@@ -971,6 +1140,8 @@ static inline int __z_erofs_vle_normalaccess_readpages(
struct address_space *mapping,
struct list_head *pages, unsigned nr_pages, bool sync)
{
+ struct z_erofs_vle_frontend fe = { .initial = true,
+ .cachedzone_la = lru_to_page(pages)->index << PAGE_SHIFT };
struct erofs_map_blocks_iter m_iter = {
.map = { .m_llen = 0, .m_plen = 0 },
.mpage = NULL
@@ -1006,7 +1177,7 @@ static inline int __z_erofs_vle_normalaccess_readpages(
/* traversal in reverse order */
head = (void *)page_private(page);
- err = z_erofs_do_read_page(page,
+ err = z_erofs_do_read_page(&fe, page,
&h, &m_iter, &owned_head, &pagepool);
if (err) {
struct erofs_vnode *vi = EROFS_V(inode);
diff --git a/fs/erofs/unzip_vle.h b/fs/erofs/unzip_vle.h
index 2a446f9..91e4a80 100644
--- a/fs/erofs/unzip_vle.h
+++ b/fs/erofs/unzip_vle.h
@@ -16,14 +16,6 @@
#include "internal.h"
#include "unzip_pagevec.h"
-#ifdef CONFIG_EROFS_FS_ZIP_BIDIRECTIONAL
-#define EROFS_FS_ZIP_CACHE_LVL (2)
-#elif defined(CONFIG_EROFS_FS_ZIP_UNIDIRECTIONAL)
-#define EROFS_FS_ZIP_CACHE_LVL (1)
-#else
-#define EROFS_FS_ZIP_CACHE_LVL (0)
-#endif
-
/*
* Structure fields follow one of the following exclusion rules.
*
--
1.9.1
