[f2fs-dev] [PATCH] fsck: refactor build_nat_area_bitmap to speed up fsck

[Junling Zheng]

Refactor build_nat_area_bitmap, move out lookup_nat_in_journal from the
dual loops. Instead of looking up all nids in journals, we traverse nat
journals, whose entries are used to replace the NAT entries with the
corresponding nid to reduce the time build_nat_area_bitmap costs.

For "fsck -p 1", it reduces over half of the time.

Empty   Fragmented Entirely
32G(ms):
  original  625.8   727.3
  improved  226.3   298.2
64G(ms):
  original  11591637.7
  improved  515.8   676.7

Signed-off-by: Junling Zheng 
---
 fsck/mount.c | 118 ---
 1 file changed, 64 insertions(+), 54 deletions(-)

diff --git a/fsck/mount.c b/fsck/mount.c
index 21a96a7..9fcb008 100644
--- a/fsck/mount.c
+++ b/fsck/mount.c
@@ -1792,10 +1792,13 @@ void write_checkpoint(struct f2fs_sb_info *sbi)
 
 void build_nat_area_bitmap(struct f2fs_sb_info *sbi)
 {
+   struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_HOT_DATA);
+   struct f2fs_journal *journal = >sum_blk->journal;
struct f2fs_fsck *fsck = F2FS_FSCK(sbi);
struct f2fs_super_block *sb = F2FS_RAW_SUPER(sbi);
struct f2fs_nm_info *nm_i = NM_I(sbi);
struct f2fs_nat_block *nat_block;
+   struct node_info ni;
u32 nid, nr_nat_blks;
pgoff_t block_off;
pgoff_t block_addr;
@@ -1834,8 +1837,6 @@ void build_nat_area_bitmap(struct f2fs_sb_info *sbi)
 
nid = block_off * NAT_ENTRY_PER_BLOCK;
for (i = 0; i < NAT_ENTRY_PER_BLOCK; i++) {
-   struct f2fs_nat_entry raw_nat;
-   struct node_info ni;
ni.nid = nid + i;
 
if ((nid + i) == F2FS_NODE_INO(sbi) ||
@@ -1851,63 +1852,72 @@ void build_nat_area_bitmap(struct f2fs_sb_info *sbi)
continue;
}
 
-   if (lookup_nat_in_journal(sbi, nid + i,
-   _nat) >= 0) {
-   node_info_from_raw_nat(, _nat);
-   if ((ni.ino == 0x0 && ni.blk_addr != 0x0))
-   ASSERT_MSG("\tError: ino[0x%8x] or 
blk_addr[0x%16x]"
-   " is invalid\n",
-   ni.ino, ni.blk_addr);
-   if (ni.ino == (nid + i) && ni.blk_addr != 0) {
-   fsck->nat_valid_inode_cnt++;
-   DBG(3, "ino[0x%8x] maybe is inode\n",
-   ni.ino);
-   }
-   if (ni.blk_addr != 0x0) {
-   f2fs_set_bit(nid + i,
-   fsck->nat_area_bitmap);
-   fsck->chk.valid_nat_entry_cnt++;
-   DBG(3, "nid[0x%x] in nat cache\n",
-   nid + i);
-   }
+   node_info_from_raw_nat(, _block->entries[i]);
+   if (ni.blk_addr == 0x0)
+   continue;
+   if (ni.ino == 0x0) {
+   ASSERT_MSG("\tError: ino[0x%8x] or 
blk_addr[0x%16x]"
+   " is invalid\n", ni.ino, ni.blk_addr);
+   }
+   if (ni.ino == (nid + i)) {
+   fsck->nat_valid_inode_cnt++;
+   DBG(3, "ino[0x%8x] maybe is inode\n", ni.ino);
+   }
+   if (nid + i == 0) {
+   /*
+* nat entry [0] must be null.  If
+* it is corrupted, set its bit in
+* nat_area_bitmap, fsck_verify will
+* nullify it
+*/
+   ASSERT_MSG("Invalid nat entry[0]: "
+   "blk_addr[0x%x]\n", ni.blk_addr);
+   c.fix_on = 1;
+   fsck->chk.valid_nat_entry_cnt--;
+   }
 
-   fsck->entries[nid + i] = raw_nat;
-   } else {
-   node_info_from_raw_nat(,
-   _block->entries[i]);
-   if ((ni.ino == 0x0 && ni.blk_addr != 0x0))
-   ASSERT_MSG("\tError: ino[0x%8x] or 
blk_addr[0x%16x]"
-   " is invalid\n",
-   

Re: [f2fs-dev] [PATCH 2/2] fscrypto: don't use on-stack buffer for key derivation

[Richard Weinberger]

On 03.11.2016 23:03, Eric Biggers wrote:
> With the new (in 4.9) option to use a virtually-mapped stack
> (CONFIG_VMAP_STACK), stack buffers cannot be used as input/output for
> the scatterlist crypto API because they may not be directly mappable to
> struct page.  get_crypt_info() was using a stack buffer to hold the

See 1/2. :-)

> output from the encryption operation used to derive the per-file key.
> Fix it by using a heap buffer.
> 
> This bug could most easily be observed in a CONFIG_DEBUG_SG kernel
> because this allowed the BUG in sg_set_buf() to be triggered.
> 
> Signed-off-by: Eric Biggers 
> ---
>  fs/crypto/keyinfo.c | 16 +---
>  1 file changed, 13 insertions(+), 3 deletions(-)
> 
> diff --git a/fs/crypto/keyinfo.c b/fs/crypto/keyinfo.c
> index 82f0285..67fb6d8 100644
> --- a/fs/crypto/keyinfo.c
> +++ b/fs/crypto/keyinfo.c
> @@ -185,7 +185,7 @@ int get_crypt_info(struct inode *inode)
>   struct crypto_skcipher *ctfm;
>   const char *cipher_str;
>   int keysize;
> - u8 raw_key[FS_MAX_KEY_SIZE];
> + u8 *raw_key = NULL;
>   int res;
>  
>   res = fscrypt_initialize();
> @@ -238,6 +238,15 @@ int get_crypt_info(struct inode *inode)
>   if (res)
>   goto out;
>  
> + /*
> +  * This cannot be a stack buffer because it is passed to the scatterlist
> +  * crypto API as part of key derivation.
> +  */
> + res = -ENOMEM;
> + raw_key = kmalloc(FS_MAX_KEY_SIZE, GFP_NOFS);
> + if (!raw_key)
> + goto out;
> +
>   if (fscrypt_dummy_context_enabled(inode)) {
>   memset(raw_key, 0x42, FS_AES_256_XTS_KEY_SIZE);
>   goto got_key;
> @@ -276,7 +285,8 @@ int get_crypt_info(struct inode *inode)
>   if (res)
>   goto out;
>  
> - memzero_explicit(raw_key, sizeof(raw_key));
> + kzfree(raw_key);
> + raw_key = NULL;
>   if (cmpxchg(>i_crypt_info, NULL, crypt_info) != NULL) {
>   put_crypt_info(crypt_info);
>   goto retry;
> @@ -287,7 +297,7 @@ int get_crypt_info(struct inode *inode)
>   if (res == -ENOKEY)
>   res = 0;
>   put_crypt_info(crypt_info);
> - memzero_explicit(raw_key, sizeof(raw_key));
> + kzfree(raw_key);
>   return res;
>  }

Reviewed-by: Richard Weinberger 

Thanks,
//richard


--
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
___
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

[f2fs-dev] [PATCH] resize.f2fs: modify condition for resize

[Yunlei He]

it better to decide resize or not by sector num.

Signed-off-by: Yunlei He 
---
 fsck/main.c   | 4 ++--
 fsck/resize.c | 5 -
 2 files changed, 2 insertions(+), 7 deletions(-)

diff --git a/fsck/main.c b/fsck/main.c
index 7e8fdf3..64537cc 100644
--- a/fsck/main.c
+++ b/fsck/main.c
@@ -444,9 +444,9 @@ static int do_resize(struct f2fs_sb_info *sbi)
return -1;
}
 
-   if (c.target_sectors ==
+   if (c.target_sectors <=
(get_sb(block_count) << get_sb(log_sectors_per_block))) 
{
-   ASSERT_MSG("Nothing to resize; it's same");
+   ASSERT_MSG("Nothing to resize, now only support resize to 
expand\n");
return -1;
}
return f2fs_resize(sbi);
diff --git a/fsck/resize.c b/fsck/resize.c
index 7691524..46aa30e 100644
--- a/fsck/resize.c
+++ b/fsck/resize.c
@@ -579,11 +579,6 @@ int f2fs_resize(struct f2fs_sb_info *sbi)
end_blkaddr = (get_sb(segment_count_main) <<
get_sb(log_blocks_per_seg)) + get_sb(main_blkaddr);
 
-   if (old_main_blkaddr > new_main_blkaddr) {
-   MSG(0, "\tError: Support resize to expand only\n");
-   return -1;
-   }
-
err = -EAGAIN;
if (new_main_blkaddr < end_blkaddr) {
err = f2fs_defragment(sbi, old_main_blkaddr, offset,
-- 
2.10.1


--
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
___
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH 1/2] fscrypto: don't use on-stack buffer for filename encryption

[Richard Weinberger]

On 03.11.2016 23:03, Eric Biggers wrote:
> With the new (in 4.9) option to use a virtually-mapped stack
> (CONFIG_VMAP_STACK), stack buffers cannot be used as input/output for
> the scatterlist crypto API because they may not be directly mappable to
> struct page.  For short filenames, fname_encrypt() was encrypting a

As Kent and Andy pointed out, they are but here are dragons.
The pages can be non-linear and on platforms with different cache architectures
extra flush operations may be needed.

> stack buffer holding the padded filename.  Fix it by encrypting the
> filename in-place in the output buffer, thereby making the temporary
> buffer unnecessary.
> 
> This bug could most easily be observed in a CONFIG_DEBUG_SG kernel
> because this allowed the BUG in sg_set_buf() to be triggered.
> 
> Signed-off-by: Eric Biggers 
> ---
>  fs/crypto/fname.c | 53 +
>  1 file changed, 21 insertions(+), 32 deletions(-)
> 
> diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c
> index 9a28133..9b774f4 100644
> --- a/fs/crypto/fname.c
> +++ b/fs/crypto/fname.c
> @@ -39,65 +39,54 @@ static void fname_crypt_complete(struct 
> crypto_async_request *req, int res)
>  static int fname_encrypt(struct inode *inode,
>   const struct qstr *iname, struct fscrypt_str *oname)
>  {
> - u32 ciphertext_len;
>   struct skcipher_request *req = NULL;
>   DECLARE_FS_COMPLETION_RESULT(ecr);
>   struct fscrypt_info *ci = inode->i_crypt_info;
>   struct crypto_skcipher *tfm = ci->ci_ctfm;
>   int res = 0;
>   char iv[FS_CRYPTO_BLOCK_SIZE];
> - struct scatterlist src_sg, dst_sg;
> + struct scatterlist sg;
>   int padding = 4 << (ci->ci_flags & FS_POLICY_FLAGS_PAD_MASK);
> - char *workbuf, buf[32], *alloc_buf = NULL;
> - unsigned lim;
> + unsigned int lim;
> + unsigned int cryptlen;
>  
>   lim = inode->i_sb->s_cop->max_namelen(inode);
>   if (iname->len <= 0 || iname->len > lim)
>   return -EIO;
>  
> - ciphertext_len = max(iname->len, (u32)FS_CRYPTO_BLOCK_SIZE);
> - ciphertext_len = round_up(ciphertext_len, padding);
> - ciphertext_len = min(ciphertext_len, lim);
> + /*
> +  * Copy the filename to the output buffer for encrypting in-place and
> +  * pad it with the needed number of NUL bytes.
> +  */
> + cryptlen = max_t(unsigned int, iname->len, FS_CRYPTO_BLOCK_SIZE);
> + cryptlen = round_up(cryptlen, padding);
> + cryptlen = min(cryptlen, lim);
> + memcpy(oname->name, iname->name, iname->len);
> + memset(oname->name + iname->len, 0, cryptlen - iname->len);
>  
> - if (ciphertext_len <= sizeof(buf)) {
> - workbuf = buf;
> - } else {
> - alloc_buf = kmalloc(ciphertext_len, GFP_NOFS);
> - if (!alloc_buf)
> - return -ENOMEM;
> - workbuf = alloc_buf;
> - }
> + /* Initialize the IV */
> + memset(iv, 0, FS_CRYPTO_BLOCK_SIZE);

You can initialize it with iv = {0} at the beginning such that you don't
need the memset here.

> - /* Allocate request */
> + /* Set up the encryption request */
>   req = skcipher_request_alloc(tfm, GFP_NOFS);
>   if (!req) {
>   printk_ratelimited(KERN_ERR
> - "%s: crypto_request_alloc() failed\n", __func__);
> - kfree(alloc_buf);
> + "%s: skcipher_request_alloc() failed\n", __func__);
>   return -ENOMEM;
>   }
>   skcipher_request_set_callback(req,
>   CRYPTO_TFM_REQ_MAY_BACKLOG | CRYPTO_TFM_REQ_MAY_SLEEP,
>   fname_crypt_complete, );
> + sg_init_one(, oname->name, cryptlen);
> + skcipher_request_set_crypt(req, , , cryptlen, iv);
>  
> - /* Copy the input */
> - memcpy(workbuf, iname->name, iname->len);
> - if (iname->len < ciphertext_len)
> - memset(workbuf + iname->len, 0, ciphertext_len - iname->len);
> -
> - /* Initialize IV */
> - memset(iv, 0, FS_CRYPTO_BLOCK_SIZE);
> -
> - /* Create encryption request */
> - sg_init_one(_sg, workbuf, ciphertext_len);
> - sg_init_one(_sg, oname->name, ciphertext_len);
> - skcipher_request_set_crypt(req, _sg, _sg, ciphertext_len, iv);
> + /* Do the encryption */
>   res = crypto_skcipher_encrypt(req);
>   if (res == -EINPROGRESS || res == -EBUSY) {
> + /* Request is being completed asynchronously; wait for it */
>   wait_for_completion();
>   res = ecr.res;
>   }
> - kfree(alloc_buf);
>   skcipher_request_free(req);
>   if (res < 0) {
>   printk_ratelimited(KERN_ERR
> @@ -105,7 +94,7 @@ static int fname_encrypt(struct inode *inode,
>   return res;
>   }
>  
> - oname->len = ciphertext_len;
> + oname->len = cryptlen;
>   return 0;
>  }

Reviewed-by: Richard Weinberger 

[f2fs-dev] [PATCH v4] f2fs:return directly if block has been removed from the victim

[Yunlei He]

If one block has been to written to a new place, just return
in move data process. This patch check it again with holding
page lock.

Signed-off-by: Yunlei He 
---
 fs/f2fs/gc.c | 16 
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
index 9c18917..a7b2c13 100644
--- a/fs/f2fs/gc.c
+++ b/fs/f2fs/gc.c
@@ -544,7 +544,8 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct 
f2fs_summary *sum,
return true;
 }
 
-static void move_encrypted_block(struct inode *inode, block_t bidx)
+static void move_encrypted_block(struct inode *inode, block_t bidx,
+   unsigned int segno, int 
off)
 {
struct f2fs_io_info fio = {
.sbi = F2FS_I_SB(inode),
@@ -565,6 +566,9 @@ static void move_encrypted_block(struct inode *inode, 
block_t bidx)
if (!page)
return;
 
+   if (!check_valid_map(F2FS_I_SB(inode), segno, off))
+   goto out;
+
set_new_dnode(, inode, NULL, NULL, 0);
err = get_dnode_of_data(, bidx, LOOKUP_NODE);
if (err)
@@ -645,7 +649,8 @@ static void move_encrypted_block(struct inode *inode, 
block_t bidx)
f2fs_put_page(page, 1);
 }
 
-static void move_data_page(struct inode *inode, block_t bidx, int gc_type)
+static void move_data_page(struct inode *inode, block_t bidx, int gc_type,
+   unsigned int segno, int 
off)
 {
struct page *page;
 
@@ -653,6 +658,9 @@ static void move_data_page(struct inode *inode, block_t 
bidx, int gc_type)
if (IS_ERR(page))
return;
 
+   if (!check_valid_map(F2FS_I_SB(inode), segno, off))
+   goto out;
+
if (gc_type == BG_GC) {
if (PageWriteback(page))
goto out;
@@ -796,9 +804,9 @@ static void gc_data_segment(struct f2fs_sb_info *sbi, 
struct f2fs_summary *sum,
start_bidx = start_bidx_of_node(nofs, inode)
+ ofs_in_node;
if (f2fs_encrypted_inode(inode) && 
S_ISREG(inode->i_mode))
-   move_encrypted_block(inode, start_bidx);
+   move_encrypted_block(inode, start_bidx, segno, 
off);
else
-   move_data_page(inode, start_bidx, gc_type);
+   move_data_page(inode, start_bidx, gc_type, 
segno, off);
 
if (locked) {
up_write(>dio_rwsem[WRITE]);
-- 
2.10.1


--
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
___
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel

Re: [f2fs-dev] [PATCH v3] f2fs:return directly if block has been removed from the victim

[Chao Yu]

On 2016/11/3 10:06, Yunlei He wrote:
> If one block has been to written to a new place, just return
> in move data process. This patch check it again with holding
> page lock.
> 
> Signed-off-by: Yunlei He 
> ---
>  fs/f2fs/gc.c | 16 
>  1 file changed, 12 insertions(+), 4 deletions(-)
> 
> diff --git a/fs/f2fs/gc.c b/fs/f2fs/gc.c
> index 9c18917..b00bcf9 100644
> --- a/fs/f2fs/gc.c
> +++ b/fs/f2fs/gc.c
> @@ -544,7 +544,8 @@ static bool is_alive(struct f2fs_sb_info *sbi, struct 
> f2fs_summary *sum,
>   return true;
>  }
>  
> -static void move_encrypted_block(struct inode *inode, block_t bidx)
> +static void move_encrypted_block(struct inode *inode, block_t bidx,
> + unsigned int segno, int 
> off)
>  {
>   struct f2fs_io_info fio = {
>   .sbi = F2FS_I_SB(inode),
> @@ -579,6 +580,9 @@ static void move_encrypted_block(struct inode *inode, 
> block_t bidx)
>* don't cache encrypted data into meta inode until previous dirty
>* data were writebacked to avoid racing between GC and flush.
>*/
> + if (!check_valid_map(F2FS_I_SB(inode), segno, off))
> + goto put_out;

Please do this check after f2fs_grab_cache_page like we do in move_data_page.
Also it will keeps above comments being consistent with below codes.

Thanks,

> +
>   f2fs_wait_on_page_writeback(page, DATA, true);
>  
>   get_node_info(fio.sbi, dn.nid, );
> @@ -645,7 +649,8 @@ static void move_encrypted_block(struct inode *inode, 
> block_t bidx)
>   f2fs_put_page(page, 1);
>  }
>  
> -static void move_data_page(struct inode *inode, block_t bidx, int gc_type)
> +static void move_data_page(struct inode *inode, block_t bidx, int gc_type,
> + unsigned int segno, int 
> off)
>  {
>   struct page *page;
>  
> @@ -653,6 +658,9 @@ static void move_data_page(struct inode *inode, block_t 
> bidx, int gc_type)
>   if (IS_ERR(page))
>   return;
>  
> + if (!check_valid_map(F2FS_I_SB(inode), segno, off))
> + goto out;
> +
>   if (gc_type == BG_GC) {
>   if (PageWriteback(page))
>   goto out;
> @@ -796,9 +804,9 @@ static void gc_data_segment(struct f2fs_sb_info *sbi, 
> struct f2fs_summary *sum,
>   start_bidx = start_bidx_of_node(nofs, inode)
>   + ofs_in_node;
>   if (f2fs_encrypted_inode(inode) && 
> S_ISREG(inode->i_mode))
> - move_encrypted_block(inode, start_bidx);
> + move_encrypted_block(inode, start_bidx, segno, 
> off);
>   else
> - move_data_page(inode, start_bidx, gc_type);
> + move_data_page(inode, start_bidx, gc_type, 
> segno, off);
>  
>   if (locked) {
>   up_write(>dio_rwsem[WRITE]);
> 


--
Developer Access Program for Intel Xeon Phi Processors
Access to Intel Xeon Phi processor-based developer platforms.
With one year of Intel Parallel Studio XE.
Training and support from Colfax.
Order your platform today. http://sdm.link/xeonphi
___
Linux-f2fs-devel mailing list
Linux-f2fs-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel