On 06/20, Chao Yu wrote:
> On 2019/6/20 1:26, Jaegeuk Kim wrote:
> > On 06/18, Chao Yu wrote:
> >> On 2019/6/14 10:46, Jaegeuk Kim wrote:
> >>> On 06/11, Chao Yu wrote:
> >>>> On 2019/6/5 2:36, Jaegeuk Kim wrote:
> >>>>> Two paths to update quota and f2fs_lock_op:
> >>>>>
> >>>>> 1.
> >>>>> - lock_op
> >>>>> | - quota_update
> >>>>> `- unlock_op
> >>>>>
> >>>>> 2.
> >>>>> - quota_update
> >>>>> - lock_op
> >>>>> `- unlock_op
> >>>>>
> >>>>> But, we need to make a transaction on quota_update + lock_op in #2 case.
> >>>>> So, this patch introduces:
> >>>>> 1. lock_op
> >>>>> 2. down_write
> >>>>> 3. check __need_flush
> >>>>> 4. up_write
> >>>>> 5. if there is dirty quota entries, flush them
> >>>>> 6. otherwise, good to go
> >>>>>
> >>>>> Signed-off-by: Jaegeuk Kim <[email protected]>
> >>>>> ---
> >>>>>
> >>>>> v3 from v2:
> >>>>> - refactor to fix quota corruption issue
> >>>>> : it seems that the previous scenario is not real and no deadlock
> >>>>> case was
> >>>>> encountered.
> >>>>
> >>>> - f2fs_dquot_commit
> >>>> - down_read(&sbi->quota_sem)
> >>>> - block_operation
> >>>> - f2fs_lock_all
> >>>> - need_flush_quota
> >>>> - down_write(&sbi->quota_sem)
> >>>> - f2fs_quota_write
> >>>> - f2fs_lock_op
> >>>>
> >>>> Why can't this happen?
> >>>>
> >>>> Once more question, should we hold quota_sem during checkpoint to avoid
> >>>> further
> >>>> quota update? f2fs_lock_op can do this job as well?
> >>>
> >>> I couldn't find write_dquot() call to make this happen, and f2fs_lock_op
> >>> was not
> >>
> >> - f2fs_dquot_commit
> >> - dquot_commit
> >> ->commit_dqblk (v2_write_dquot)
> >> - qtree_write_dquot
> >> ->quota_write (f2fs_quota_write)
> >> - f2fs_lock_op
> >>
> >> Do you mean there is no such way that calling f2fs_lock_op() from
> >> f2fs_quota_write()? So that deadlock condition is not existing?
> >
> > I mean write_dquot->f2fs_dquot_commit and block_operation seems not racing
> > together.
>
> quota ioctl has the path calling write_dquot->f2fs_dquot_commit as below,
> which
> can race with checkpoint().
>
> - do_quotactl
> - sb->s_qcop->quota_sync (f2fs_quota_sync)
> - down_read(&sbi->quota_sem); ---- First
> - dquot_writeback_dquots
> - sb->dq_op->write_dquot (f2fs_dquot_commit)
> - block_operation can
> race here
> - down_read(&sbi->quota_sem); ---- Second
Adding f2fs_lock_op() in f2fs_quota_sync() should be fine?
>
> Thanks,
>
> >
> >>
> >> Thanks,
> >>
> >>> enough to cover quota updates. Current stress & power-cut tests are
> >>> running for
> >>> several days without problem with this patch.
> >>>
> >>>>
> >>>> Thanks,
> >>>>
> >>>>>
> >>>>> fs/f2fs/checkpoint.c | 41 +++++++++++++++++++----------------------
> >>>>> fs/f2fs/f2fs.h | 1 +
> >>>>> fs/f2fs/super.c | 26 +++++++++++++++++++++-----
> >>>>> 3 files changed, 41 insertions(+), 27 deletions(-)
> >>>>>
> >>>>> diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
> >>>>> index 89825261d474..43f65f0962e5 100644
> >>>>> --- a/fs/f2fs/checkpoint.c
> >>>>> +++ b/fs/f2fs/checkpoint.c
> >>>>> @@ -1131,17 +1131,24 @@ static void __prepare_cp_block(struct
> >>>>> f2fs_sb_info *sbi)
> >>>>>
> >>>>> static bool __need_flush_quota(struct f2fs_sb_info *sbi)
> >>>>> {
> >>>>> + bool ret = false;
> >>>>> +
> >>>>> if (!is_journalled_quota(sbi))
> >>>>> return false;
> >>>>> - if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH))
> >>>>> - return false;
> >>>>> - if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR))
> >>>>> - return false;
> >>>>> - if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH))
> >>>>> - return true;
> >>>>> - if (get_pages(sbi, F2FS_DIRTY_QDATA))
> >>>>> - return true;
> >>>>> - return false;
> >>>>> +
> >>>>> + down_write(&sbi->quota_sem);
> >>>>> + if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
> >>>>> + ret = false;
> >>>>> + } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
> >>>>> + ret = false;
> >>>>> + } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
> >>>>> + clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
> >>>>> + ret = true;
> >>>>> + } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
> >>>>> + ret = true;
> >>>>> + }
> >>>>> + up_write(&sbi->quota_sem);
> >>>>> + return ret;
> >>>>> }
> >>>>>
> >>>>> /*
> >>>>> @@ -1160,26 +1167,22 @@ static int block_operations(struct f2fs_sb_info
> >>>>> *sbi)
> >>>>> blk_start_plug(&plug);
> >>>>>
> >>>>> retry_flush_quotas:
> >>>>> + f2fs_lock_all(sbi);
> >>>>> if (__need_flush_quota(sbi)) {
> >>>>> int locked;
> >>>>>
> >>>>> if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) {
> >>>>> set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
> >>>>> - f2fs_lock_all(sbi);
> >>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
> >>>>> goto retry_flush_dents;
> >>>>> }
> >>>>> - clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
> >>>>> + f2fs_unlock_all(sbi);
> >>>>>
> >>>>> /* only failed during mount/umount/freeze/quotactl */
> >>>>> locked = down_read_trylock(&sbi->sb->s_umount);
> >>>>> f2fs_quota_sync(sbi->sb, -1);
> >>>>> if (locked)
> >>>>> up_read(&sbi->sb->s_umount);
> >>>>> - }
> >>>>> -
> >>>>> - f2fs_lock_all(sbi);
> >>>>> - if (__need_flush_quota(sbi)) {
> >>>>> - f2fs_unlock_all(sbi);
> >>>>> cond_resched();
> >>>>> goto retry_flush_quotas;
> >>>>> }
> >>>>> @@ -1201,12 +1204,6 @@ static int block_operations(struct f2fs_sb_info
> >>>>> *sbi)
> >>>>> */
> >>>>> down_write(&sbi->node_change);
> >>>>>
> >>>>> - if (__need_flush_quota(sbi)) {
> >>>>> - up_write(&sbi->node_change);
> >>>>> - f2fs_unlock_all(sbi);
> >>>>> - goto retry_flush_quotas;
> >>>>> - }
> >>>>> -
> >>>>> if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
> >>>>> up_write(&sbi->node_change);
> >>>>> f2fs_unlock_all(sbi);
> >>>>> diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
> >>>>> index 9674a85154b2..9bd2bf0f559b 100644
> >>>>> --- a/fs/f2fs/f2fs.h
> >>>>> +++ b/fs/f2fs/f2fs.h
> >>>>> @@ -1253,6 +1253,7 @@ struct f2fs_sb_info {
> >>>>> block_t unusable_block_count; /* # of blocks saved by
> >>>>> last cp */
> >>>>>
> >>>>> unsigned int nquota_files; /* # of quota sysfile */
> >>>>> + struct rw_semaphore quota_sem; /* blocking cp for
> >>>>> flags */
> >>>>>
> >>>>> /* # of pages, see count_type */
> >>>>> atomic_t nr_pages[NR_COUNT_TYPE];
> >>>>> diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
> >>>>> index 15d7e30bfc72..5a318399a2fa 100644
> >>>>> --- a/fs/f2fs/super.c
> >>>>> +++ b/fs/f2fs/super.c
> >>>>> @@ -1964,6 +1964,7 @@ int f2fs_quota_sync(struct super_block *sb, int
> >>>>> type)
> >>>>> int cnt;
> >>>>> int ret;
> >>>>>
> >>>>> + down_read(&sbi->quota_sem);
> >>>>> ret = dquot_writeback_dquots(sb, type);
> >>>>> if (ret)
> >>>>> goto out;
> >>>>> @@ -2001,6 +2002,7 @@ int f2fs_quota_sync(struct super_block *sb, int
> >>>>> type)
> >>>>> out:
> >>>>> if (ret)
> >>>>> set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR);
> >>>>> + up_read(&sbi->quota_sem);
> >>>>> return ret;
> >>>>> }
> >>>>>
> >>>>> @@ -2094,32 +2096,40 @@ static void
> >>>>> f2fs_truncate_quota_inode_pages(struct super_block *sb)
> >>>>>
> >>>>> static int f2fs_dquot_commit(struct dquot *dquot)
> >>>>> {
> >>>>> + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
> >>>>> int ret;
> >>>>>
> >>>>> + down_read(&sbi->quota_sem);
> >>>>> ret = dquot_commit(dquot);
> >>>>> if (ret < 0)
> >>>>> - set_sbi_flag(F2FS_SB(dquot->dq_sb),
> >>>>> SBI_QUOTA_NEED_REPAIR);
> >>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> >>>>> + up_read(&sbi->quota_sem);
> >>>>> return ret;
> >>>>> }
> >>>>>
> >>>>> static int f2fs_dquot_acquire(struct dquot *dquot)
> >>>>> {
> >>>>> + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
> >>>>> int ret;
> >>>>>
> >>>>> + down_read(&sbi->quota_sem);
> >>>>> ret = dquot_acquire(dquot);
> >>>>> if (ret < 0)
> >>>>> - set_sbi_flag(F2FS_SB(dquot->dq_sb),
> >>>>> SBI_QUOTA_NEED_REPAIR);
> >>>>> -
> >>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> >>>>> + up_read(&sbi->quota_sem);
> >>>>> return ret;
> >>>>> }
> >>>>>
> >>>>> static int f2fs_dquot_release(struct dquot *dquot)
> >>>>> {
> >>>>> + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
> >>>>> int ret;
> >>>>>
> >>>>> + down_read(&sbi->quota_sem);
> >>>>> ret = dquot_release(dquot);
> >>>>> if (ret < 0)
> >>>>> - set_sbi_flag(F2FS_SB(dquot->dq_sb),
> >>>>> SBI_QUOTA_NEED_REPAIR);
> >>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> >>>>> + up_read(&sbi->quota_sem);
> >>>>> return ret;
> >>>>> }
> >>>>>
> >>>>> @@ -2129,22 +2139,27 @@ static int f2fs_dquot_mark_dquot_dirty(struct
> >>>>> dquot *dquot)
> >>>>> struct f2fs_sb_info *sbi = F2FS_SB(sb);
> >>>>> int ret;
> >>>>>
> >>>>> + down_read(&sbi->quota_sem);
> >>>>> ret = dquot_mark_dquot_dirty(dquot);
> >>>>>
> >>>>> /* if we are using journalled quota */
> >>>>> if (is_journalled_quota(sbi))
> >>>>> set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
> >>>>>
> >>>>> + up_read(&sbi->quota_sem);
> >>>>> return ret;
> >>>>> }
> >>>>>
> >>>>> static int f2fs_dquot_commit_info(struct super_block *sb, int type)
> >>>>> {
> >>>>> + struct f2fs_sb_info *sbi = F2FS_SB(sb);
> >>>>> int ret;
> >>>>>
> >>>>> + down_read(&sbi->quota_sem);
> >>>>> ret = dquot_commit_info(sb, type);
> >>>>> if (ret < 0)
> >>>>> - set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR);
> >>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
> >>>>> + up_read(&sbi->quota_sem);
> >>>>> return ret;
> >>>>> }
> >>>>>
> >>>>> @@ -3253,6 +3268,7 @@ static int f2fs_fill_super(struct super_block
> >>>>> *sb, void *data, int silent)
> >>>>> }
> >>>>>
> >>>>> init_rwsem(&sbi->cp_rwsem);
> >>>>> + init_rwsem(&sbi->quota_sem);
> >>>>> init_waitqueue_head(&sbi->cp_wait);
> >>>>> init_sb_info(sbi);
> >>>>>
> >>>>>
> >>> .
> >>>
> > .
> >
