On 2019/6/20 1:26, Jaegeuk Kim wrote:
> On 06/18, Chao Yu wrote:
>> On 2019/6/14 10:46, Jaegeuk Kim wrote:
>>> On 06/11, Chao Yu wrote:
>>>> On 2019/6/5 2:36, Jaegeuk Kim wrote:
>>>>> Two paths to update quota and f2fs_lock_op:
>>>>>
>>>>> 1.
>>>>> - lock_op
>>>>> | - quota_update
>>>>> `- unlock_op
>>>>>
>>>>> 2.
>>>>> - quota_update
>>>>> - lock_op
>>>>> `- unlock_op
>>>>>
>>>>> But, we need to make a transaction on quota_update + lock_op in #2 case.
>>>>> So, this patch introduces:
>>>>> 1. lock_op
>>>>> 2. down_write
>>>>> 3. check __need_flush
>>>>> 4. up_write
>>>>> 5. if there is dirty quota entries, flush them
>>>>> 6. otherwise, good to go
>>>>>
>>>>> Signed-off-by: Jaegeuk Kim <[email protected]>
>>>>> ---
>>>>>
>>>>> v3 from v2:
>>>>> - refactor to fix quota corruption issue
>>>>> : it seems that the previous scenario is not real and no deadlock case
>>>>> was
>>>>> encountered.
>>>>
>>>> - f2fs_dquot_commit
>>>> - down_read(&sbi->quota_sem)
>>>> - block_operation
>>>> - f2fs_lock_all
>>>> - need_flush_quota
>>>> - down_write(&sbi->quota_sem)
>>>> - f2fs_quota_write
>>>> - f2fs_lock_op
>>>>
>>>> Why can't this happen?
>>>>
>>>> Once more question, should we hold quota_sem during checkpoint to avoid
>>>> further
>>>> quota update? f2fs_lock_op can do this job as well?
>>>
>>> I couldn't find write_dquot() call to make this happen, and f2fs_lock_op
>>> was not
>>
>> - f2fs_dquot_commit
>> - dquot_commit
>> ->commit_dqblk (v2_write_dquot)
>> - qtree_write_dquot
>> ->quota_write (f2fs_quota_write)
>> - f2fs_lock_op
>>
>> Do you mean there is no such way that calling f2fs_lock_op() from
>> f2fs_quota_write()? So that deadlock condition is not existing?
>
> I mean write_dquot->f2fs_dquot_commit and block_operation seems not racing
> together.
quota ioctl has the path calling write_dquot->f2fs_dquot_commit as below, which
can race with checkpoint().
- do_quotactl
- sb->s_qcop->quota_sync (f2fs_quota_sync)
- down_read(&sbi->quota_sem); ---- First
- dquot_writeback_dquots
- sb->dq_op->write_dquot (f2fs_dquot_commit)
- block_operation can
race here
- down_read(&sbi->quota_sem); ---- Second
Thanks,
>
>>
>> Thanks,
>>
>>> enough to cover quota updates. Current stress & power-cut tests are running
>>> for
>>> several days without problem with this patch.
>>>
>>>>
>>>> Thanks,
>>>>
>>>>>
>>>>> fs/f2fs/checkpoint.c | 41 +++++++++++++++++++----------------------
>>>>> fs/f2fs/f2fs.h | 1 +
>>>>> fs/f2fs/super.c | 26 +++++++++++++++++++++-----
>>>>> 3 files changed, 41 insertions(+), 27 deletions(-)
>>>>>
>>>>> diff --git a/fs/f2fs/checkpoint.c b/fs/f2fs/checkpoint.c
>>>>> index 89825261d474..43f65f0962e5 100644
>>>>> --- a/fs/f2fs/checkpoint.c
>>>>> +++ b/fs/f2fs/checkpoint.c
>>>>> @@ -1131,17 +1131,24 @@ static void __prepare_cp_block(struct
>>>>> f2fs_sb_info *sbi)
>>>>>
>>>>> static bool __need_flush_quota(struct f2fs_sb_info *sbi)
>>>>> {
>>>>> + bool ret = false;
>>>>> +
>>>>> if (!is_journalled_quota(sbi))
>>>>> return false;
>>>>> - if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH))
>>>>> - return false;
>>>>> - if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR))
>>>>> - return false;
>>>>> - if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH))
>>>>> - return true;
>>>>> - if (get_pages(sbi, F2FS_DIRTY_QDATA))
>>>>> - return true;
>>>>> - return false;
>>>>> +
>>>>> + down_write(&sbi->quota_sem);
>>>>> + if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
>>>>> + ret = false;
>>>>> + } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
>>>>> + ret = false;
>>>>> + } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
>>>>> + clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
>>>>> + ret = true;
>>>>> + } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
>>>>> + ret = true;
>>>>> + }
>>>>> + up_write(&sbi->quota_sem);
>>>>> + return ret;
>>>>> }
>>>>>
>>>>> /*
>>>>> @@ -1160,26 +1167,22 @@ static int block_operations(struct f2fs_sb_info
>>>>> *sbi)
>>>>> blk_start_plug(&plug);
>>>>>
>>>>> retry_flush_quotas:
>>>>> + f2fs_lock_all(sbi);
>>>>> if (__need_flush_quota(sbi)) {
>>>>> int locked;
>>>>>
>>>>> if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) {
>>>>> set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
>>>>> - f2fs_lock_all(sbi);
>>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
>>>>> goto retry_flush_dents;
>>>>> }
>>>>> - clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
>>>>> + f2fs_unlock_all(sbi);
>>>>>
>>>>> /* only failed during mount/umount/freeze/quotactl */
>>>>> locked = down_read_trylock(&sbi->sb->s_umount);
>>>>> f2fs_quota_sync(sbi->sb, -1);
>>>>> if (locked)
>>>>> up_read(&sbi->sb->s_umount);
>>>>> - }
>>>>> -
>>>>> - f2fs_lock_all(sbi);
>>>>> - if (__need_flush_quota(sbi)) {
>>>>> - f2fs_unlock_all(sbi);
>>>>> cond_resched();
>>>>> goto retry_flush_quotas;
>>>>> }
>>>>> @@ -1201,12 +1204,6 @@ static int block_operations(struct f2fs_sb_info
>>>>> *sbi)
>>>>> */
>>>>> down_write(&sbi->node_change);
>>>>>
>>>>> - if (__need_flush_quota(sbi)) {
>>>>> - up_write(&sbi->node_change);
>>>>> - f2fs_unlock_all(sbi);
>>>>> - goto retry_flush_quotas;
>>>>> - }
>>>>> -
>>>>> if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
>>>>> up_write(&sbi->node_change);
>>>>> f2fs_unlock_all(sbi);
>>>>> diff --git a/fs/f2fs/f2fs.h b/fs/f2fs/f2fs.h
>>>>> index 9674a85154b2..9bd2bf0f559b 100644
>>>>> --- a/fs/f2fs/f2fs.h
>>>>> +++ b/fs/f2fs/f2fs.h
>>>>> @@ -1253,6 +1253,7 @@ struct f2fs_sb_info {
>>>>> block_t unusable_block_count; /* # of blocks saved by last cp
>>>>> */
>>>>>
>>>>> unsigned int nquota_files; /* # of quota sysfile */
>>>>> + struct rw_semaphore quota_sem; /* blocking cp for flags */
>>>>>
>>>>> /* # of pages, see count_type */
>>>>> atomic_t nr_pages[NR_COUNT_TYPE];
>>>>> diff --git a/fs/f2fs/super.c b/fs/f2fs/super.c
>>>>> index 15d7e30bfc72..5a318399a2fa 100644
>>>>> --- a/fs/f2fs/super.c
>>>>> +++ b/fs/f2fs/super.c
>>>>> @@ -1964,6 +1964,7 @@ int f2fs_quota_sync(struct super_block *sb, int
>>>>> type)
>>>>> int cnt;
>>>>> int ret;
>>>>>
>>>>> + down_read(&sbi->quota_sem);
>>>>> ret = dquot_writeback_dquots(sb, type);
>>>>> if (ret)
>>>>> goto out;
>>>>> @@ -2001,6 +2002,7 @@ int f2fs_quota_sync(struct super_block *sb, int
>>>>> type)
>>>>> out:
>>>>> if (ret)
>>>>> set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR);
>>>>> + up_read(&sbi->quota_sem);
>>>>> return ret;
>>>>> }
>>>>>
>>>>> @@ -2094,32 +2096,40 @@ static void
>>>>> f2fs_truncate_quota_inode_pages(struct super_block *sb)
>>>>>
>>>>> static int f2fs_dquot_commit(struct dquot *dquot)
>>>>> {
>>>>> + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
>>>>> int ret;
>>>>>
>>>>> + down_read(&sbi->quota_sem);
>>>>> ret = dquot_commit(dquot);
>>>>> if (ret < 0)
>>>>> - set_sbi_flag(F2FS_SB(dquot->dq_sb), SBI_QUOTA_NEED_REPAIR);
>>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
>>>>> + up_read(&sbi->quota_sem);
>>>>> return ret;
>>>>> }
>>>>>
>>>>> static int f2fs_dquot_acquire(struct dquot *dquot)
>>>>> {
>>>>> + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
>>>>> int ret;
>>>>>
>>>>> + down_read(&sbi->quota_sem);
>>>>> ret = dquot_acquire(dquot);
>>>>> if (ret < 0)
>>>>> - set_sbi_flag(F2FS_SB(dquot->dq_sb), SBI_QUOTA_NEED_REPAIR);
>>>>> -
>>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
>>>>> + up_read(&sbi->quota_sem);
>>>>> return ret;
>>>>> }
>>>>>
>>>>> static int f2fs_dquot_release(struct dquot *dquot)
>>>>> {
>>>>> + struct f2fs_sb_info *sbi = F2FS_SB(dquot->dq_sb);
>>>>> int ret;
>>>>>
>>>>> + down_read(&sbi->quota_sem);
>>>>> ret = dquot_release(dquot);
>>>>> if (ret < 0)
>>>>> - set_sbi_flag(F2FS_SB(dquot->dq_sb), SBI_QUOTA_NEED_REPAIR);
>>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
>>>>> + up_read(&sbi->quota_sem);
>>>>> return ret;
>>>>> }
>>>>>
>>>>> @@ -2129,22 +2139,27 @@ static int f2fs_dquot_mark_dquot_dirty(struct
>>>>> dquot *dquot)
>>>>> struct f2fs_sb_info *sbi = F2FS_SB(sb);
>>>>> int ret;
>>>>>
>>>>> + down_read(&sbi->quota_sem);
>>>>> ret = dquot_mark_dquot_dirty(dquot);
>>>>>
>>>>> /* if we are using journalled quota */
>>>>> if (is_journalled_quota(sbi))
>>>>> set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
>>>>>
>>>>> + up_read(&sbi->quota_sem);
>>>>> return ret;
>>>>> }
>>>>>
>>>>> static int f2fs_dquot_commit_info(struct super_block *sb, int type)
>>>>> {
>>>>> + struct f2fs_sb_info *sbi = F2FS_SB(sb);
>>>>> int ret;
>>>>>
>>>>> + down_read(&sbi->quota_sem);
>>>>> ret = dquot_commit_info(sb, type);
>>>>> if (ret < 0)
>>>>> - set_sbi_flag(F2FS_SB(sb), SBI_QUOTA_NEED_REPAIR);
>>>>> + set_sbi_flag(sbi, SBI_QUOTA_NEED_REPAIR);
>>>>> + up_read(&sbi->quota_sem);
>>>>> return ret;
>>>>> }
>>>>>
>>>>> @@ -3253,6 +3268,7 @@ static int f2fs_fill_super(struct super_block *sb,
>>>>> void *data, int silent)
>>>>> }
>>>>>
>>>>> init_rwsem(&sbi->cp_rwsem);
>>>>> + init_rwsem(&sbi->quota_sem);
>>>>> init_waitqueue_head(&sbi->cp_wait);
>>>>> init_sb_info(sbi);
>>>>>
>>>>>
>>> .
>>>
> .
>
