The branch main has been updated by mjg: URL: https://cgit.FreeBSD.org/src/commit/?id=a1a8f8ada13da1cce2cd3a4b4335a1243bd34f3e
commit a1a8f8ada13da1cce2cd3a4b4335a1243bd34f3e Author: Mateusz Guzik <[email protected]> AuthorDate: 2020-12-31 20:23:08 +0000 Commit: Mateusz Guzik <[email protected]> CommitDate: 2021-01-06 06:05:22 +0000 cache: deinline state handling The intent is to reduce branchfest when finishing the lookup. Tested by: pho --- sys/kern/vfs_cache.c | 157 ++++++++++++++++++++++++++------------------------- 1 file changed, 81 insertions(+), 76 deletions(-) diff --git a/sys/kern/vfs_cache.c b/sys/kern/vfs_cache.c index 7260896df452..9d54997c518c 100644 --- a/sys/kern/vfs_cache.c +++ b/sys/kern/vfs_cache.c @@ -3590,18 +3590,23 @@ struct nameidata_saved { struct cache_fpl { struct nameidata *ndp; struct componentname *cnp; - struct pwd *pwd; + struct pwd **pwd; struct vnode *dvp; struct vnode *tvp; seqc_t dvp_seqc; seqc_t tvp_seqc; struct nameidata_saved snd; + struct nameidata_saved snd_orig; int line; enum cache_fpl_status status:8; bool in_smr; bool fsearch; }; +static bool cache_fplookup_is_mp(struct cache_fpl *fpl); +static int cache_fplookup_cross_mount(struct cache_fpl *fpl); +static int cache_fplookup_partial_setup(struct cache_fpl *fpl); + static void cache_fpl_cleanup_cnp(struct componentname *cnp) { @@ -3676,9 +3681,24 @@ cache_fpl_restore_abort(struct cache_fpl *fpl, struct nameidata_saved *snd) MPASS(_fpl->in_smr == false); \ VFS_SMR_ASSERT_NOT_ENTERED(); \ }) +static void +cache_fpl_assert_status(struct cache_fpl *fpl) +{ + + switch (fpl->status) { + case CACHE_FPL_STATUS_UNSET: + __assert_unreachable(); + break; + case CACHE_FPL_STATUS_ABORTED: + case CACHE_FPL_STATUS_PARTIAL: + case CACHE_FPL_STATUS_HANDLED: + break; + } +} #else #define cache_fpl_smr_assert_entered(fpl) do { } while (0) #define cache_fpl_smr_assert_not_entered(fpl) do { } while (0) +#define cache_fpl_assert_status(fpl) do { } while (0) #endif #define cache_fpl_smr_enter_initial(fpl) ({ \ @@ -3702,6 +3722,23 @@ cache_fpl_restore_abort(struct cache_fpl *fpl, struct nameidata_saved *snd) }) static int +cache_fpl_aborted_early_impl(struct cache_fpl *fpl, int line) +{ + + if (fpl->status != CACHE_FPL_STATUS_UNSET) { + KASSERT(fpl->status == CACHE_FPL_STATUS_PARTIAL, + ("%s: converting to abort from %d at %d, set at %d\n", + __func__, fpl->status, line, fpl->line)); + } + cache_fpl_smr_assert_not_entered(fpl); + fpl->status = CACHE_FPL_STATUS_ABORTED; + fpl->line = line; + return (CACHE_FPL_FAILED); +} + +#define cache_fpl_aborted_early(x) cache_fpl_aborted_early_impl((x), __LINE__) + +static int __noinline cache_fpl_aborted_impl(struct cache_fpl *fpl, int line) { @@ -3712,12 +3749,15 @@ cache_fpl_aborted_impl(struct cache_fpl *fpl, int line) } fpl->status = CACHE_FPL_STATUS_ABORTED; fpl->line = line; + if (fpl->in_smr) + cache_fpl_smr_exit(fpl); + cache_fpl_restore_abort(fpl, &fpl->snd_orig); return (CACHE_FPL_FAILED); } #define cache_fpl_aborted(x) cache_fpl_aborted_impl((x), __LINE__) -static int +static int __noinline cache_fpl_partial_impl(struct cache_fpl *fpl, int line) { @@ -3727,7 +3767,7 @@ cache_fpl_partial_impl(struct cache_fpl *fpl, int line) cache_fpl_smr_assert_entered(fpl); fpl->status = CACHE_FPL_STATUS_PARTIAL; fpl->line = line; - return (CACHE_FPL_FAILED); + return (cache_fplookup_partial_setup(fpl)); } #define cache_fpl_partial(x) cache_fpl_partial_impl((x), __LINE__) @@ -3766,9 +3806,6 @@ cache_fpl_terminated(struct cache_fpl *fpl) _Static_assert((CACHE_FPL_SUPPORTED_CN_FLAGS & CACHE_FPL_INTERNAL_CN_FLAGS) == 0, "supported and internal flags overlap"); -static bool cache_fplookup_is_mp(struct cache_fpl *fpl); -static int cache_fplookup_cross_mount(struct cache_fpl *fpl); - static bool cache_fpl_islastcn(struct nameidata *ndp) { @@ -3798,29 +3835,29 @@ cache_can_fplookup(struct cache_fpl *fpl) td = cnp->cn_thread; if (!cache_fast_lookup) { - cache_fpl_aborted(fpl); + cache_fpl_aborted_early(fpl); return (false); } #ifdef MAC if (mac_vnode_check_lookup_enabled()) { - cache_fpl_aborted(fpl); + cache_fpl_aborted_early(fpl); return (false); } #endif if ((cnp->cn_flags & ~CACHE_FPL_SUPPORTED_CN_FLAGS) != 0) { - cache_fpl_aborted(fpl); + cache_fpl_aborted_early(fpl); return (false); } if (IN_CAPABILITY_MODE(td)) { - cache_fpl_aborted(fpl); + cache_fpl_aborted_early(fpl); return (false); } if (AUDITING_TD(td)) { - cache_fpl_aborted(fpl); + cache_fpl_aborted_early(fpl); return (false); } if (ndp->ni_startdir != NULL) { - cache_fpl_aborted(fpl); + cache_fpl_aborted_early(fpl); return (false); } return (true); @@ -3836,7 +3873,6 @@ cache_fplookup_dirfd(struct cache_fpl *fpl, struct vnode **vpp) ndp = fpl->ndp; error = fgetvp_lookup_smr(ndp->ni_dirfd, ndp, vpp, &fsearch); if (__predict_false(error != 0)) { - cache_fpl_smr_exit(fpl); return (cache_fpl_aborted(fpl)); } fpl->fsearch = fsearch; @@ -3882,12 +3918,11 @@ cache_fplookup_partial_setup(struct cache_fpl *fpl) ndp = fpl->ndp; cnp = fpl->cnp; - pwd = fpl->pwd; + pwd = *(fpl->pwd); dvp = fpl->dvp; dvp_seqc = fpl->dvp_seqc; if (!pwd_hold_smr(pwd)) { - cache_fpl_smr_exit(fpl); return (cache_fpl_aborted(fpl)); } @@ -4981,7 +5016,6 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl) struct mount *mp; int error; - error = CACHE_FPL_FAILED; ndp = fpl->ndp; cnp = fpl->cnp; @@ -4990,20 +5024,18 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl) fpl->dvp = dvp; fpl->dvp_seqc = vn_seqc_read_any(fpl->dvp); if (seqc_in_modify(fpl->dvp_seqc)) { - cache_fpl_aborted(fpl); - goto out; + return (cache_fpl_aborted(fpl)); } mp = atomic_load_ptr(&dvp->v_mount); if (__predict_false(mp == NULL || !cache_fplookup_mp_supported(mp))) { - cache_fpl_aborted(fpl); - goto out; + return (cache_fpl_aborted(fpl)); } VNPASS(cache_fplookup_vnode_supported(fpl->dvp), fpl->dvp); error = cache_fplookup_preparse(fpl); if (__predict_false(cache_fpl_terminated(fpl))) { - goto out; + return (error); } for (;;) { @@ -5043,39 +5075,8 @@ cache_fplookup_impl(struct vnode *dvp, struct cache_fpl *fpl) cache_fplookup_parse_advance(fpl); cache_fpl_checkpoint(fpl, &fpl->snd); } -out: - switch (fpl->status) { - case CACHE_FPL_STATUS_UNSET: - __assert_unreachable(); - break; - case CACHE_FPL_STATUS_PARTIAL: - cache_fpl_smr_assert_entered(fpl); - return (cache_fplookup_partial_setup(fpl)); - case CACHE_FPL_STATUS_ABORTED: - if (fpl->in_smr) - cache_fpl_smr_exit(fpl); - return (CACHE_FPL_FAILED); - case CACHE_FPL_STATUS_HANDLED: - MPASS(error != CACHE_FPL_FAILED); - cache_fpl_smr_assert_not_entered(fpl); - /* - * A common error is ENOENT. - */ - if (error != 0) { - ndp->ni_dvp = NULL; - ndp->ni_vp = NULL; - cache_fpl_cleanup_cnp(cnp); - return (error); - } - ndp->ni_dvp = fpl->dvp; - ndp->ni_vp = fpl->tvp; - if (cnp->cn_flags & SAVENAME) - cnp->cn_flags |= HASBUF; - else - cache_fpl_cleanup_cnp(cnp); - return (error); - } - __assert_unreachable(); + + return (error); } /* @@ -5162,10 +5163,10 @@ cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status, struct pwd *pwd; struct vnode *dvp; struct componentname *cnp; - struct nameidata_saved orig; int error; fpl.status = CACHE_FPL_STATUS_UNSET; + fpl.in_smr = false; fpl.ndp = ndp; fpl.cnp = cnp = &ndp->ni_cnd; @@ -5185,12 +5186,13 @@ cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status, return (EOPNOTSUPP); } - cache_fpl_checkpoint(&fpl, &orig); + cache_fpl_checkpoint(&fpl, &fpl.snd_orig); cache_fpl_smr_enter_initial(&fpl); fpl.fsearch = false; + fpl.pwd = pwdp; pwd = pwd_get_smr(); - fpl.pwd = pwd; + *(fpl.pwd) = pwd; ndp->ni_rootdir = pwd->pwd_rdir; ndp->ni_topdir = pwd->pwd_jdir; @@ -5209,31 +5211,34 @@ cache_fplookup(struct nameidata *ndp, enum cache_fpl_status *status, } SDT_PROBE4(vfs, namei, lookup, entry, dvp, cnp->cn_pnbuf, cnp->cn_flags, true); - error = cache_fplookup_impl(dvp, &fpl); out: cache_fpl_smr_assert_not_entered(&fpl); - SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status); - + cache_fpl_assert_status(&fpl); *status = fpl.status; - switch (fpl.status) { - case CACHE_FPL_STATUS_UNSET: - __assert_unreachable(); - break; - case CACHE_FPL_STATUS_HANDLED: - if (error != 0) - MPASS(ndp->ni_vp == NULL); - SDT_PROBE3(vfs, namei, lookup, return, error, ndp->ni_vp, true); - break; - case CACHE_FPL_STATUS_PARTIAL: - *pwdp = fpl.pwd; + if (SDT_PROBES_ENABLED()) { + SDT_PROBE3(vfs, fplookup, lookup, done, ndp, fpl.line, fpl.status); + if (fpl.status == CACHE_FPL_STATUS_HANDLED) + SDT_PROBE3(vfs, namei, lookup, return, error, ndp->ni_vp, true); + } + + if (__predict_true(fpl.status == CACHE_FPL_STATUS_HANDLED)) { + MPASS(error != CACHE_FPL_FAILED); /* - * Status restored by cache_fplookup_partial_setup. + * A common error is ENOENT. */ - break; - case CACHE_FPL_STATUS_ABORTED: - cache_fpl_restore_abort(&fpl, &orig); - break; + if (error != 0) { + ndp->ni_dvp = NULL; + ndp->ni_vp = NULL; + cache_fpl_cleanup_cnp(cnp); + return (error); + } + ndp->ni_dvp = fpl.dvp; + ndp->ni_vp = fpl.tvp; + if (cnp->cn_flags & SAVENAME) + cnp->cn_flags |= HASBUF; + else + cache_fpl_cleanup_cnp(cnp); } return (error); } _______________________________________________ [email protected] mailing list https://lists.freebsd.org/mailman/listinfo/dev-commits-src-main To unsubscribe, send any mail to "[email protected]"
