Add some helper functions to wrap the handling of the NETFS_RREQ_ALL_QUEUED flag and to insert the appropriate barriers.
Signed-off-by: David Howells <[email protected]> cc: Paulo Alcantara <[email protected]> cc: [email protected] cc: [email protected] --- fs/netfs/buffered_read.c | 9 +++------ fs/netfs/direct_read.c | 9 +++------ fs/netfs/internal.h | 19 +++++++++++++++++++ fs/netfs/misc.c | 2 +- fs/netfs/read_collect.c | 4 +--- fs/netfs/read_pgpriv2.c | 3 +-- fs/netfs/read_single.c | 9 +++------ fs/netfs/write_collect.c | 3 +-- fs/netfs/write_issue.c | 7 ++----- include/trace/events/netfs.h | 1 + 10 files changed, 35 insertions(+), 31 deletions(-) diff --git a/fs/netfs/buffered_read.c b/fs/netfs/buffered_read.c index 61cf82b4b60d..e7fa30376862 100644 --- a/fs/netfs/buffered_read.c +++ b/fs/netfs/buffered_read.c @@ -355,10 +355,8 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq) } start += slice; size -= slice; - if (size <= 0) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); - } + if (size <= 0) + netfs_all_subreqs_queued(rreq); if (fq) { /* See if the cache indicated this should be cached. */ @@ -378,8 +376,7 @@ static void netfs_read_to_pagecache(struct netfs_io_request *rreq) } while (size > 0); if (unlikely(size > 0)) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); netfs_wake_collector(rreq); } diff --git a/fs/netfs/direct_read.c b/fs/netfs/direct_read.c index aa10af5171a8..5405e108b7a3 100644 --- a/fs/netfs/direct_read.c +++ b/fs/netfs/direct_read.c @@ -84,10 +84,8 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) size -= slice; start += slice; rreq->submitted += slice; - if (size <= 0) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); - } + if (size <= 0) + netfs_all_subreqs_queued(rreq); rreq->netfs_ops->issue_read(subreq); @@ -99,8 +97,7 @@ static void netfs_dispatch_unbuffered_reads(struct netfs_io_request *rreq) } while (size > 0); if (unlikely(size > 0)) { - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); netfs_wake_collector(rreq); } } diff --git a/fs/netfs/internal.h b/fs/netfs/internal.h index 67f2406e00cd..1346e750075a 100644 --- a/fs/netfs/internal.h +++ b/fs/netfs/internal.h @@ -315,6 +315,25 @@ static inline bool netfs_check_subreq_in_progress(const struct netfs_io_subreque return test_bit_acquire(NETFS_SREQ_IN_PROGRESS, &subreq->flags); } +/* + * Indicate that we've generated and queued all the subrequests we're going to. + */ +static inline void netfs_all_subreqs_queued(struct netfs_io_request *rreq) +{ + smp_wmb(); /* Write lists before ALL_QUEUED. */ + set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + trace_netfs_rreq(rreq, netfs_rreq_trace_all_queued); +} + +/* + * Query if all subrequests are queued. + */ +static inline bool netfs_are_all_subreqs_queued(const struct netfs_io_request *rreq) +{ + /* Read lists after ALL_QUEUED. */ + return test_bit_acquire(NETFS_RREQ_ALL_QUEUED, &rreq->flags); +} + /* * fscache-cache.c */ diff --git a/fs/netfs/misc.c b/fs/netfs/misc.c index eafc4edae6a0..a3cd76d584b8 100644 --- a/fs/netfs/misc.c +++ b/fs/netfs/misc.c @@ -424,7 +424,7 @@ static int netfs_collect_in_app(struct netfs_io_request *rreq, need_collect = true; break; } - if (subreq || !test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) + if (subreq || !netfs_are_all_subreqs_queued(rreq)) done = false; } diff --git a/fs/netfs/read_collect.c b/fs/netfs/read_collect.c index 61f2664de6d3..7420b90233e6 100644 --- a/fs/netfs/read_collect.c +++ b/fs/netfs/read_collect.c @@ -462,10 +462,8 @@ bool netfs_read_collection(struct netfs_io_request *rreq) /* We're done when the app thread has finished posting subreqs and the * queue is empty. */ - if (!test_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags)) + if (!netfs_are_all_subreqs_queued(rreq)) return false; - smp_rmb(); /* Read ALL_QUEUED before subreq lists. */ - if (!list_empty(&stream->subrequests)) return false; diff --git a/fs/netfs/read_pgpriv2.c b/fs/netfs/read_pgpriv2.c index bf4d9d6877d7..5280b606fda4 100644 --- a/fs/netfs/read_pgpriv2.c +++ b/fs/netfs/read_pgpriv2.c @@ -158,8 +158,7 @@ void netfs_pgpriv2_end_copy_to_cache(struct netfs_io_request *rreq) return; netfs_issue_write(creq, &creq->io_streams[1]); - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &creq->flags); + netfs_all_subreqs_queued(creq); trace_netfs_rreq(rreq, netfs_rreq_trace_end_copy_to_cache); if (list_empty_careful(&creq->io_streams[1].subrequests)) netfs_wake_collector(creq); diff --git a/fs/netfs/read_single.c b/fs/netfs/read_single.c index de67ac41548d..ccb5fc809d99 100644 --- a/fs/netfs/read_single.c +++ b/fs/netfs/read_single.c @@ -113,14 +113,12 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) goto cancel; } - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); rreq->netfs_ops->issue_read(subreq); rreq->submitted += subreq->len; break; case NETFS_READ_FROM_CACHE: - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); trace_netfs_sreq(subreq, netfs_sreq_trace_submit); netfs_single_read_cache(rreq, subreq); rreq->submitted += subreq->len; @@ -136,8 +134,7 @@ static int netfs_single_dispatch_read(struct netfs_io_request *rreq) return ret; cancel: netfs_cancel_read(subreq, ret); - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &rreq->flags); + netfs_all_subreqs_queued(rreq); netfs_wake_collector(rreq); return ret; } diff --git a/fs/netfs/write_collect.c b/fs/netfs/write_collect.c index e01233de41f5..b328463a73d9 100644 --- a/fs/netfs/write_collect.c +++ b/fs/netfs/write_collect.c @@ -422,9 +422,8 @@ bool netfs_write_collection(struct netfs_io_request *wreq) /* We're done when the app thread has finished posting subreqs and all * the queues in all the streams are empty. */ - if (!test_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags)) + if (!netfs_are_all_subreqs_queued(wreq)) return false; - smp_rmb(); /* Read ALL_QUEUED before lists. */ transferred = LONG_MAX; for (s = 0; s < NR_IO_STREAMS; s++) { diff --git a/fs/netfs/write_issue.c b/fs/netfs/write_issue.c index c3db732a20af..2783cf8a408b 100644 --- a/fs/netfs/write_issue.c +++ b/fs/netfs/write_issue.c @@ -520,8 +520,7 @@ static void netfs_end_issue_write(struct netfs_io_request *wreq) { bool needs_poke = true; - smp_wmb(); /* Write subreq lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); + netfs_all_subreqs_queued(wreq); for (int s = 0; s < NR_IO_STREAMS; s++) { struct netfs_io_stream *stream = &wreq->io_streams[s]; @@ -667,9 +666,7 @@ int netfs_writeback_single(struct address_space *mapping, netfs_issue_write(wreq, stream); } - smp_wmb(); /* Write lists before ALL_QUEUED. */ - set_bit(NETFS_RREQ_ALL_QUEUED, &wreq->flags); - + netfs_all_subreqs_queued(wreq); netfs_wake_collector(wreq); /* TODO: Might want to be async here if WB_SYNC_NONE, but then need to diff --git a/include/trace/events/netfs.h b/include/trace/events/netfs.h index b18952627f16..e94df94abf8c 100644 --- a/include/trace/events/netfs.h +++ b/include/trace/events/netfs.h @@ -47,6 +47,7 @@ E_(NETFS_PGPRIV2_COPY_TO_CACHE, "2C") #define netfs_rreq_traces \ + EM(netfs_rreq_trace_all_queued, "ALL-Q ") \ EM(netfs_rreq_trace_assess, "ASSESS ") \ EM(netfs_rreq_trace_collect, "COLLECT") \ EM(netfs_rreq_trace_complete, "COMPLET") \
