Update skb_splice_bits() (spd_fill_page(), sock_spd_release(), and skb_pipe_buf_ops) and skb_append_pagefrags() to acquire and release references via netmem_inc_ref(), netmem_dec_ref(), and skb_frag_ref() so page_pool RX fragments spliced into a pipe remain pinned in the pool while in the pipe and pair with skb_append_pagefrags() and skb_release_data() when spliced into a TX socket once netmem_inc_ref() and netmem_dec_ref() handle page_pool netmems.
In tls_push_data(), fall back to copying when MSG_SPLICE_PAGES encounters a page_pool page because kTLS device offload record->frags[] tracks pages via raw struct page references rather than pp_ref_count before handing them to tls_push_sg(). Caveats: - Because pipe buffers created by skb_splice_bits() hold pp_ref_count until released (including across tee()), user pipes holding unconsumed spliced data keep those page_pool pages in-flight and DMA-mapped in the pool (bounded by pipe capacity and RLIMIT_NOFILE rather than sk_rcvbuf) until the pipe buffers are consumed or closed, with page_pool_destroy() deferring pool release via its periodic workqueue and page_pool_scrub() unmapping DMA. Signed-off-by: Mina Almasry <[email protected]> --- net/core/skbuff.c | 31 +++++++++++++++++++++++++++---- net/tls/tls_device.c | 6 ++++++ 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 48f6764117243..19d4ce347c41b 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3114,9 +3114,32 @@ EXPORT_SYMBOL(skb_copy_bits); */ static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) { - put_page(spd->pages[i]); + netmem_dec_ref(page_to_netmem(spd->pages[i])); } +static void skb_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + netmem_dec_ref(page_to_netmem(buf->page)); +} + +static bool skb_pipe_buf_get(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct page *page = compound_head(buf->page); + + if (WARN_ON_ONCE(page_ref_count(page) <= 0)) + return false; + + netmem_inc_ref(page_to_netmem(page)); + return true; +} + +static const struct pipe_buf_operations skb_pipe_buf_ops = { + .release = skb_pipe_buf_release, + .get = skb_pipe_buf_get, +}; + static struct page *linear_to_page(struct page *page, unsigned int *len, unsigned int *offset, struct sock *sk) @@ -3165,7 +3188,7 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, spd->partial[spd->nr_pages - 1].len += *len; return false; } - get_page(page); + netmem_inc_ref(page_to_netmem(page)); spd->pages[spd->nr_pages] = page; spd->partial[spd->nr_pages].len = *len; spd->partial[spd->nr_pages].offset = offset; @@ -3281,7 +3304,7 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, .pages = pages, .partial = partial, .nr_pages_max = MAX_SKB_FRAGS, - .ops = &nosteal_pipe_buf_ops, + .ops = &skb_pipe_buf_ops, .spd_release = sock_spd_release, }; int ret = 0; @@ -4692,8 +4715,8 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page, skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); } else if (i < max_frags) { skb_zcopy_downgrade_managed(skb); - get_page(page); skb_fill_page_desc_noacc(skb, i, page, offset, size); + skb_frag_ref(skb, i); } else { return -EMSGSIZE; } diff --git a/net/tls/tls_device.c b/net/tls/tls_device.c index 4cdfbb57ecff0..05abf8e23f78a 100644 --- a/net/tls/tls_device.c +++ b/net/tls/tls_device.c @@ -513,10 +513,16 @@ static int tls_push_data(struct sock *sk, goto handle_error; } + if (page_pool_page_is_pp(compound_head(*pages))) { + iov_iter_revert(iter, copy); + goto copy_data; + } + zc_pfrag.offset = off; zc_pfrag.size = copy; tls_append_frag(record, &zc_pfrag, copy); } else if (copy) { +copy_data: copy = min_t(size_t, copy, pfrag->size - pfrag->offset); rc = tls_device_copy_data(page_address(pfrag->page) + -- 2.56.0.455.g6c5a503b38-goog
