Update skb_splice_bits() (spd_fill_page(), sock_spd_release(), and skb_pipe_buf_ops) and skb_append_pagefrags() to acquire and release references via netmem_inc_ref(), netmem_dec_ref(), and skb_frag_ref() so page_pool RX fragments spliced into a pipe remain pinned in the pool while in the pipe and pair with skb_append_pagefrags() and skb_release_data() when spliced into a TX socket once netmem_inc_ref() and netmem_dec_ref() handle page_pool netmems.
Caveats: - Because pipe buffers created by skb_splice_bits() hold pp_ref_count until released (including across tee()), user pipes holding unconsumed spliced data keep those page_pool pages in-flight and DMA-mapped in the pool (bounded by pipe capacity and RLIMIT_NOFILE rather than sk_rcvbuf) until the pipe buffers are consumed or closed, with page_pool_destroy() deferring pool release via its periodic workqueue and page_pool_scrub() unmapping DMA. Signed-off-by: Mina Almasry <[email protected]> --- net/core/skbuff.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/net/core/skbuff.c b/net/core/skbuff.c index 48f6764117243..32a9dcdf2cb22 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -3114,9 +3114,27 @@ EXPORT_SYMBOL(skb_copy_bits); */ static void sock_spd_release(struct splice_pipe_desc *spd, unsigned int i) { - put_page(spd->pages[i]); + netmem_dec_ref(page_to_netmem(spd->pages[i])); } +static void skb_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + netmem_dec_ref(page_to_netmem(buf->page)); +} + +static bool skb_pipe_buf_get(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + netmem_inc_ref(page_to_netmem(buf->page)); + return true; +} + +static const struct pipe_buf_operations skb_pipe_buf_ops = { + .release = skb_pipe_buf_release, + .get = skb_pipe_buf_get, +}; + static struct page *linear_to_page(struct page *page, unsigned int *len, unsigned int *offset, struct sock *sk) @@ -3165,7 +3183,7 @@ static bool spd_fill_page(struct splice_pipe_desc *spd, struct page *page, spd->partial[spd->nr_pages - 1].len += *len; return false; } - get_page(page); + netmem_inc_ref(page_to_netmem(page)); spd->pages[spd->nr_pages] = page; spd->partial[spd->nr_pages].len = *len; spd->partial[spd->nr_pages].offset = offset; @@ -3281,7 +3299,7 @@ int skb_splice_bits(struct sk_buff *skb, struct sock *sk, unsigned int offset, .pages = pages, .partial = partial, .nr_pages_max = MAX_SKB_FRAGS, - .ops = &nosteal_pipe_buf_ops, + .ops = &skb_pipe_buf_ops, .spd_release = sock_spd_release, }; int ret = 0; @@ -4692,8 +4710,8 @@ int skb_append_pagefrags(struct sk_buff *skb, struct page *page, skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], size); } else if (i < max_frags) { skb_zcopy_downgrade_managed(skb); - get_page(page); skb_fill_page_desc_noacc(skb, i, page, offset, size); + skb_frag_ref(skb, i); } else { return -EMSGSIZE; } -- 2.56.0.385.gac671c819a-goog
