The UDP GSO path currently allocates one packet ID for every input segment. This performs one atomic update per segment even though the whole batch is prepared by a single caller.
Reserve the complete consecutive range before dequeuing any input skb, then assign its IDs to segments in queue order. This reduces packet-ID allocation to one atomic operation per batch and leaves the input queue intact when no complete range remains, allowing the ordinary transmit path to retry with a newly installed key. If the requested range would cross the end of the packet-ID space, publish exhaustion immediately. Packet IDs abandoned after a later encryption failure remain unused, so an AEAD IV is never reused. Signed-off-by: Ralf Lici <[email protected]> --- No changes since v3 https://lore.kernel.org/openvpn-devel/b7eeace194390a817c6b1105a5f2e294329fc4b9.1794223133.git.r...@mandelbit.com/ Changes since v2 https://lore.kernel.org/openvpn-devel/[email protected]/ - Adapt to the synchronous UDP GSO path; no intended functional change. No changes since v1 https://lore.kernel.org/openvpn-devel/a7138d54319d501b6ba7f1d50dce65cb8d2c3e46.1790843153.git.r...@mandelbit.com/ drivers/net/ovpn/crypto_aead.c | 7 ++--- drivers/net/ovpn/crypto_aead.h | 2 +- drivers/net/ovpn/io.c | 17 +++++++++--- drivers/net/ovpn/pktid.h | 48 ++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 9 deletions(-) diff --git a/drivers/net/ovpn/crypto_aead.c b/drivers/net/ovpn/crypto_aead.c index 26305984197e..a75a0bf65c89 100644 --- a/drivers/net/ovpn/crypto_aead.c +++ b/drivers/net/ovpn/crypto_aead.c @@ -175,7 +175,8 @@ static void ovpn_aead_encrypt_header(struct ovpn_peer *peer, int ovpn_aead_encrypt_gso(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, - struct sk_buff *gso_skb, unsigned int offset) + struct sk_buff *gso_skb, unsigned int offset, + u32 pktid) { const unsigned int dst_nents = skb_shinfo(gso_skb)->nr_frags + 4; const unsigned int src_nents = skb_shinfo(skb)->nr_frags + 2; @@ -184,7 +185,6 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer, unsigned int payload_off; int dst_idx, mapped, ret; u8 *aad, *iv; - u32 pktid; req = ovpn_aead_request_alloc(ks->encrypt, skb, src_nents + dst_nents, OVPN_AAD_SIZE, &iv); @@ -194,9 +194,6 @@ int ovpn_aead_encrypt_gso(struct ovpn_peer *peer, dst = src + src_nents; aad = (u8 *)(dst + dst_nents); - ret = ovpn_pktid_xmit_next(&ks->pid_xmit, &pktid); - if (unlikely(ret < 0)) - return ret; ovpn_aead_encrypt_header(peer, ks, pktid, iv, aad); ret = skb_store_bits(gso_skb, offset, aad, OVPN_AAD_SIZE); diff --git a/drivers/net/ovpn/crypto_aead.h b/drivers/net/ovpn/crypto_aead.h index 8b444744944e..92e8ed76fba6 100644 --- a/drivers/net/ovpn/crypto_aead.h +++ b/drivers/net/ovpn/crypto_aead.h @@ -20,7 +20,7 @@ int ovpn_aead_encrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, int ovpn_aead_encrypt_gso(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, struct sk_buff *skb, struct sk_buff *gso_skb, - unsigned int offset); + unsigned int offset, u32 pktid); int ovpn_aead_decrypt(struct ovpn_peer *peer, struct ovpn_crypto_key_slot *ks, struct sk_buff *skb); diff --git a/drivers/net/ovpn/io.c b/drivers/net/ovpn/io.c index c9f926a9e55d..41ced8291a91 100644 --- a/drivers/net/ovpn/io.c +++ b/drivers/net/ovpn/io.c @@ -377,20 +377,31 @@ static bool ovpn_encrypt_gso_queue(struct sk_buff_head *skbs, { unsigned int offset = 0, i, len; struct sk_buff *skb; + u32 first_pktid; int ret; + /* reserve the complete range before dequeuing any input skb so that a + * failure leaves the ordinary transmit path available as a fallback + */ + if (unlikely(ovpn_pktid_xmit_range(&ks->pid_xmit, segments, + &first_pktid))) { + ovpn_encrypt_key_exhausted(peer, ks); + ovpn_crypto_key_slot_put(ks); + ovpn_peer_put(peer); + return false; + } + for (i = 0; i < segments; i++) { skb = __skb_dequeue(skbs); len = skb->len + OVPN_DATA_V2_OVERHEAD; memset(ovpn_skb_cb(skb), 0, sizeof(struct ovpn_cb)); - ret = ovpn_aead_encrypt_gso(peer, ks, skb, batch, offset); + ret = ovpn_aead_encrypt_gso(peer, ks, skb, batch, offset, + first_pktid + i); kfree(ovpn_skb_cb(skb)->crypto_tmp); if (unlikely(ret < 0)) { DEBUG_NET_WARN_ON_ONCE(ret == -EINPROGRESS || ret == -EBUSY); - if (ret == -ERANGE) - ovpn_encrypt_key_exhausted(peer, ks); ovpn_dev_dstats_tx_dropped(peer->ovpn->dev, i + 1); kfree_skb(skb); goto err; diff --git a/drivers/net/ovpn/pktid.h b/drivers/net/ovpn/pktid.h index 21845f353bc8..c922e11da6a0 100644 --- a/drivers/net/ovpn/pktid.h +++ b/drivers/net/ovpn/pktid.h @@ -68,6 +68,54 @@ static inline int ovpn_pktid_xmit_next(struct ovpn_pktid_xmit *pid, u32 *pktid) return 0; } +/** + * ovpn_pktid_xmit_range - reserve consecutive packet IDs + * @pid: transmit packet-ID state + * @count: non-zero number of IDs to reserve + * @pktid: first reserved packet ID + * + * The reservation is all-or-nothing. If the requested range would cross the + * end of the 32-bit packet-ID space, exhaust the state without returning the + * unused tail. Packet IDs can be skipped but never reused as an AEAD IV. + * + * Return: 0 on success or -ERANGE when no complete range remains + */ +static inline int ovpn_pktid_xmit_range(struct ovpn_pktid_xmit *pid, + unsigned int count, u32 *pktid) +{ + int old = atomic_read(&pid->seq_num); + u32 first, next; + bool fits; + + for (;;) { + first = (u32)old; + next = first + count; + + /* space is already exhausted */ + if (unlikely(!first)) + return -ERANGE; + + fits = count - 1 <= U32_MAX - first; + /* if it doesn't fit we still have to publish exhaustion (0) + * so concurrent allocators stop using this key + */ + if (unlikely(!fits)) + next = 0; + + /* if another CPU changed the state, we retry using the new + * seq_num value + */ + if (!atomic_try_cmpxchg(&pid->seq_num, &old, (int)next)) + continue; + + if (unlikely(!fits)) + return -ERANGE; + + *pktid = first; + return 0; + } +} + /* Write 12-byte AEAD IV to dest */ static inline void ovpn_pktid_aead_write(const u32 pktid, const u8 nt[], -- 2.56.0 _______________________________________________ Openvpn-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/openvpn-devel
