This commit adds the AVX512 implementation of the pop_vlan action. The implementation here is auto-validated by the miniflow extract autovalidator, hence its correctness can be easily tested and verified.
Signed-off-by: Emma Finn <[email protected]> --- lib/odp-execute-avx512.c | 76 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/lib/odp-execute-avx512.c b/lib/odp-execute-avx512.c index c46638e3f..9885c68e6 100644 --- a/lib/odp-execute-avx512.c +++ b/lib/odp-execute-avx512.c @@ -14,6 +14,11 @@ * limitations under the License. */ +#ifdef __x86_64__ +/* Sparse cannot handle the AVX512 instructions. */ +#if !defined(__CHECKER__) + + #include <config.h> #include <errno.h> @@ -21,9 +26,75 @@ #include "odp-netlink.h" #include "dp-packet.h" #include "openvswitch/vlog.h" +#include "dpdk.h" #include "immintrin.h" +VLOG_DEFINE_THIS_MODULE(odp_execute_avx512); +BUILD_ASSERT_DECL(offsetof(struct dp_packet, l2_5_ofs) + + MEMBER_SIZEOF(struct dp_packet, l2_5_ofs) == + offsetof(struct dp_packet, l3_ofs)); + +BUILD_ASSERT_DECL(offsetof(struct dp_packet, l3_ofs) + + MEMBER_SIZEOF(struct dp_packet, l3_ofs) == + offsetof(struct dp_packet, l4_ofs)); + +static inline void ALWAYS_INLINE +avx512_dp_packet_resize_l2(struct dp_packet *b, int increment) +{ + /* update packet size/data pointers */ + dp_packet_set_data(b, (char *) dp_packet_data(b) - increment); + dp_packet_set_size(b, dp_packet_size(b) + increment); + + /* Increment u16 packet offset values */ + const __m128i v_zeros = _mm_setzero_si128(); + const __m128i v_u16_max = _mm_cmpeq_epi16(v_zeros, v_zeros); + + /* Only these lanes can be incremented for push-VLAN action. */ + const uint8_t k_lanes = 0b1110; + __m128i v_offset = _mm_set1_epi16(VLAN_HEADER_LEN); + + /* Load packet and compare with UINT16_MAX */ + void *adjust_ptr = &b->l2_pad_size; + __m128i v_adjust_src = _mm_loadu_si128(adjust_ptr); + __mmask8 k_cmp = _mm_mask_cmpneq_epu16_mask(k_lanes, v_adjust_src, + v_u16_max); + + /* Add VLAN_HEADER_LEN using compare mask, store results. */ + __m128i v_adjust_wip = _mm_mask_sub_epi16(v_adjust_src, k_cmp, + v_adjust_src, v_offset); + _mm_storeu_si128(adjust_ptr, v_adjust_wip); + +} + +static inline void ALWAYS_INLINE +avx512_eth_pop_vlan(struct dp_packet *packet) +{ + struct vlan_eth_header *veh = dp_packet_eth(packet); + + if (veh && dp_packet_size(packet) >= sizeof *veh && + eth_type_vlan(veh->veth_type)) { + + __m128i v_ether = _mm_loadu_si128((void *) veh); + __m128i v_realign = _mm_alignr_epi8(v_ether, _mm_setzero_si128(), + 16 - VLAN_HEADER_LEN); + _mm_storeu_si128((void *) veh, v_realign); + avx512_dp_packet_resize_l2(packet, -VLAN_HEADER_LEN); + + } +} + +static void +action_avx512_pop_vlan(void *dp OVS_UNUSED, struct dp_packet_batch *batch, + const struct nlattr *a OVS_UNUSED, + bool should_steal OVS_UNUSED) +{ + struct dp_packet *packet; + + DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { + avx512_eth_pop_vlan(packet); + } +} /* Probe functions to check ISA requirements. */ static int32_t @@ -64,5 +135,10 @@ int32_t action_avx512_init(struct odp_execute_action_impl *self) { avx512_isa_probe(0); + self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_avx512_pop_vlan; + return 0; } + +#endif +#endif -- 2.25.1 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
