Add AVX512 Ipv6 optimized profile for vlan/IPv6/UDP and vlan/IPv6/TCP. MFEX autovalidaton test-case already has the IPv6 support for validating against the scalar mfex.
Signed-off-by: Kumar Amber <kumar.am...@intel.com> Signed-off-by: Harry van Haaren <harry.van.haa...@intel.com> Co-authored-by: Harry van Haaren <harry.van.haa...@intel.com> --- v2: - Fix check-patch sign-offs --- NEWS | 2 + lib/dpif-netdev-extract-avx512.c | 94 +++++++++++++++++++++++++++++++ lib/dpif-netdev-private-extract.c | 23 ++++++++ lib/dpif-netdev-private-extract.h | 6 ++ 4 files changed, 125 insertions(+) diff --git a/NEWS b/NEWS index f18e2c572..959df3add 100644 --- a/NEWS +++ b/NEWS @@ -11,6 +11,8 @@ Post-v2.16.0 - Userspace datapath: * Add AVX512 optimized profiles to miniflow extract for IPv6/UDP and IPv6/TCP. + * Add AVX512 optimized profiles to miniflow extract for VLAN/IPv6/UDP + and VLAN/IPv6/TCP. v2.16.0 - 16 Aug 2021 diff --git a/lib/dpif-netdev-extract-avx512.c b/lib/dpif-netdev-extract-avx512.c index 3384a8dba..11bca0144 100644 --- a/lib/dpif-netdev-extract-avx512.c +++ b/lib/dpif-netdev-extract-avx512.c @@ -214,6 +214,21 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a) 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, /* IPv6 */ \ NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, NU, /* Unused */ +/* VLAN (Dot1Q) patterns and masks. */ +#define PATTERN_DT1Q_MASK \ + 0x00, 0x00, 0xFF, 0xFF, +#define PATTERN_DT1Q_IPV6 \ + 0x00, 0x00, 0x86, 0xDD, + +#define PATTERN_DT1Q_IPV6_SHUFFLE \ + /* Ether (2 blocks): Note that *VLAN* type is written here. */ \ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 0, 0, \ + /* VLAN (1 block): Note that the *EtherHdr->Type* is written here. */ \ + 12, 13, 14, 15, 0, 0, 0, 0, \ + 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, /* IPv6 */ \ + 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, /* IPv6 */ \ + NU, NU, NU, NU, NU, NU, NU, NU, /* Unused */ + /* Generation of K-mask bitmask values, to zero out data in result. Note that * these correspond 1:1 to the above "*_SHUFFLE" values, and bit used must be * set in this K-mask, and "NU" values must be zero in the k-mask. Each mask @@ -228,6 +243,8 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a) #define KMASK_TCP 0x0F00ULL #define KMASK_IPV6 0xFFFFULL #define KMASK_ETHER_IPV6 0x3FFFULL +#define KMASK_DT1Q_IPV6 0xFF0FULL +#define KMASK_IPV6_NOHDR 0x00FFULL #define PATTERN_IPV4_UDP_KMASK \ (KMASK_ETHER | (KMASK_IPV4 << 16) | (KMASK_UDP << 32)) @@ -244,6 +261,10 @@ _mm512_maskz_permutexvar_epi8_wrap(__mmask64 kmask, __m512i idx, __m512i a) #define PATTERN_IPV6_KMASK \ (KMASK_ETHER_IPV6 | (KMASK_IPV6 << 16) | (KMASK_IPV6 << 32)) +#define PATTERN_DT1Q_IPV6_KMASK \ + (KMASK_ETHER_IPV6 | (KMASK_DT1Q_IPV6 << 16) | (KMASK_IPV6 << 32) | \ + (KMASK_IPV6_NOHDR << 48)) + /* This union allows initializing static data as u8, but easily loading it * into AVX512 registers too. The union ensures proper alignment for the zmm. */ @@ -324,6 +345,8 @@ enum MFEX_PROFILES { PROFILE_ETH_VLAN_IPV4_TCP, PROFILE_ETH_IPV6_UDP, PROFILE_ETH_IPV6_TCP, + PROFILE_ETH_VLAN_IPV6_TCP, + PROFILE_ETH_VLAN_IPV6_UDP, PROFILE_COUNT, }; @@ -426,6 +449,37 @@ static const struct mfex_profile mfex_profiles[PROFILE_COUNT] = .dp_pkt_min_size = 54, }, + [PROFILE_ETH_VLAN_IPV6_TCP] = { + .probe_mask.u8_data = { + PATTERN_ETHERTYPE_MASK PATTERN_DT1Q_MASK PATTERN_IPV6_MASK }, + .probe_data.u8_data = { + PATTERN_ETHERTYPE_DT1Q PATTERN_DT1Q_IPV6 PATTERN_IPV6_TCP }, + + .store_shuf.u8_data = { PATTERN_DT1Q_IPV6_SHUFFLE }, + .store_kmsk = PATTERN_DT1Q_IPV6_KMASK, + + .mf_bits = { 0x38a0000000000000, 0x000000000004443c}, + .dp_pkt_offs = { + 14, UINT16_MAX, 18, 58, + }, + .dp_pkt_min_size = 66, + }, + + [PROFILE_ETH_VLAN_IPV6_UDP] = { + .probe_mask.u8_data = { + PATTERN_ETHERTYPE_MASK PATTERN_DT1Q_MASK PATTERN_IPV6_MASK }, + .probe_data.u8_data = { + PATTERN_ETHERTYPE_DT1Q PATTERN_DT1Q_IPV6 PATTERN_IPV6_UDP }, + + .store_shuf.u8_data = { PATTERN_DT1Q_IPV6_SHUFFLE }, + .store_kmsk = PATTERN_DT1Q_IPV6_KMASK, + + .mf_bits = { 0x38a0000000000000, 0x000000000004043c}, + .dp_pkt_offs = { + 14, UINT16_MAX, 18, 58, + }, + .dp_pkt_min_size = 66, + }, }; /* IPv6 header helper function to fix TC, flow label and next header. */ @@ -676,6 +730,44 @@ mfex_avx512_process(struct dp_packet_batch *packets, mfex_handle_tcp_flags(tcp, &blocks[9]); } break; + + case PROFILE_ETH_VLAN_IPV6_TCP: { + mfex_vlan_pcp(pkt[14], &keys[i].buf[4]); + + /* Handle dynamic l2_pad_size. */ + uint32_t payload_size_ipv6 = size - VLAN_ETH_HEADER_LEN; + struct ovs_16aligned_ip6_hdr *nh = (void *)&pkt + [VLAN_ETH_HEADER_LEN]; + mfex_ipv6_set_l2_pad_size(packet, nh, payload_size_ipv6); + + /* Process IPv6 header for TC, flow Label and next header. */ + mfex_handle_ipv6_hdr_block(&pkt[VLAN_ETH_HEADER_LEN], + &blocks[9]); + + /* Process TCP header. */ + mfex_handle_ipv6_l4((void *)&pkt[58], &blocks[11]); + const struct tcp_header *tcp = (void *)&pkt[58]; + mfex_handle_tcp_flags(tcp, &blocks[10]); + + } break; + + case PROFILE_ETH_VLAN_IPV6_UDP: { + mfex_vlan_pcp(pkt[14], &keys[i].buf[4]); + + /* Handle dynamic l2_pad_size. */ + uint32_t payload_size_ipv6 = size - VLAN_ETH_HEADER_LEN; + struct ovs_16aligned_ip6_hdr *nh = (void *)&pkt + [VLAN_ETH_HEADER_LEN]; + mfex_ipv6_set_l2_pad_size(packet, nh, payload_size_ipv6); + + /* Process IPv6 header for TC, flow Label and next header. */ + mfex_handle_ipv6_hdr_block(&pkt[VLAN_ETH_HEADER_LEN], + &blocks[9]); + + /* Process UDP header. */ + mfex_handle_ipv6_l4((void *)&pkt[58], &blocks[10]); + + } break; default: break; }; @@ -723,6 +815,8 @@ DECLARE_MFEX_FUNC(dot1q_ip_udp, PROFILE_ETH_VLAN_IPV4_UDP) DECLARE_MFEX_FUNC(dot1q_ip_tcp, PROFILE_ETH_VLAN_IPV4_TCP) DECLARE_MFEX_FUNC(ipv6_udp, PROFILE_ETH_IPV6_UDP) DECLARE_MFEX_FUNC(ipv6_tcp, PROFILE_ETH_IPV6_TCP) +DECLARE_MFEX_FUNC(dot1q_ipv6_tcp, PROFILE_ETH_VLAN_IPV6_TCP) +DECLARE_MFEX_FUNC(dot1q_ipv6_udp, PROFILE_ETH_VLAN_IPV6_UDP) static int32_t avx512_isa_probe(uint32_t needs_vbmi) diff --git a/lib/dpif-netdev-private-extract.c b/lib/dpif-netdev-private-extract.c index 0b665bced..b3d96075c 100644 --- a/lib/dpif-netdev-private-extract.c +++ b/lib/dpif-netdev-private-extract.c @@ -120,6 +120,29 @@ static struct dpif_miniflow_extract_impl mfex_impls[] = { .name = "avx512_ipv6_tcp", }, + [MFEX_IMPL_VMBI_DOT1Q_IPv6_TCP] = { + .probe = mfex_avx512_vbmi_probe, + .extract_func = mfex_avx512_vbmi_dot1q_ipv6_tcp, + .name = "avx512_vbmi_avx512_dot1q_ipv6_tcp", + }, + + [MFEX_IMPL_DOT1Q_IPv6_TCP] = { + .probe = mfex_avx512_probe, + .extract_func = mfex_avx512_dot1q_ipv6_tcp, + .name = "avx512_dot1q_ipv6_tcp", + }, + + [MFEX_IMPL_VMBI_DOT1Q_IPv6_UDP] = { + .probe = mfex_avx512_vbmi_probe, + .extract_func = mfex_avx512_vbmi_dot1q_ipv6_udp, + .name = "avx512_vbmi_avx512_dot1q_ipv6_udp", + }, + + [MFEX_IMPL_DOT1Q_IPv6_UDP] = { + .probe = mfex_avx512_probe, + .extract_func = mfex_avx512_dot1q_ipv6_udp, + .name = "avx512_dot1q_ipv6_udp", + }, #endif }; diff --git a/lib/dpif-netdev-private-extract.h b/lib/dpif-netdev-private-extract.h index 4bbe70f26..e10d840f3 100644 --- a/lib/dpif-netdev-private-extract.h +++ b/lib/dpif-netdev-private-extract.h @@ -94,6 +94,10 @@ enum dpif_miniflow_extract_impl_idx { MFEX_IMPL_IPv6_UDP, MFEX_IMPL_VMBI_IPv6_TCP, MFEX_IMPL_IPv6_TCP, + MFEX_IMPL_VMBI_DOT1Q_IPv6_TCP, + MFEX_IMPL_DOT1Q_IPv6_TCP, + MFEX_IMPL_VMBI_DOT1Q_IPv6_UDP, + MFEX_IMPL_DOT1Q_IPv6_UDP, #endif MFEX_IMPL_MAX }; @@ -203,6 +207,8 @@ DECLARE_AVX512_MFEX_PROTOTYPE(dot1q_ip_udp); DECLARE_AVX512_MFEX_PROTOTYPE(dot1q_ip_tcp); DECLARE_AVX512_MFEX_PROTOTYPE(ipv6_udp); DECLARE_AVX512_MFEX_PROTOTYPE(ipv6_tcp); +DECLARE_AVX512_MFEX_PROTOTYPE(dot1q_ipv6_tcp); +DECLARE_AVX512_MFEX_PROTOTYPE(dot1q_ipv6_udp); #endif /* __x86_64__ */ -- 2.25.1 _______________________________________________ dev mailing list d...@openvswitch.org https://mail.openvswitch.org/mailman/listinfo/ovs-dev