This commit adds the AVX512 implementation of the
push_vlan action.
Signed-off-by: Emma Finn <[email protected]>
---
lib/odp-execute-avx512.c | 56 ++++++++++++++++++++++++++++++++++++---
lib/odp-execute-private.c | 1 +
lib/odp-execute.c | 24 ++++++++++-------
3 files changed, 69 insertions(+), 12 deletions(-)
diff --git a/lib/odp-execute-avx512.c b/lib/odp-execute-avx512.c
index 637956236..5d095f867 100644
--- a/lib/odp-execute-avx512.c
+++ b/lib/odp-execute-avx512.c
@@ -42,6 +42,13 @@ static inline void ALWAYS_INLINE
avx512_dp_packet_resize_l2(struct dp_packet *b, int resize_by_bytes)
{
/* Update packet size/data pointers */
+ if (resize_by_bytes >= 0) {
+ dp_packet_prealloc_headroom(b, resize_by_bytes);
+ } else {
+ ovs_assert(dp_packet_size(b) - dp_packet_l2_pad_size(b) >=
+ -resize_by_bytes);
+ }
+
dp_packet_set_data(b, (char *) dp_packet_data(b) - resize_by_bytes);
dp_packet_set_size(b, dp_packet_size(b) + resize_by_bytes);
@@ -51,7 +58,7 @@ avx512_dp_packet_resize_l2(struct dp_packet *b, int
resize_by_bytes)
/* Only these lanes can be incremented/decremented for L2. */
const uint8_t k_lanes = 0b1110;
- __m128i v_offset = _mm_set1_epi16(VLAN_HEADER_LEN);
+ __m128i v_offset = _mm_set1_epi16(abs(resize_by_bytes));
/* Load packet and compare with UINT16_MAX */
void *adjust_ptr = &b->l2_pad_size;
@@ -63,8 +70,16 @@ avx512_dp_packet_resize_l2(struct dp_packet *b, int
resize_by_bytes)
v_u16_max);
/* Update VLAN_HEADER_LEN using compare mask, store results. */
- __m128i v_adjust_wip = _mm_mask_sub_epi16(v_adjust_src, k_cmp,
- v_adjust_src, v_offset);
+ __m128i v_adjust_wip;
+
+ if (resize_by_bytes >= 0) {
+ v_adjust_wip = _mm_mask_add_epi16(v_adjust_src, k_cmp,
+ v_adjust_src, v_offset);
+ } else {
+ v_adjust_wip = _mm_mask_sub_epi16(v_adjust_src, k_cmp,
+ v_adjust_src, v_offset);
+ }
+
_mm_storeu_si128(adjust_ptr, v_adjust_wip);
}
@@ -90,6 +105,40 @@ action_avx512_pop_vlan(void *dp OVS_UNUSED, struct
dp_packet_batch *batch,
}
}
+static void
+action_avx512_push_vlan(void *dp OVS_UNUSED, struct dp_packet_batch *batch,
+ const struct nlattr *a,
+ bool should_steal OVS_UNUSED)
+{
+ struct dp_packet *packet;
+ const struct ovs_action_push_vlan *vlan = nl_attr_get(a);
+ ovs_be16 tpid, tci;
+
+ DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+ tpid = vlan->vlan_tpid;
+ tci = vlan->vlan_tci;
+
+ avx512_dp_packet_resize_l2(packet, VLAN_HEADER_LEN);
+
+ /* Build up the VLAN TCI/TPID, and merge with the moving of Ether. */
+ char *pkt_data = (char *) dp_packet_data(packet);
+ const uint16_t tci_proc = tci & htons(~VLAN_CFI);
+ const uint32_t tpid_tci = (tci_proc << 16) | tpid;
+
+ static const uint8_t vlan_push_shuffle_mask[16] = {
+ 4, 5, 6, 7, 8, 9, 10, 11,
+ 12, 13, 14, 15, 0xFF, 0xFF, 0xFF, 0xFF
+ };
+
+ __m128i v_ether = _mm_loadu_si128((void *) pkt_data);
+ __m128i v_index = _mm_loadu_si128((void *) vlan_push_shuffle_mask);
+ __m128i v_shift = _mm_shuffle_epi8(v_ether, v_index);
+ __m128i v_vlan_hdr = _mm_insert_epi32(v_shift, tpid_tci, 3);
+ _mm_storeu_si128((void *) pkt_data, v_vlan_hdr);
+
+ }
+}
+
/* Probe functions to check ISA requirements. */
static int32_t
avx512_isa_probe(uint32_t needs_vbmi)
@@ -130,6 +179,7 @@ action_avx512_init(struct odp_execute_action_impl *self)
{
avx512_isa_probe(0);
self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_avx512_pop_vlan;
+ self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_avx512_push_vlan;
return 0;
}
diff --git a/lib/odp-execute-private.c b/lib/odp-execute-private.c
index 8257bba80..6b09ad353 100644
--- a/lib/odp-execute-private.c
+++ b/lib/odp-execute-private.c
@@ -228,6 +228,7 @@ int32_t
action_autoval_init(struct odp_execute_action_impl *self)
{
self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_autoval_generic;
+ self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_autoval_generic;
return 0;
}
diff --git a/lib/odp-execute.c b/lib/odp-execute.c
index ba532101f..690e7e1ce 100644
--- a/lib/odp-execute.c
+++ b/lib/odp-execute.c
@@ -845,6 +845,19 @@ action_pop_vlan(void *dp OVS_UNUSED, struct
dp_packet_batch *batch,
}
}
+static void
+action_push_vlan(void *dp OVS_UNUSED, struct dp_packet_batch *batch,
+ const struct nlattr *a OVS_UNUSED,
+ bool should_steal OVS_UNUSED)
+{
+ struct dp_packet *packet;
+ const struct ovs_action_push_vlan *vlan = nl_attr_get(a);
+
+ DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
+ eth_push_vlan(packet, vlan->vlan_tpid, vlan->vlan_tci);
+ }
+}
+
/* Implementation of the scalar actions impl init function. Build up the
* array of func ptrs here.
*/
@@ -852,6 +865,7 @@ int32_t
odp_action_scalar_init(struct odp_execute_action_impl *self)
{
self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_pop_vlan;
+ self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_push_vlan;
return 0;
}
@@ -994,15 +1008,6 @@ odp_execute_actions(void *dp, struct dp_packet_batch
*batch, bool steal,
break;
}
- case OVS_ACTION_ATTR_PUSH_VLAN: {
- const struct ovs_action_push_vlan *vlan = nl_attr_get(a);
-
- DP_PACKET_BATCH_FOR_EACH (i, packet, batch) {
- eth_push_vlan(packet, vlan->vlan_tpid, vlan->vlan_tci);
- }
- break;
- }
-
case OVS_ACTION_ATTR_PUSH_MPLS: {
const struct ovs_action_push_mpls *mpls = nl_attr_get(a);
@@ -1147,6 +1152,7 @@ odp_execute_actions(void *dp, struct dp_packet_batch
*batch, bool steal,
case OVS_ACTION_ATTR_OUTPUT:
case OVS_ACTION_ATTR_LB_OUTPUT:
case OVS_ACTION_ATTR_POP_VLAN:
+ case OVS_ACTION_ATTR_PUSH_VLAN:
case OVS_ACTION_ATTR_TUNNEL_PUSH:
case OVS_ACTION_ATTR_TUNNEL_POP:
case OVS_ACTION_ATTR_USERSPACE:
--
2.25.1
_______________________________________________
dev mailing list
[email protected]
https://mail.openvswitch.org/mailman/listinfo/ovs-dev
