This commit includes infrastructure changes for enabling set_masked_X actions and also adds support for the AVX512 implementation of the eth_set_addrs action.
Signed-off-by: Emma Finn <[email protected]> --- lib/odp-execute-avx512.c | 69 +++++++++++++++++++++++++++++++++++++++ lib/odp-execute-private.c | 56 +++++++++++++++++++++++++++++-- lib/odp-execute-private.h | 4 +++ lib/odp-execute.c | 65 +++++++++++++++++++++++++----------- lib/odp-execute.h | 3 ++ 5 files changed, 175 insertions(+), 22 deletions(-) diff --git a/lib/odp-execute-avx512.c b/lib/odp-execute-avx512.c index bb178cbac..ffe25b41d 100644 --- a/lib/odp-execute-avx512.c +++ b/lib/odp-execute-avx512.c @@ -38,6 +38,12 @@ BUILD_ASSERT_DECL(offsetof(struct dp_packet, l3_ofs) + MEMBER_SIZEOF(struct dp_packet, l3_ofs) == offsetof(struct dp_packet, l4_ofs)); +BUILD_ASSERT_DECL(offsetof(struct ovs_key_ethernet, eth_src) + + MEMBER_SIZEOF(struct ovs_key_ethernet, eth_src) == + offsetof(struct ovs_key_ethernet, eth_dst)); + +static struct odp_execute_action_impl avx512_impl; + /* Adjust the size of the l2 portion of the dp_packet, updating the l2 * pointer and the layer offsets. The function will broadcast resize_by_bytes * across a register and uses a kmask to identify which lanes should be @@ -144,6 +150,61 @@ action_avx512_push_vlan(struct dp_packet_batch *batch, const struct nlattr *a) } } +/* This function will load the contents of eth_header into a 128-bit wide + * register. Then an 8-byte shuffle is required to shuffle both key and + * mask to match the layout of the eth_header struct. A bitwise ANDNOT and OR + * is performed on the entire header and results are stored back. */ +static void +action_avx512_eth_set_addrs(struct dp_packet_batch *batch, + const struct nlattr *a) +{ + a = nl_attr_get(a); + const struct ovs_key_ethernet *key = nl_attr_get(a); + const struct ovs_key_ethernet *mask = get_mask(a, struct ovs_key_ethernet); + struct dp_packet *packet; + + __m128i v_src = _mm_loadu_si128((void *) key); + __m128i v_mask = _mm_loadu_si128((void *) mask); + + DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { + + struct eth_header *eh = dp_packet_eth(packet); + + if (!eh) { + continue; + } + + static const uint8_t eth_shuffle[16] = { + 6, 7, 8, 9, 10, 11, 0, 1, + 2, 3, 4, 5, 12, 13, 14, 15 + }; + + __m128i v_dst = _mm_loadu_si128((void *) eh); + __m128i v_shuf = _mm_loadu_si128((void *) eth_shuffle); + + v_src = _mm_shuffle_epi8(v_src, v_shuf); + v_mask = _mm_shuffle_epi8(v_mask, v_shuf); + + __m128i dst_masked = _mm_andnot_si128(v_mask, v_dst); + __m128i res = _mm_or_si128(v_src, dst_masked); + + __m128i res_blend = _mm_blend_epi16(v_dst, res, 0x3F); + _mm_storeu_si128((void *) eh, res_blend); + } +} + +static void +action_avx512_set_masked(struct dp_packet_batch *batch OVS_UNUSED, + const struct nlattr *a) +{ + a = nl_attr_get(a); + enum ovs_key_attr attr_type = nl_attr_type(a); + + if (avx512_impl.set_masked_funcs[attr_type]) { + avx512_impl.set_masked_funcs[attr_type](batch, a); + } +} + /* Probe functions to check ISA requirements. */ static bool avx512_isa_probe(void) @@ -176,6 +237,14 @@ action_avx512_init(struct odp_execute_action_impl *self) * are identified by OVS_ACTION_ATTR_*. */ self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_avx512_pop_vlan; self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_avx512_push_vlan; + self->funcs[OVS_ACTION_ATTR_SET_MASKED] = action_avx512_set_masked; + + /* Set function pointers that need a 2nd-level function. SET_MASKED action + * requires further processing for action type. Note that 2nd level items + * are identified by OVS_KEY_ATTR_*. */ + self->set_masked_funcs[OVS_KEY_ATTR_ETHERNET] = + action_avx512_eth_set_addrs; + avx512_impl = *self; return 0; } diff --git a/lib/odp-execute-private.c b/lib/odp-execute-private.c index 751a68fe3..e2d650779 100644 --- a/lib/odp-execute-private.c +++ b/lib/odp-execute-private.c @@ -29,6 +29,8 @@ VLOG_DEFINE_THIS_MODULE(odp_execute_impl); static int active_action_impl_index; static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 1); +static struct odp_execute_action_impl autoval_impl; +static bool set_masked = false; static struct odp_execute_action_impl action_impls[] = { [ACTION_IMPL_AUTOVALIDATOR] = { @@ -59,6 +61,11 @@ action_impl_copy_funcs(struct odp_execute_action_impl *src, for (int i = 0; i < __OVS_ACTION_ATTR_MAX; i++) { atomic_store_relaxed(&src->funcs[i], dst->funcs[i]); } + + for (uint32_t i = 0; i < __OVS_KEY_ATTR_MAX; i++) { + atomic_store_relaxed(&src->set_masked_funcs[i], + dst->set_masked_funcs[i]); + } } int @@ -135,19 +142,36 @@ action_autoval_generic(struct dp_packet_batch *batch, const struct nlattr *a) bool failed = false; int type = nl_attr_type(a); enum ovs_action_attr attr_type = (enum ovs_action_attr) type; + enum ovs_key_attr key_attr_type = (enum ovs_key_attr) type; + + if (attr_type == OVS_ACTION_ATTR_SET_MASKED) { + set_masked = true; + const struct nlattr *key = nl_attr_get(a); + key_attr_type = nl_attr_type(key); + } + struct odp_execute_action_impl *scalar = &action_impls[ACTION_IMPL_SCALAR]; struct dp_packet_batch good_batch; dp_packet_batch_clone(&good_batch, batch); - scalar->funcs[attr_type](&good_batch, a); + if (!set_masked) { + scalar->funcs[attr_type](&good_batch, a); + } else { + scalar->set_masked_funcs[key_attr_type](&good_batch, a); + } for (int impl = ACTION_IMPL_BEGIN; impl < ACTION_IMPL_MAX; impl++) { /* Clone original batch and execute implementation under test. */ struct dp_packet_batch test_batch; dp_packet_batch_clone(&test_batch, batch); - action_impls[impl].funcs[attr_type](&test_batch, a); + + if (!set_masked) { + action_impls[impl].funcs[attr_type](&test_batch, a); + } else { + action_impls[impl].set_masked_funcs[key_attr_type](&test_batch, a); + } /* Loop over implementations, checking each one. */ for (int pidx = 0; pidx < batch->count; pidx++) { @@ -200,7 +224,26 @@ action_autoval_generic(struct dp_packet_batch *batch, const struct nlattr *a) dp_packet_delete_batch(&good_batch, 1); /* Apply the action to the original batch for continued processing. */ - scalar->funcs[attr_type](batch, a); + if (!set_masked) { + scalar->funcs[attr_type](batch, a); + } else { + scalar->set_masked_funcs[key_attr_type](batch, a); + } + + set_masked = false; +} + +static void +action_set_masked_init(struct dp_packet_batch *batch OVS_UNUSED, + const struct nlattr *a) +{ + const struct nlattr *type = nl_attr_get(a); + enum ovs_key_attr attr_type = nl_attr_type(type); + + if (autoval_impl.set_masked_funcs[attr_type]) { + set_masked = true; + autoval_impl.set_masked_funcs[attr_type](batch, a); + } } int @@ -210,6 +253,13 @@ action_autoval_init(struct odp_execute_action_impl *self) * are identified by OVS_ACTION_ATTR_*. */ self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_autoval_generic; self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_autoval_generic; + self->funcs[OVS_ACTION_ATTR_SET_MASKED] = action_set_masked_init; + + /* Set function pointers that need a 2nd-level function. SET_MASKED action + * requires further processing for action type. Note that 2nd level items + * are identified by OVS_KEY_ATTR_*. */ + self->set_masked_funcs[OVS_KEY_ATTR_ETHERNET] = action_autoval_generic; + autoval_impl = *self; return 0; } diff --git a/lib/odp-execute-private.h b/lib/odp-execute-private.h index e4724b8b2..1f4d614ca 100644 --- a/lib/odp-execute-private.h +++ b/lib/odp-execute-private.h @@ -49,6 +49,10 @@ struct odp_execute_action_impl { /* An array of callback functions, one for each action. */ ATOMIC(odp_execute_action_cb) funcs[__OVS_ACTION_ATTR_MAX]; + + /* An array of callback functions, one for each action type. */ + ATOMIC(odp_execute_action_cb) set_masked_funcs[__OVS_KEY_ATTR_MAX]; + }; /* Order of Actions implementations. */ diff --git a/lib/odp-execute.c b/lib/odp-execute.c index 59f6bdc64..db6e1ec03 100644 --- a/lib/odp-execute.c +++ b/lib/odp-execute.c @@ -561,8 +561,6 @@ odp_execute_set_action(struct dp_packet *packet, const struct nlattr *a) } } -#define get_mask(a, type) ((const type *)(const void *)(a + 1) + 1) - static void odp_execute_masked_set_action(struct dp_packet *packet, const struct nlattr *a) @@ -582,11 +580,6 @@ odp_execute_masked_set_action(struct dp_packet *packet, | (md->pkt_mark & ~*get_mask(a, uint32_t)); break; - case OVS_KEY_ATTR_ETHERNET: - odp_eth_set_addrs(packet, nl_attr_get(a), - get_mask(a, struct ovs_key_ethernet)); - break; - case OVS_KEY_ATTR_NSH: { odp_set_nsh(packet, a, true); break; @@ -669,6 +662,8 @@ odp_execute_masked_set_action(struct dp_packet *packet, case OVS_KEY_ATTR_TCP_FLAGS: case OVS_KEY_ATTR_TUNNEL_INFO: case __OVS_KEY_ATTR_MAX: + /* The following action types are handled by the scalar implementation. */ + case OVS_KEY_ATTR_ETHERNET: default: OVS_NOT_REACHED(); } @@ -834,6 +829,12 @@ requires_datapath_assistance(const struct nlattr *a) return false; } +/* The active function pointers on the datapath. ISA optimized implementations + * are enabled by plugging them into this static arary, which is consulted when + * applying actions on the datapath. + */ +static struct odp_execute_action_impl actions_active_impl; + static void action_pop_vlan(struct dp_packet_batch *batch, const struct nlattr *a OVS_UNUSED) @@ -856,6 +857,36 @@ action_push_vlan(struct dp_packet_batch *batch, const struct nlattr *a) } } +static void +action_set_masked(struct dp_packet_batch *batch, const struct nlattr *a) +{ + struct dp_packet *packet; + + const struct nlattr *key = nl_attr_get(a); + enum ovs_key_attr key_type = nl_attr_type(key); + + if (actions_active_impl.set_masked_funcs[key_type]) { + actions_active_impl.set_masked_funcs[key_type](batch, a); + } else { + a = nl_attr_get(a); + DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { + odp_execute_masked_set_action(packet, a); + } + } +} + +static void +action_mod_eth(struct dp_packet_batch *batch, const struct nlattr *a) +{ + a = nl_attr_get(a); + struct dp_packet *packet; + + DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { + odp_eth_set_addrs(packet, nl_attr_get(a), + get_mask(a, struct ovs_key_ethernet)); + } +} + /* Implementation of the scalar actions impl init function. Build up the * array of func ptrs here. */ @@ -866,16 +897,17 @@ odp_action_scalar_init(struct odp_execute_action_impl *self) * are identified by OVS_ACTION_ATTR_*. */ self->funcs[OVS_ACTION_ATTR_POP_VLAN] = action_pop_vlan; self->funcs[OVS_ACTION_ATTR_PUSH_VLAN] = action_push_vlan; + self->funcs[OVS_ACTION_ATTR_SET_MASKED] = action_set_masked; + + /* Set function pointers that need a 2nd-level function. SET_MASKED action + * requires further processing for action type. Note that 2nd level items + * are identified by OVS_KEY_ATTR_*. */ + self->set_masked_funcs[OVS_KEY_ATTR_ETHERNET] = action_mod_eth; + actions_active_impl = *self; return 0; } -/* The active function pointers on the datapath. ISA optimized implementations - * are enabled by plugging them into this static arary, which is consulted when - * applying actions on the datapath. - */ -static struct odp_execute_action_impl actions_active_impl; - void odp_execute_init(void) { @@ -1028,12 +1060,6 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal, } break; - case OVS_ACTION_ATTR_SET_MASKED: - DP_PACKET_BATCH_FOR_EACH(i, packet, batch) { - odp_execute_masked_set_action(packet, nl_attr_get(a)); - } - break; - case OVS_ACTION_ATTR_SAMPLE: DP_PACKET_BATCH_FOR_EACH (i, packet, batch) { odp_execute_sample(dp, packet, steal && last_action, a, @@ -1160,6 +1186,7 @@ odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal, /* The following actions are handled by the scalar implementation. */ case OVS_ACTION_ATTR_POP_VLAN: case OVS_ACTION_ATTR_PUSH_VLAN: + case OVS_ACTION_ATTR_SET_MASKED: OVS_NOT_REACHED(); } diff --git a/lib/odp-execute.h b/lib/odp-execute.h index 8668ab73f..762b99473 100644 --- a/lib/odp-execute.h +++ b/lib/odp-execute.h @@ -50,4 +50,7 @@ void odp_execute_actions(void *dp, struct dp_packet_batch *batch, bool steal, const struct nlattr *actions, size_t actions_len, odp_execute_cb dp_execute_action); + +#define get_mask(a, type) ((const type *)(const void *)(a + 1) + 1) + #endif -- 2.32.0 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
