From: Peng Zhang <[email protected]> For dpif-netdev, meters are mapped by DPDK meter with one-to-one relationship. Implement meter offload API to set/get/del the DPDK meter.
Creating the meter data hash table to store the meter data configure. With set/get/del DPDK meter offload API, store/update/get/del the hash node in the meter data hash table. Signed-off-by: Peng Zhang <[email protected]> Signed-off-by: Jin Liu <[email protected]> Co-authored-by: Jin Liu <[email protected]> Signed-off-by: Simon Horman <[email protected]> --- lib/netdev-dpdk.c | 191 ++++++++++++++++++++++++++++++++++++++ lib/netdev-dpdk.h | 41 ++++++++ lib/netdev-offload-dpdk.c | 191 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 423 insertions(+) diff --git a/lib/netdev-dpdk.c b/lib/netdev-dpdk.c index 278d6afc08af..c2f527a1456f 100644 --- a/lib/netdev-dpdk.c +++ b/lib/netdev-dpdk.c @@ -37,6 +37,7 @@ #include <rte_malloc.h> #include <rte_mbuf.h> #include <rte_meter.h> +#include <rte_mtr.h> #include <rte_pci.h> #include <rte_version.h> #include <rte_vhost.h> @@ -5341,8 +5342,198 @@ netdev_dpdk_rte_flow_query_count(struct netdev *netdev, return ret; } +static int OVS_UNUSED +netdev_dpdk_meter_profile_init(struct rte_mtr_meter_profile *profile, + struct rte_mtr_capabilities *cap, + const uint64_t rate, + const uint64_t burst, + const int flag) +{ + if (!cap->meter_srtcm_rfc2697_n_max) { + return EOPNOTSUPP; + } + + profile->alg = RTE_MTR_SRTCM_RFC2697; + profile->packet_mode = flag; + profile->srtcm_rfc2697.cir = rate; + profile->srtcm_rfc2697.cbs = burst; + profile->srtcm_rfc2697.ebs = burst; + + return 0; +} + #ifdef ALLOW_EXPERIMENTAL_API +static int +netdev_dpdk_rte_mtr_meter_add(struct rte_mtr_meter_profile *profile, + const int proxy_port_id, + uint32_t meter_id, + const uint32_t rate, + const uint32_t burst, + const int flag, + struct rte_mtr_error *error) +{ + uint32_t meter_profile_id = meter_id; + uint32_t meter_policy_id = meter_id; + struct rte_mtr_capabilities cap; + struct rte_mtr_stats mtr_stats; + struct rte_mtr_params params; + uint64_t stats_mask = 0; + int clear = 0; + int mod; + int ret; + + memset(&mtr_stats, 0, sizeof(struct rte_mtr_stats)); + memset(&cap, 0, sizeof(cap)); + + ret = rte_mtr_capabilities_get(proxy_port_id, &cap, error); + if (ret) { + return ret; + } + + ret = netdev_dpdk_meter_profile_init(profile, &cap, rate, burst, flag); + if (ret) { + return ret; + } + + /* If can get the meter stats, the meter is offload in the HW. + * So the operate is mod, just update the meter_profile. + * + * If can't get the meter stats, the meter is not offload in the HW. + * So the operate is add, need create the profile, policy, mtr. */ + mod = rte_mtr_stats_read(proxy_port_id, meter_id, &mtr_stats, &stats_mask, + clear, error); + ret = rte_mtr_meter_profile_add(proxy_port_id, meter_profile_id, profile, + error); + if (!mod || ret) { + return ret; + } + + rte_mtr_policy_drop_red(policy); + ret = rte_mtr_meter_policy_add(proxy_port_id, meter_policy_id, &policy, + error); + + if (ret) { + return ret; + } + + memset(¶ms, 0 , sizeof(struct rte_mtr_params)); + params.meter_profile_id = meter_profile_id; + params.meter_policy_id = meter_policy_id; + params.stats_mask = cap.stats_mask; + params.meter_enable = 1; + + ret = rte_mtr_create(proxy_port_id, meter_id, ¶ms, 1, error); + return ret; +} + +int +netdev_dpdk_meter_create(const int proxy_port_id, + const uint32_t meter_profile_id, + const uint64_t rate, + const uint64_t burst, + const int flag) +{ + struct rte_mtr_meter_profile profile; + struct rte_mtr_error error; + int ret; + + memset(&profile, 0 , sizeof(struct rte_mtr_meter_profile)); + memset(&error, 0 , sizeof(struct rte_mtr_error)); + + ret = netdev_dpdk_rte_mtr_meter_add(&profile, proxy_port_id, + meter_profile_id, rate, burst, flag, + &error); + if (ret) { + VLOG_DBG("port %d: rte_mtr creation failed: %u (%s).", proxy_port_id, + error.type, error.message); + return ret; + } + + VLOG_DBG("port %d: rte_meter_id %u mtr create ", proxy_port_id, + meter_profile_id); + + return ret; +} + +int +netdev_dpdk_meter_del(const int proxy_port_id, + const uint32_t meter_id, + const uint32_t meter_profile_id, + const uint32_t meter_policy_id) +{ + struct rte_mtr_stats mtr_stats; + struct rte_mtr_error error; + uint64_t stats_mask = 0; + int clear = 0; + int ret = 0 ; + + memset(&mtr_stats, 0, sizeof(struct rte_mtr_stats)); + memset(&error, 0 , sizeof(struct rte_mtr_error)); + + ret = rte_mtr_stats_read(proxy_port_id, meter_id, &mtr_stats, &stats_mask, + clear, &error); + if (ret) { + VLOG_DBG("port %d: rte_mtr find mtr_id %d failed: %d (%s).", + proxy_port_id, meter_id, error.type, error.message); + return ret; + } + + ret = rte_mtr_destroy(proxy_port_id, meter_id, &error); + if (ret) { + VLOG_DBG("port %d: rte_mtr delete mtr_id %d failed: %d (%s).", + proxy_port_id, meter_id, error.type, error.message); + return ret; + } + + ret = rte_mtr_meter_policy_delete(proxy_port_id, meter_policy_id, &error); + if (ret) { + VLOG_DBG("port %d: rte_mtr delete meter_policy %d failed: %d (%s).", + proxy_port_id, meter_policy_id, error.type, error.message); + return ret; + } + + ret = rte_mtr_meter_profile_delete(proxy_port_id, meter_profile_id, + &error); + if (ret) { + VLOG_DBG("port %d: rte_mtr delete meter_profile %d failed: %d (%s).", + proxy_port_id, meter_profile_id, error.type, error.message); + } + + return ret; +} + +int +netdev_dpdk_meter_get(const int proxy_port_id, + const uint32_t meter_id, + uint64_t *byte_in_count, + uint64_t *packet_in_count) +{ + struct rte_mtr_stats mtr_stats; + struct rte_mtr_error error; + uint64_t stats_mask = 0; + int clear = 0; + int ret = 0 ; + + memset(&mtr_stats, 0, sizeof(struct rte_mtr_stats)); + memset(&error, 0, sizeof(struct rte_mtr_error)); + + ret = rte_mtr_stats_read(proxy_port_id, meter_id, + &mtr_stats, &stats_mask, clear, &error); + if (ret) { + VLOG_DBG("port %d: rte_mtr get mtr_id %d stats failed: %d (%s).", + proxy_port_id, meter_id, error.type, error.message); + return ret; + } + + *byte_in_count = mtr_stats.n_bytes[RTE_COLOR_GREEN]; + *packet_in_count = mtr_stats.n_pkts[RTE_COLOR_GREEN]; + VLOG_DBG("port %d: rte_meter_id %d mtr get stats success ", proxy_port_id, + meter_id); + + return ret; +} + int netdev_dpdk_rte_flow_tunnel_decap_set(struct netdev *netdev, struct rte_flow_tunnel *tunnel, diff --git a/lib/netdev-dpdk.h b/lib/netdev-dpdk.h index 5ba59a82e8a7..ffa866bb5291 100644 --- a/lib/netdev-dpdk.h +++ b/lib/netdev-dpdk.h @@ -57,6 +57,19 @@ netdev_dpdk_get_prox_port_id(struct netdev *netdev); #ifdef ALLOW_EXPERIMENTAL_API +int netdev_dpdk_meter_create(const int proxy_port_id, + const uint32_t meter_profile_id, + const uint64_t rate, + const uint64_t burst, + const int flag); +int netdev_dpdk_meter_del(const int proxy_port_id, + const uint32_t meter_id, + const uint32_t meter_profile_id, + const uint32_t meter_policy_id); +int netdev_dpdk_meter_get(const int proxy_port_id, + const uint32_t meter_id, + uint64_t *byte_in_count, + uint64_t *packet_in_count); int netdev_dpdk_rte_flow_tunnel_decap_set(struct netdev *, struct rte_flow_tunnel *, struct rte_flow_action **, @@ -82,6 +95,34 @@ int netdev_dpdk_rte_flow_tunnel_item_release(struct netdev *, #else +static inline int +netdev_dpdk_meter_create(const int proxy_port_id OVS_UNUSED, + const uint32_t meter_profile_id OVS_UNUSED, + const uint64_t rate OVS_UNUSED, + const uint64_t burst OVS_UNUSED, + const int flag OVS_UNUSED) +{ + return -1; +} + +static inline int +netdev_dpdk_meter_del(const int proxy_port_id OVS_UNUSED, + const uint32_t meter_id OVS_UNUSED, + const uint32_t meter_profile_id OVS_UNUSED, + const uint32_t meter_policy_id OVS_UNUSED) +{ + return -1; +} + +static inline int +netdev_dpdk_meter_get(const int proxy_port_id OVS_UNUSED, + const uint32_t meter_id OVS_UNUSED, + uint64_t *byte_in_count OVS_UNUSED, + uint64_t *packet_in_count OVS_UNUSED) +{ + return -1; +} + static inline void set_error(struct rte_flow_error *error, enum rte_flow_error_type type) { diff --git a/lib/netdev-offload-dpdk.c b/lib/netdev-offload-dpdk.c index 9b594ac1ef90..9d010d6003f3 100644 --- a/lib/netdev-offload-dpdk.c +++ b/lib/netdev-offload-dpdk.c @@ -75,6 +75,39 @@ struct netdev_offload_dpdk_data { struct ovs_mutex map_lock; }; +static struct ovs_mutex netdev_meter_mutex = OVS_MUTEX_INITIALIZER; + +static struct cmap netdev_meter = CMAP_INITIALIZER; + +struct dpdk_meter_proxy { + int proxy_id; + uint64_t refcnt; + /* In meter_id_to_meter_data's "meter_relate_proxy" list. */ + struct ovs_list node; +}; + +struct meter_id_to_meter_data { + struct ovs_list meter_relate_proxy; + struct cmap_node cmap_node; + ofproto_meter_id meter_id; + uint64_t burst; + uint64_t rate; + int flag; +}; + +static struct meter_id_to_meter_data * +netdev_lookup_meter_data(const uint32_t meter_id) +{ + struct meter_id_to_meter_data *meter_data; + CMAP_FOR_EACH_WITH_HASH (meter_data, cmap_node, meter_id, + &netdev_meter) { + if (meter_id == meter_data->meter_id.uint32) { + return meter_data; + } + } + return NULL; +} + static int offload_data_init(struct netdev *netdev) { @@ -2456,6 +2489,161 @@ netdev_offload_dpdk_flow_del(struct netdev *netdev OVS_UNUSED, return netdev_offload_dpdk_flow_destroy(rte_flow_data); } +static void +netdev_offload_dpdk_meter_update(uint64_t rate, uint64_t burst, + uint32_t mid, int flag, + struct meter_id_to_meter_data *meter_data) +{ + struct dpdk_meter_proxy *dmp; + int ret; + + meter_data->rate = rate; + meter_data->burst = burst; + meter_data->flag = flag; + + LIST_FOR_EACH (dmp, node, &meter_data->meter_relate_proxy) { + if (!dmp->refcnt) { + continue; + } + + ret = netdev_dpdk_meter_create(dmp->proxy_id, mid, rate, burst, flag); + if (ret) { + VLOG_ERR("Failed update the meter %u to the port %d", + mid, dmp->proxy_id); + } + } +} + +static int +netdev_offload_dpdk_meter_set(const char *dpif_type, + ofproto_meter_id meter_id, + struct ofputil_meter_config *config) +{ + struct meter_id_to_meter_data *meter_data; + uint32_t mid = meter_id.uint32; + uint64_t burst; + uint64_t rate; + int flag; + + if (!strcmp(dpif_type, "system")) { + VLOG_DBG("meter belongs to the system datapath. Skipping."); + return EOPNOTSUPP; + } + + if (config->n_bands != 1 || config->bands[0].type != OFPMBT13_DROP) { + return 0; + } + + if (!(config->flags & (OFPMF13_KBPS | OFPMF13_PKTPS))) { + return EBADF; + } + + flag = !(config->flags & OFPMF13_KBPS); + rate = config->bands[0].rate; + burst = config->bands[0].burst_size; + if (flag == 0) { + rate *= 1024 / 8; + burst *= 1024 / 8; + } + + if (!config->bands[0].burst_size) { + burst = rate / 5; + } + + ovs_mutex_lock(&netdev_meter_mutex); + meter_data = netdev_lookup_meter_data(meter_id.uint32); + if (meter_data != NULL) { + netdev_offload_dpdk_meter_update(rate, burst, flag, mid, meter_data); + goto out; + } + + meter_data = xmalloc(sizeof *meter_data); + meter_data->meter_id = meter_id; + meter_data->rate = rate; + meter_data->burst = burst; + meter_data->flag = flag; + ovs_list_init(&meter_data->meter_relate_proxy); + cmap_insert(&netdev_meter, &meter_data->cmap_node, meter_id.uint32); + VLOG_DBG("netdev: meter '%u' is stored .", meter_id.uint32); + +out: + ovs_mutex_unlock(&netdev_meter_mutex); + return 0; +} + +static int +netdev_offload_dpdk_meter_del(const char *dpif_type, + ofproto_meter_id meter_id, + struct ofputil_meter_stats *stats) +{ + struct meter_id_to_meter_data *meter_data; + + if (!strcmp(dpif_type, "system")) { + VLOG_DBG("meter belongs to the system datapath. Skipping."); + return EOPNOTSUPP; + } + + ovs_mutex_lock(&netdev_meter_mutex); + meter_data = netdev_lookup_meter_data(meter_id.uint32); + if (meter_data == NULL) { + VLOG_WARN("attempted to remove meter data that is not exist: %u", + meter_id.uint32); + goto out; + } + + cmap_remove(&netdev_meter, &meter_data->cmap_node, meter_id.uint32); + ovsrcu_postpone(free, meter_data); + if (stats) { + memset(stats, 0, sizeof *stats); + } + +out: + ovs_mutex_unlock(&netdev_meter_mutex); + return 0; +} + +static int +netdev_offload_dpdk_meter_get(const char *dpif_type, + ofproto_meter_id meter_id, + struct ofputil_meter_stats *stats) +{ + struct meter_id_to_meter_data *meter_data; + struct dpdk_meter_proxy *dmp; + uint64_t packet_in_count; + uint64_t byte_in_count; + int ret = 0; + + if (!strcmp(dpif_type, "system")) { + VLOG_DBG("meter belongs to the system datapath. Skipping."); + return EOPNOTSUPP; + } + + ovs_mutex_lock(&netdev_meter_mutex); + meter_data = netdev_lookup_meter_data(meter_id.uint32); + if (meter_data == NULL) { + VLOG_WARN("attempted to get meter status that is not exist: %u", + meter_id.uint32); + goto out; + } + + LIST_FOR_EACH (dmp, node, &meter_data->meter_relate_proxy) { + byte_in_count = 0; + packet_in_count = 0; + ret = netdev_dpdk_meter_get(dmp->proxy_id, meter_id.uint32, + &byte_in_count, &packet_in_count); + if (ret) { + VLOG_ERR("Failed get the meter frome the port %d", dmp->proxy_id); + } + + stats->byte_in_count += byte_in_count; + stats->packet_in_count += packet_in_count; + } + +out: + ovs_mutex_unlock(&netdev_meter_mutex); + return ret; +} + static int netdev_offload_dpdk_init_flow_api(struct netdev *netdev) { @@ -2742,6 +2930,9 @@ const struct netdev_flow_api netdev_offload_dpdk = { .type = "dpdk_flow_api", .flow_put = netdev_offload_dpdk_flow_put, .flow_del = netdev_offload_dpdk_flow_del, + .meter_set = netdev_offload_dpdk_meter_set, + .meter_get = netdev_offload_dpdk_meter_get, + .meter_del = netdev_offload_dpdk_meter_del, .init_flow_api = netdev_offload_dpdk_init_flow_api, .uninit_flow_api = netdev_offload_dpdk_uninit_flow_api, .flow_get = netdev_offload_dpdk_flow_get, -- 2.30.2 _______________________________________________ dev mailing list [email protected] https://mail.openvswitch.org/mailman/listinfo/ovs-dev
