Author: hselasky
Date: Wed Mar  7 15:17:36 2018
New Revision: 330607
URL: https://svnweb.freebsd.org/changeset/base/330607

Log:
  Implement rate limit per traffic class in mlx5core.
  
  Add support for rate limiting traffic class via sysctl.
  
  Submitted by: Slava Shwartsman <slav...@mellanox.com>
  MFC after:    1 week
  Sponsored by: Mellanox Technologies

Modified:
  head/sys/dev/mlx5/mlx5_core/mlx5_port.c
  head/sys/dev/mlx5/mlx5_en/en.h
  head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
  head/sys/dev/mlx5/mlx5_ifc.h
  head/sys/dev/mlx5/port.h

Modified: head/sys/dev/mlx5/mlx5_core/mlx5_port.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_core/mlx5_port.c     Wed Mar  7 15:03:11 2018        
(r330606)
+++ head/sys/dev/mlx5/mlx5_core/mlx5_port.c     Wed Mar  7 15:17:36 2018        
(r330607)
@@ -860,6 +860,89 @@ int mlx5_query_port_cong_params(struct mlx5_core_dev *
                                          out, out_size);
 }
 
+static int mlx5_query_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *out,
+                                    int outlen)
+{
+       u32 in[MLX5_ST_SZ_DW(qtct_reg)];
+
+       if (!MLX5_CAP_GEN(mdev, ets))
+               return -ENOTSUPP;
+
+       memset(in, 0, sizeof(in));
+       return mlx5_core_access_reg(mdev, in, sizeof(in), out, outlen,
+                                   MLX5_REG_QETCR, 0, 0);
+}
+
+int mlx5_max_tc(struct mlx5_core_dev *mdev)
+{
+       u8 num_tc = MLX5_CAP_GEN(mdev, max_tc) ? : 8;
+
+       return num_tc - 1;
+}
+EXPORT_SYMBOL_GPL(mlx5_max_tc);
+
+static int mlx5_set_port_qetcr_reg(struct mlx5_core_dev *mdev, u32 *in,
+                                  int inlen)
+{
+       u32 out[MLX5_ST_SZ_DW(qtct_reg)];
+
+       if (!MLX5_CAP_GEN(mdev, ets))
+               return -ENOTSUPP;
+
+       return mlx5_core_access_reg(mdev, in, inlen, out, sizeof(out),
+                                   MLX5_REG_QETCR, 0, 1);
+}
+
+int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+                                  u8 *max_bw_value,
+                                  u8 *max_bw_units)
+{
+       u32 out[MLX5_ST_SZ_DW(qetc_reg)];
+       void *ets_tcn_conf;
+       int err;
+       int i;
+
+       err = mlx5_query_port_qetcr_reg(mdev, out, sizeof(out));
+       if (err)
+               return err;
+
+       for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+               ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, out, tc_configuration[i]);
+
+               max_bw_value[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf,
+                                          max_bw_value);
+               max_bw_units[i] = MLX5_GET(ets_tcn_config_reg, ets_tcn_conf,
+                                          max_bw_units);
+       }
+
+       return 0;
+}
+EXPORT_SYMBOL_GPL(mlx5_query_port_tc_rate_limit);
+
+int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+                                  const u8 *max_bw_value,
+                                  const u8 *max_bw_units)
+{
+       u32 in[MLX5_ST_SZ_DW(qetc_reg)] = {};
+       void *ets_tcn_conf;
+       int i;
+
+       MLX5_SET(qetc_reg, in, port_number, 1);
+
+       for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+               ets_tcn_conf = MLX5_ADDR_OF(qetc_reg, in, tc_configuration[i]);
+
+               MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, r, 1);
+               MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_units,
+                        max_bw_units[i]);
+               MLX5_SET(ets_tcn_config_reg, ets_tcn_conf, max_bw_value,
+                        max_bw_value[i]);
+       }
+
+       return mlx5_set_port_qetcr_reg(mdev, in, sizeof(in));
+}
+EXPORT_SYMBOL_GPL(mlx5_modify_port_tc_rate_limit);
+
 int mlx5_modify_port_cong_params(struct mlx5_core_dev *mdev,
                                 void *in, int in_size)
 {

Modified: head/sys/dev/mlx5/mlx5_en/en.h
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/en.h      Wed Mar  7 15:03:11 2018        
(r330606)
+++ head/sys/dev/mlx5/mlx5_en/en.h      Wed Mar  7 15:17:36 2018        
(r330607)
@@ -70,6 +70,8 @@
 #include <dev/mlx5/mlx5_core/transobj.h>
 #include <dev/mlx5/mlx5_core/mlx5_core.h>
 
+#define        IEEE_8021QAZ_MAX_TCS    8
+
 #define        MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE                0x7
 #define        MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE                0xa
 #define        MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE                0xe
@@ -114,6 +116,9 @@
   (MLX5E_MAX_TX_HEADER - sizeof(struct mlx5e_tx_wqe) + \
   sizeof(((struct mlx5e_tx_wqe *)0)->eth.inline_hdr_start))    /* bytes */
 
+#define        MLX5E_100MB (100000)
+#define        MLX5E_1GB   (1000000)
+
 MALLOC_DECLARE(M_MLX5EN);
 
 struct mlx5_core_dev;
@@ -417,11 +422,13 @@ struct mlx5e_params {
   m(+1, u64 mc_local_lb, "mc_local_lb", "0: Local multicast loopback enabled 
1: Disabled") \
   m(+1, u64 uc_local_lb, "uc_local_lb", "0: Local unicast loopback enabled 1: 
Disabled")
 
+
 #define        MLX5E_PARAMS_NUM (0 MLX5E_PARAMS(MLX5E_STATS_COUNT))
 
 struct mlx5e_params_ethtool {
        u64     arg [0];
        MLX5E_PARAMS(MLX5E_STATS_VAR)
+       u64     max_bw_value[IEEE_8021QAZ_MAX_TCS];
 };
 
 /* EEPROM Standards for plug in modules */
@@ -629,6 +636,12 @@ enum {
        MLX5E_STATE_OPENED,
 };
 
+enum {
+       MLX5_BW_NO_LIMIT   = 0,
+       MLX5_100_MBPS_UNIT = 3,
+       MLX5_GBPS_UNIT     = 4,
+};
+
 struct mlx5e_vlan_db {
        unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)];
        struct mlx5_flow_rule   *active_vlans_ft_rule[VLAN_N_VID];
@@ -747,6 +760,19 @@ struct mlx5e_eeprom {
        int     page_valid;
        u32     *data;
 };
+
+/*
+ * This structure contains rate limit extension to the IEEE 802.1Qaz ETS
+ * managed object.
+ * Values are 64 bits long and specified in Kbps to enable usage over both
+ * slow and very fast networks.
+ *
+ * @tc_maxrate: maximal tc tx bandwidth indexed by traffic class
+ */
+struct ieee_maxrate {
+       __u64   tc_maxrate[IEEE_8021QAZ_MAX_TCS];
+};
+
 
 #define        MLX5E_FLD_MAX(typ, fld) ((1ULL << __mlx5_bit_sz(typ, fld)) - 
1ULL)
 

Modified: head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c
==============================================================================
--- head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c Wed Mar  7 15:03:11 2018        
(r330606)
+++ head/sys/dev/mlx5/mlx5_en/mlx5_en_ethtool.c Wed Mar  7 15:17:36 2018        
(r330607)
@@ -84,6 +84,97 @@ mlx5e_ethtool_sync_tx_completion_fact(struct mlx5e_pri
                priv->params_ethtool.tx_completion_fact = max;
 }
 
+static int
+mlx5e_getmaxrate(struct mlx5e_priv *priv)
+{
+       struct mlx5_core_dev *mdev = priv->mdev;
+       u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS];
+       u8 max_bw_value[IEEE_8021QAZ_MAX_TCS];
+       int err;
+       int i;
+
+       PRIV_LOCK(priv);
+       err = -mlx5_query_port_tc_rate_limit(mdev, max_bw_value, max_bw_unit);
+       if (err)
+               goto done;
+
+       for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+               switch (max_bw_unit[i]) {
+               case MLX5_100_MBPS_UNIT:
+                       priv->params_ethtool.max_bw_value[i] = max_bw_value[i] 
* MLX5E_100MB;
+                       break;
+               case MLX5_GBPS_UNIT:
+                       priv->params_ethtool.max_bw_value[i] = max_bw_value[i] 
* MLX5E_1GB;
+                       break;
+               case MLX5_BW_NO_LIMIT:
+                       priv->params_ethtool.max_bw_value[i] = 0;
+                       break;
+               default:
+                       priv->params_ethtool.max_bw_value[i] = -1;
+                       WARN_ONCE(true, "non-supported BW unit");
+                       break;
+               }
+       }
+done:
+       PRIV_UNLOCK(priv);
+       return (err);
+}
+
+static int
+mlx5e_tc_maxrate_handler(SYSCTL_HANDLER_ARGS)
+{
+       struct mlx5e_priv *priv = arg1;
+       int prio_index = arg2;
+       struct mlx5_core_dev *mdev = priv->mdev;
+       u8 max_bw_unit[IEEE_8021QAZ_MAX_TCS];
+       u8 max_bw_value[IEEE_8021QAZ_MAX_TCS];
+       int i, err;
+       u64 bw_val;
+       u64 result = priv->params_ethtool.max_bw_value[prio_index];
+       const u64 upper_limit_mbps = 255 * MLX5E_100MB;
+       const u64 upper_limit_gbps = 255 * MLX5E_1GB;
+
+       PRIV_LOCK(priv);
+       err = sysctl_handle_64(oidp, &result, 0, req);
+       if (err || !req->newptr ||
+           result == priv->params_ethtool.max_bw_value[prio_index])
+               goto done;
+
+       if (result % MLX5E_100MB) {
+               err = ERANGE;
+               goto done;
+       }
+
+       memset(max_bw_value, 0, sizeof(max_bw_value));
+       memset(max_bw_unit, 0, sizeof(max_bw_unit));
+
+       for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+               bw_val = (i == prio_index) ? result : 
priv->params_ethtool.max_bw_value[i];
+
+               if (!bw_val) {
+                       max_bw_unit[i] = MLX5_BW_NO_LIMIT;
+               } else if (bw_val > upper_limit_gbps) {
+                       result = 0;
+                       max_bw_unit[i] = MLX5_BW_NO_LIMIT;
+               } else if (bw_val <= upper_limit_mbps) {
+                       max_bw_value[i] = howmany(bw_val, MLX5E_100MB);
+                       max_bw_unit[i]  = MLX5_100_MBPS_UNIT;
+               } else {
+                       max_bw_value[i] = howmany(bw_val, MLX5E_1GB);
+                       max_bw_unit[i]  = MLX5_GBPS_UNIT;
+               }
+       }
+
+       err = -mlx5_modify_port_tc_rate_limit(mdev, max_bw_value, max_bw_unit);
+       if (err)
+               goto done;
+
+       priv->params_ethtool.max_bw_value[prio_index] = result;
+done:
+       PRIV_UNLOCK(priv);
+       return (err);
+}
+
 #define        MLX5_PARAM_OFFSET(n)                            \
     __offsetof(struct mlx5e_priv, params_ethtool.n)
 
@@ -734,9 +825,11 @@ mlx5e_create_diagnostics(struct mlx5e_priv *priv)
 void
 mlx5e_create_ethtool(struct mlx5e_priv *priv)
 {
-       struct sysctl_oid *node;
+       struct mlx5_core_dev *mdev = priv->mdev;
+       struct sysctl_oid *node, *qos_node;
        const char *pnameunit;
        unsigned x;
+       int i;
 
        /* set some defaults */
        priv->params_ethtool.tx_queue_size_max = 1 << 
MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE;
@@ -830,4 +923,25 @@ mlx5e_create_ethtool(struct mlx5e_priv *priv)
 
        /* Diagnostics support */
        mlx5e_create_diagnostics(priv);
+
+       /* create qos node */
+       qos_node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
+           SYSCTL_CHILDREN(node), OID_AUTO,
+           "qos", CTLFLAG_RW, NULL, "Quality Of Service configuration");
+       if (node == NULL)
+               return;
+
+       /* Prioriry rate limit support */
+       if (mlx5e_getmaxrate(priv))
+               return;
+
+       for (i = 0; i <= mlx5_max_tc(mdev); i++) {
+               char name[32];
+               snprintf(name, sizeof(name), "tc_%d_max_rate", i);
+               SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(qos_node),
+                               OID_AUTO, name, CTLTYPE_U64 | CTLFLAG_RW | 
CTLFLAG_MPSAFE,
+                               priv, i, mlx5e_tc_maxrate_handler, "QU",
+                               "Max rate for priority, specified in kilobits, 
where kilo=1000, \
+                               max_rate must be divisible by 100000");
+       }
 }

Modified: head/sys/dev/mlx5/mlx5_ifc.h
==============================================================================
--- head/sys/dev/mlx5/mlx5_ifc.h        Wed Mar  7 15:03:11 2018        
(r330606)
+++ head/sys/dev/mlx5/mlx5_ifc.h        Wed Mar  7 15:17:36 2018        
(r330607)
@@ -9096,6 +9096,15 @@ struct mlx5_ifc_ets_global_config_reg_bits {
        u8         max_bw_value[0x8];
 };
 
+struct mlx5_ifc_qetc_reg_bits {
+       u8                                         reserved_at_0[0x8];
+       u8                                         port_number[0x8];
+       u8                                         reserved_at_10[0x30];
+
+       struct mlx5_ifc_ets_tcn_config_reg_bits    tc_configuration[0x8];
+       struct mlx5_ifc_ets_global_config_reg_bits global_configuration;
+};
+
 struct mlx5_ifc_nodnic_mac_filters_bits {
        struct mlx5_ifc_mac_address_layout_bits mac_filter0;
 

Modified: head/sys/dev/mlx5/port.h
==============================================================================
--- head/sys/dev/mlx5/port.h    Wed Mar  7 15:03:11 2018        (r330606)
+++ head/sys/dev/mlx5/port.h    Wed Mar  7 15:17:36 2018        (r330607)
@@ -144,4 +144,11 @@ int mlx5_query_eeprom(struct mlx5_core_dev *dev, int i
                      int device_addr, int size, int module_num, u32 *data,
                      int *size_read);
 
+int mlx5_max_tc(struct mlx5_core_dev *mdev);
+int mlx5_query_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+                                  u8 *max_bw_value,
+                                  u8 *max_bw_units);
+int mlx5_modify_port_tc_rate_limit(struct mlx5_core_dev *mdev,
+                                  const u8 *max_bw_value,
+                                  const u8 *max_bw_units);
 #endif /* __MLX5_PORT_H__ */
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"

Reply via email to