Author: hselasky
Date: Thu May 16 15:57:59 2019
New Revision: 347732
URL: https://svnweb.freebsd.org/changeset/base/347732
Log:
MFC r347268:
Add Fast teardown support to mlx5core.
Today mlx5 devices support two teardown modes:
1- Regular teardown
2- Force teardown
This change introduces the enhanced version of the "Force teardown" that
allows SW to perform teardown in a faster way without the need to reclaim
all the pages.
Fast teardown provides the following advantages:
1- Fix a FW race condition that could cause command timeout
2- Avoid moving to polling mode
3- Close the vport to prevent PCI ACK to be sent without been
scattered to memory
Linux commit:
fcd29ad17c6ff885dfae58f557e9323941e63ba2
Sponsored by: Mellanox Technologies
Modified:
stable/12/sys/dev/mlx5/mlx5_core/mlx5_core.h
stable/12/sys/dev/mlx5/mlx5_core/mlx5_fw.c
stable/12/sys/dev/mlx5/mlx5_core/mlx5_main.c
stable/12/sys/dev/mlx5/mlx5_ifc.h
Directory Properties:
stable/12/ (props changed)
Modified: stable/12/sys/dev/mlx5/mlx5_core/mlx5_core.h
==============================================================================
--- stable/12/sys/dev/mlx5/mlx5_core/mlx5_core.h Thu May 16 15:57:14
2019 (r347731)
+++ stable/12/sys/dev/mlx5/mlx5_core/mlx5_core.h Thu May 16 15:57:59
2019 (r347732)
@@ -79,6 +79,7 @@ int mlx5_query_qcam_reg(struct mlx5_core_dev *mdev, u3
int mlx5_cmd_init_hca(struct mlx5_core_dev *dev);
int mlx5_cmd_teardown_hca(struct mlx5_core_dev *dev);
int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *dev);
+int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev);
void mlx5_core_event(struct mlx5_core_dev *dev, enum mlx5_dev_event event,
unsigned long param);
void mlx5_enter_error_state(struct mlx5_core_dev *dev, bool force);
Modified: stable/12/sys/dev/mlx5/mlx5_core/mlx5_fw.c
==============================================================================
--- stable/12/sys/dev/mlx5/mlx5_core/mlx5_fw.c Thu May 16 15:57:14 2019
(r347731)
+++ stable/12/sys/dev/mlx5/mlx5_core/mlx5_fw.c Thu May 16 15:57:59 2019
(r347732)
@@ -249,12 +249,59 @@ int mlx5_cmd_force_teardown_hca(struct mlx5_core_dev *
if (ret)
return ret;
- force_state = MLX5_GET(teardown_hca_out, out, force_state);
+ force_state = MLX5_GET(teardown_hca_out, out, state);
if (force_state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) {
mlx5_core_err(dev, "teardown with force mode failed\n");
return -EIO;
}
+ return 0;
+}
+
+#define MLX5_FAST_TEARDOWN_WAIT_MS 3000
+int mlx5_cmd_fast_teardown_hca(struct mlx5_core_dev *dev)
+{
+ int end, delay_ms = MLX5_FAST_TEARDOWN_WAIT_MS;
+ u32 out[MLX5_ST_SZ_DW(teardown_hca_out)] = {};
+ u32 in[MLX5_ST_SZ_DW(teardown_hca_in)] = {};
+ int state;
+ int ret;
+
+ if (!MLX5_CAP_GEN(dev, fast_teardown)) {
+ mlx5_core_dbg(dev, "fast teardown is not supported in the
firmware\n");
+ return -EOPNOTSUPP;
+ }
+
+ MLX5_SET(teardown_hca_in, in, opcode, MLX5_CMD_OP_TEARDOWN_HCA);
+ MLX5_SET(teardown_hca_in, in, profile,
+ MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN);
+
+ ret = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
+ if (ret)
+ return ret;
+
+ state = MLX5_GET(teardown_hca_out, out, state);
+ if (state == MLX5_TEARDOWN_HCA_OUT_FORCE_STATE_FAIL) {
+ mlx5_core_warn(dev, "teardown with fast mode failed\n");
+ return -EIO;
+ }
+
+ mlx5_set_nic_state(dev, MLX5_NIC_IFC_DISABLED);
+
+ /* Loop until device state turns to disable */
+ end = jiffies + msecs_to_jiffies(delay_ms);
+ do {
+ if (mlx5_get_nic_state(dev) == MLX5_NIC_IFC_DISABLED)
+ break;
+
+ pause("W", 1);
+ } while (!time_after(jiffies, end));
+
+ if (mlx5_get_nic_state(dev) != MLX5_NIC_IFC_DISABLED) {
+ dev_err(&dev->pdev->dev, "NIC IFC still %d after %ums.\n",
+ mlx5_get_nic_state(dev), delay_ms);
+ return -EIO;
+ }
return 0;
}
Modified: stable/12/sys/dev/mlx5/mlx5_core/mlx5_main.c
==============================================================================
--- stable/12/sys/dev/mlx5/mlx5_core/mlx5_main.c Thu May 16 15:57:14
2019 (r347731)
+++ stable/12/sys/dev/mlx5/mlx5_core/mlx5_main.c Thu May 16 15:57:59
2019 (r347732)
@@ -1415,12 +1415,17 @@ static const struct pci_error_handlers mlx5_err_handle
static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
{
+ bool fast_teardown, force_teardown;
int err;
- if (!MLX5_CAP_GEN(dev, force_teardown)) {
- mlx5_core_dbg(dev, "force teardown is not supported in the
firmware\n");
+ fast_teardown = MLX5_CAP_GEN(dev, fast_teardown);
+ force_teardown = MLX5_CAP_GEN(dev, force_teardown);
+
+ mlx5_core_dbg(dev, "force teardown firmware support=%d\n",
force_teardown);
+ mlx5_core_dbg(dev, "fast teardown firmware support=%d\n",
fast_teardown);
+
+ if (!fast_teardown && !force_teardown)
return -EOPNOTSUPP;
- }
if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
mlx5_core_dbg(dev, "Device in internal error state, giving
up\n");
@@ -1433,14 +1438,19 @@ static int mlx5_try_fast_unload(struct mlx5_core_dev *
mlx5_drain_health_wq(dev);
mlx5_stop_health_poll(dev, false);
+ err = mlx5_cmd_fast_teardown_hca(dev);
+ if (!err)
+ goto done;
+
err = mlx5_cmd_force_teardown_hca(dev);
- if (err) {
- mlx5_core_dbg(dev, "Firmware couldn't do fast unload error:
%d\n", err);
- return err;
- }
+ if (!err)
+ goto done;
+ mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", err);
+ mlx5_start_health_poll(dev);
+ return err;
+done:
mlx5_enter_error_state(dev, true);
-
return 0;
}
Modified: stable/12/sys/dev/mlx5/mlx5_ifc.h
==============================================================================
--- stable/12/sys/dev/mlx5/mlx5_ifc.h Thu May 16 15:57:14 2019
(r347731)
+++ stable/12/sys/dev/mlx5/mlx5_ifc.h Thu May 16 15:57:59 2019
(r347732)
@@ -1054,7 +1054,8 @@ struct mlx5_ifc_cmd_hca_cap_bits {
u8 relaxed_ordering_write[1];
u8 reserved_6[0x1];
u8 log_max_mkey[0x6];
- u8 reserved_7[0xc];
+ u8 reserved_7[0xb];
+ u8 fast_teardown[0x1];
u8 log_max_eq[0x4];
u8 max_indirection[0x8];
@@ -3289,12 +3290,13 @@ struct mlx5_ifc_teardown_hca_out_bits {
u8 reserved_1[0x3f];
- u8 force_state[0x1];
+ u8 state[0x1];
};
enum {
MLX5_TEARDOWN_HCA_IN_PROFILE_GRACEFUL_CLOSE = 0x0,
MLX5_TEARDOWN_HCA_IN_PROFILE_FORCE_CLOSE = 0x1,
+ MLX5_TEARDOWN_HCA_IN_PROFILE_PREPARE_FAST_TEARDOWN = 0x2,
};
struct mlx5_ifc_teardown_hca_in_bits {
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
