The branch main has been updated by kib: URL: https://cgit.FreeBSD.org/src/commit/?id=e23731db48ef9c6568d4768b1f87d48514339faa
commit e23731db48ef9c6568d4768b1f87d48514339faa Author: Konstantin Belousov <[email protected]> AuthorDate: 2024-07-22 11:31:35 +0000 Commit: Konstantin Belousov <[email protected]> CommitDate: 2024-07-30 15:00:04 +0000 mlx5en: add IPSEC_OFFLOAD support Right now, only IPv4 transport mode, with aes-gcm ESP, is supported. Driver also cooperates with NAT-T, and obeys socket policies, which makes IKEd like StrongSwan working. Sponsored by: NVIDIA networking --- sys/conf/files | 20 +- sys/dev/mlx5/cq.h | 6 + sys/dev/mlx5/crypto.h | 36 + sys/dev/mlx5/device.h | 125 +- sys/dev/mlx5/doorbell.h | 6 +- sys/dev/mlx5/driver.h | 7 + sys/dev/mlx5/fs.h | 226 +- sys/dev/mlx5/mlx5_accel/ipsec.h | 346 ++- sys/dev/mlx5/mlx5_accel/mlx5_ipsec.c | 747 ++++++ sys/dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c | 2047 +++++++++++++++ sys/dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c | 486 ++++ sys/dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c | 76 + sys/dev/mlx5/mlx5_core/eswitch.h | 8 +- sys/dev/mlx5/mlx5_core/fs_chains.h | 71 + sys/dev/mlx5/mlx5_core/fs_cmd.h | 120 + sys/dev/mlx5/mlx5_core/fs_core.h | 490 ++-- sys/dev/mlx5/mlx5_core/fs_ft_pool.h | 23 + sys/dev/mlx5/mlx5_core/fs_tcp.h | 6 +- sys/dev/mlx5/mlx5_core/mlx5_cmd.c | 3 + sys/dev/mlx5/mlx5_core/mlx5_core.h | 10 + sys/dev/mlx5/mlx5_core/mlx5_crypto.c | 94 + sys/dev/mlx5/mlx5_core/mlx5_eq.c | 11 + sys/dev/mlx5/mlx5_core/mlx5_eswitch.c | 140 +- sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.c | 102 - sys/dev/mlx5/mlx5_core/mlx5_fc_cmd.h | 54 - sys/dev/mlx5/mlx5_core/mlx5_fs_chains.c | 664 +++++ sys/dev/mlx5/mlx5_core/mlx5_fs_cmd.c | 1239 ++++++--- sys/dev/mlx5/mlx5_core/mlx5_fs_core.c | 3514 ++++++++++++++++++++++++++ sys/dev/mlx5/mlx5_core/mlx5_fs_counters.c | 2 +- sys/dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c | 85 + sys/dev/mlx5/mlx5_core/mlx5_fs_tcp.c | 50 +- sys/dev/mlx5/mlx5_core/mlx5_fs_tree.c | 2874 --------------------- sys/dev/mlx5/mlx5_core/mlx5_fw.c | 6 + sys/dev/mlx5/mlx5_core/mlx5_main.c | 20 +- sys/dev/mlx5/mlx5_core/mlx5_tls.c | 60 - sys/dev/mlx5/mlx5_core/wq.h | 17 + sys/dev/mlx5/mlx5_en/en.h | 22 +- sys/dev/mlx5/mlx5_en/en_hw_tls_rx.h | 2 +- sys/dev/mlx5/mlx5_en/mlx5_en_flow_table.c | 350 ++- sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c | 2 + sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls_rx.c | 4 +- sys/dev/mlx5/mlx5_en/mlx5_en_main.c | 38 +- sys/dev/mlx5/mlx5_en/mlx5_en_rx.c | 10 +- sys/dev/mlx5/mlx5_en/mlx5_en_tx.c | 3 + sys/dev/mlx5/mlx5_ib/mlx5_ib.h | 2 +- sys/dev/mlx5/mlx5_ib/mlx5_ib_cq.c | 8 +- sys/dev/mlx5/mlx5_ib/mlx5_ib_main.c | 37 +- sys/dev/mlx5/mlx5_ifc.h | 719 ++++-- sys/dev/mlx5/mlx5_lib/aso.h | 92 + sys/dev/mlx5/mlx5_lib/mlx5_aso.c | 428 ++++ sys/dev/mlx5/qp.h | 13 +- sys/dev/mlx5/tls.h | 3 - sys/modules/mlx5/Makefile | 18 +- 53 files changed, 11214 insertions(+), 4328 deletions(-) diff --git a/sys/conf/files b/sys/conf/files index 5250b65135e6..6e69c128ee65 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -4952,6 +4952,8 @@ dev/mlx5/mlx5_core/mlx5_alloc.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_cmd.c optional mlx5 pci \ compile-with "${OFED_C}" +dev/mlx5/mlx5_core/mlx5_crypto.c optional mlx5 pci \ + compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_cq.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_diag_cnt.c optional mlx5 pci \ @@ -4962,15 +4964,17 @@ dev/mlx5/mlx5_core/mlx5_eq.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_eswitch.c optional mlx5 pci \ compile-with "${OFED_C}" -dev/mlx5/mlx5_core/mlx5_fc_cmd.c optional mlx5 pci \ +dev/mlx5/mlx5_core/mlx5_fs_chains.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_fs_cmd.c optional mlx5 pci \ compile-with "${OFED_C}" +dev/mlx5/mlx5_core/mlx5_fs_core.c optional mlx5 pci \ + compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_fs_counters.c optional mlx5 pci \ compile-with "${OFED_C}" -dev/mlx5/mlx5_core/mlx5_fs_tcp.c optional mlx5 pci \ +dev/mlx5/mlx5_core/mlx5_fs_ft_pool.c optional mlx5 pci \ compile-with "${OFED_C}" -dev/mlx5/mlx5_core/mlx5_fs_tree.c optional mlx5 pci \ +dev/mlx5/mlx5_core/mlx5_fs_tcp.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_fw.c optional mlx5 pci \ compile-with "${OFED_C}" @@ -5012,8 +5016,18 @@ dev/mlx5/mlx5_core/mlx5_vsc.c optional mlx5 pci \ compile-with "${OFED_C}" dev/mlx5/mlx5_core/mlx5_wq.c optional mlx5 pci \ compile-with "${OFED_C}" +dev/mlx5/mlx5_lib/mlx5_aso.c optional mlx5 pci \ + compile-with "${OFED_C}" dev/mlx5/mlx5_lib/mlx5_gid.c optional mlx5 pci \ compile-with "${OFED_C}" +dev/mlx5/mlx5_accel/mlx5_ipsec_fs.c optional mlx5 pci \ + compile-with "${OFED_C}" +dev/mlx5/mlx5_accel/mlx5_ipsec_offload.c optional mlx5 pci \ + compile-with "${OFED_C}" +dev/mlx5/mlx5_accel/mlx5_ipsec.c optional mlx5 pci \ + compile-with "${OFED_C}" +dev/mlx5/mlx5_accel/mlx5_ipsec_rxtx.c optional mlx5 pci \ + compile-with "${OFED_C}" dev/mlx5/mlx5_en/mlx5_en_dim.c optional mlx5en pci inet inet6 \ compile-with "${OFED_C}" diff --git a/sys/dev/mlx5/cq.h b/sys/dev/mlx5/cq.h index 02de7f66e260..cc551e65dead 100644 --- a/sys/dev/mlx5/cq.h +++ b/sys/dev/mlx5/cq.h @@ -111,6 +111,12 @@ struct mlx5_cq_modify_params { } params; }; +enum { + CQE_STRIDE_64 = 0, + CQE_STRIDE_128 = 1, + CQE_STRIDE_128_PAD = 2, +}; + static inline int cqe_sz_to_mlx_sz(u8 size) { return size == 64 ? CQE_SIZE_64 : CQE_SIZE_128; diff --git a/sys/dev/mlx5/crypto.h b/sys/dev/mlx5/crypto.h new file mode 100644 index 000000000000..3b2c4c218ef2 --- /dev/null +++ b/sys/dev/mlx5/crypto.h @@ -0,0 +1,36 @@ +/*- + * Copyright (c) 2023, NVIDIA Technologies. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#ifndef __MLX5_CRYPTO_H__ +#define __MLX5_CRYPTO_H__ + +struct mlx5_core_dev; + +int mlx5_encryption_key_create(struct mlx5_core_dev *, u32 pdn, u32 key_type, + const void *p_key, u32 key_len, u32 *p_obj_id); +int mlx5_encryption_key_destroy(struct mlx5_core_dev *mdev, u32 oid); + +#endif /* __MLX5_CRYPTO_H__ */ diff --git a/sys/dev/mlx5/device.h b/sys/dev/mlx5/device.h index 4b7cf8686794..50995d4f70a7 100644 --- a/sys/dev/mlx5/device.h +++ b/sys/dev/mlx5/device.h @@ -148,12 +148,14 @@ __mlx5_mask16(typ, fld)) tmp; \ }) -#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 8 -#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 8 +#define MLX5_RDMA_RX_NUM_COUNTERS_PRIOS 2 +#define MLX5_RDMA_TX_NUM_COUNTERS_PRIOS 1 +#define MLX5_BY_PASS_NUM_REGULAR_PRIOS 16 +#define MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS 16 #define MLX5_BY_PASS_NUM_MULTICAST_PRIOS 1 #define MLX5_BY_PASS_NUM_PRIOS (MLX5_BY_PASS_NUM_REGULAR_PRIOS +\ - MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\ - MLX5_BY_PASS_NUM_MULTICAST_PRIOS) + MLX5_BY_PASS_NUM_DONT_TRAP_PRIOS +\ + MLX5_BY_PASS_NUM_MULTICAST_PRIOS) /* insert a value to a struct */ #define MLX5_VSC_SET(typ, p, fld, v) do { \ @@ -391,6 +393,8 @@ enum { MLX5_OPCODE_UMR = 0x25, MLX5_OPCODE_QOS_REMAP = 0x2a, + MLX5_OPCODE_ACCESS_ASO = 0x2d, + MLX5_OPCODE_SIGNATURE_CANCELED = (1 << 15), }; @@ -567,6 +571,11 @@ struct mlx5_eqe_vport_change { __be32 rsvd1[6]; }; +struct mlx5_eqe_obj_change { + u8 rsvd0[2]; + __be16 obj_type; + __be32 obj_id; +}; #define PORT_MODULE_EVENT_MODULE_STATUS_MASK 0xF #define PORT_MODULE_EVENT_ERROR_TYPE_MASK 0xF @@ -638,6 +647,7 @@ union ev_data { struct mlx5_eqe_dct dct; struct mlx5_eqe_temp_warning temp_warning; struct mlx5_eqe_xrq_err xrq_err; + struct mlx5_eqe_obj_change obj_change; } __packed; struct mlx5_eqe { @@ -703,7 +713,12 @@ struct mlx5_cqe64 { u8 l4_hdr_type_etc; __be16 vlan_info; __be32 srqn; /* [31:24]: lro_num_seg, [23:0]: srqn */ - __be32 imm_inval_pkey; + union { + __be32 immediate; + __be32 inval_rkey; + __be32 pkey; + __be32 ft_metadata; + }; u8 rsvd40[4]; __be32 byte_cnt; __be64 timestamp; @@ -919,6 +934,7 @@ enum { MLX5_MATCH_OUTER_HEADERS = 1 << 0, MLX5_MATCH_MISC_PARAMETERS = 1 << 1, MLX5_MATCH_INNER_HEADERS = 1 << 2, + MLX5_MATCH_MISC_PARAMETERS_2 = 1 << 3, }; @@ -988,12 +1004,19 @@ enum mlx5_cap_type { MLX5_CAP_VECTOR_CALC, MLX5_CAP_QOS, MLX5_CAP_DEBUG, - MLX5_CAP_NVME, - MLX5_CAP_DMC, - MLX5_CAP_DEC, + MLX5_CAP_RESERVED_14, + MLX5_CAP_DEV_MEM, + MLX5_CAP_RESERVED_16, MLX5_CAP_TLS, + MLX5_CAP_VDPA_EMULATION = 0x13, MLX5_CAP_DEV_EVENT = 0x14, + MLX5_CAP_IPSEC, + MLX5_CAP_CRYPTO = 0x1a, + MLX5_CAP_DEV_SHAMPO = 0x1d, + MLX5_CAP_MACSEC = 0x1f, MLX5_CAP_GENERAL_2 = 0x20, + MLX5_CAP_PORT_SELECTION = 0x25, + MLX5_CAP_ADV_VIRTUALIZATION = 0x26, /* NUM OF CAP Types */ MLX5_CAP_NUM }; @@ -1058,6 +1081,9 @@ enum mlx5_mcam_feature_groups { #define MLX5_CAP_FLOWTABLE(mdev, cap) \ MLX5_GET(flow_table_nic_cap, mdev->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap) +#define MLX5_CAP64_FLOWTABLE(mdev, cap) \ + MLX5_GET64(flow_table_nic_cap, (mdev)->hca_caps_cur[MLX5_CAP_FLOW_TABLE], cap) + #define MLX5_CAP_FLOWTABLE_MAX(mdev, cap) \ MLX5_GET(flow_table_nic_cap, mdev->hca_caps_max[MLX5_CAP_FLOW_TABLE], cap) @@ -1067,6 +1093,54 @@ enum mlx5_mcam_feature_groups { #define MLX5_CAP_FLOWTABLE_NIC_RX_MAX(mdev, cap) \ MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive.cap) +#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap) + +#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap) + +#define MLX5_CAP_FLOWTABLE_SNIFFER_RX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_sniffer.cap) + +#define MLX5_CAP_FLOWTABLE_SNIFFER_RX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_sniffer.cap) + +#define MLX5_CAP_FLOWTABLE_SNIFFER_TX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_sniffer.cap) + +#define MLX5_CAP_FLOWTABLE_SNIFFER_TX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_sniffer.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit.cap) + +#define MLX5_CAP_FLOWTABLE_NIC_TX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_RX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_RX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_receive_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_TX(mdev, cap) \ + MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_transmit_rdma.cap) + +#define MLX5_CAP_FLOWTABLE_RDMA_TX_MAX(mdev, cap) \ + MLX5_CAP_FLOWTABLE_MAX(mdev, flow_table_properties_nic_transmit_rdma.cap) + #define MLX5_CAP_ESW_FLOWTABLE(mdev, cap) \ MLX5_GET(flow_table_eswitch_cap, \ mdev->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap) @@ -1093,14 +1167,46 @@ enum mlx5_mcam_feature_groups { #define MLX5_CAP_ESW_INGRESS_ACL_MAX(mdev, cap) \ MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, flow_table_properties_esw_acl_ingress.cap) +#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2(mdev, cap) \ + MLX5_CAP_ESW_FLOWTABLE(mdev, ft_field_support_2_esw_fdb.cap) + +#define MLX5_CAP_ESW_FT_FIELD_SUPPORT_2_MAX(mdev, cap) \ + MLX5_CAP_ESW_FLOWTABLE_MAX(mdev, ft_field_support_2_esw_fdb.cap) + #define MLX5_CAP_ESW(mdev, cap) \ MLX5_GET(e_switch_cap, \ mdev->hca_caps_cur[MLX5_CAP_ESWITCH], cap) +#define MLX5_CAP64_ESW_FLOWTABLE(mdev, cap) \ + MLX5_GET64(flow_table_eswitch_cap, \ + (mdev)->hca_caps_cur[MLX5_CAP_ESWITCH_FLOW_TABLE], cap) + #define MLX5_CAP_ESW_MAX(mdev, cap) \ MLX5_GET(e_switch_cap, \ mdev->hca_caps_max[MLX5_CAP_ESWITCH], cap) +#define MLX5_CAP_PORT_SELECTION(mdev, cap) \ + MLX5_GET(port_selection_cap, \ + mdev->hca_caps_cur[MLX5_CAP_PORT_SELECTION], cap) + +#define MLX5_CAP_PORT_SELECTION_MAX(mdev, cap) \ + MLX5_GET(port_selection_cap, \ + mdev->hca_caps_max[MLX5_CAP_PORT_SELECTION], cap) + +#define MLX5_CAP_ADV_VIRTUALIZATION(mdev, cap) \ + MLX5_GET(adv_virtualization_cap, \ + mdev->hca_caps_cur[MLX5_CAP_ADV_VIRTUALIZATION], cap) + +#define MLX5_CAP_ADV_VIRTUALIZATION_MAX(mdev, cap) \ + MLX5_GET(adv_virtualization_cap, \ + mdev->hca_caps_max[MLX5_CAP_ADV_VIRTUALIZATION], cap) + +#define MLX5_CAP_FLOWTABLE_PORT_SELECTION(mdev, cap) \ + MLX5_CAP_PORT_SELECTION(mdev, flow_table_properties_port_selection.cap) + +#define MLX5_CAP_FLOWTABLE_PORT_SELECTION_MAX(mdev, cap) \ + MLX5_CAP_PORT_SELECTION_MAX(mdev, flow_table_properties_port_selection.cap) + #define MLX5_CAP_ODP(mdev, cap)\ MLX5_GET(odp_cap, mdev->hca_caps_cur[MLX5_CAP_ODP], cap) @@ -1169,6 +1275,9 @@ enum mlx5_mcam_feature_groups { #define MLX5_CAP_DEV_EVENT(mdev, cap)\ MLX5_ADDR_OF(device_event_cap, (mdev)->hca_caps_cur[MLX5_CAP_DEV_EVENT], cap) +#define MLX5_CAP_IPSEC(mdev, cap) \ + MLX5_GET(ipsec_cap, (mdev)->hca_caps_cur[MLX5_CAP_IPSEC], cap) + enum { MLX5_CMD_STAT_OK = 0x0, MLX5_CMD_STAT_INT_ERR = 0x1, diff --git a/sys/dev/mlx5/doorbell.h b/sys/dev/mlx5/doorbell.h index 11ce9ae6649f..4a32e74623a0 100644 --- a/sys/dev/mlx5/doorbell.h +++ b/sys/dev/mlx5/doorbell.h @@ -61,10 +61,12 @@ static inline void mlx5_write64(__be32 val[2], void __iomem *dest, { unsigned long flags; - spin_lock_irqsave(doorbell_lock, flags); + if (doorbell_lock) + spin_lock_irqsave(doorbell_lock, flags); __raw_writel((__force u32) val[0], dest); __raw_writel((__force u32) val[1], dest + 4); - spin_unlock_irqrestore(doorbell_lock, flags); + if (doorbell_lock) + spin_unlock_irqrestore(doorbell_lock, flags); } #endif diff --git a/sys/dev/mlx5/driver.h b/sys/dev/mlx5/driver.h index 6ebe4171a503..db1c9f0bb46c 100644 --- a/sys/dev/mlx5/driver.h +++ b/sys/dev/mlx5/driver.h @@ -634,11 +634,13 @@ struct mlx5_priv { #endif struct mlx5_pme_stats pme_stats; + struct mlx5_flow_steering *steering; struct mlx5_eswitch *eswitch; struct mlx5_bfreg_data bfregs; struct mlx5_uars_page *uar; struct mlx5_fc_stats fc_stats; + struct mlx5_ft_pool *ft_pool; }; enum mlx5_device_state { @@ -728,6 +730,10 @@ struct mlx5_core_dev { struct mlx5_flow_root_namespace *esw_ingress_root_ns; struct mlx5_flow_root_namespace *sniffer_rx_root_ns; struct mlx5_flow_root_namespace *sniffer_tx_root_ns; + struct mlx5_flow_root_namespace *nic_tx_root_ns; + struct mlx5_flow_root_namespace *rdma_tx_root_ns; + struct mlx5_flow_root_namespace *rdma_rx_root_ns; + u32 num_q_counter_allocated[MLX5_INTERFACE_NUMBER]; struct mlx5_crspace_regmap *dump_rege; uint32_t *dump_data; @@ -756,6 +762,7 @@ struct mlx5_core_dev { #ifdef CONFIG_MLX5_FPGA struct mlx5_fpga_device *fpga; #endif + struct xarray ipsec_sadb; }; enum { diff --git a/sys/dev/mlx5/fs.h b/sys/dev/mlx5/fs.h index 8107726bc76a..ee0d28b6758a 100644 --- a/sys/dev/mlx5/fs.h +++ b/sys/dev/mlx5/fs.h @@ -33,8 +33,33 @@ #include <dev/mlx5/device.h> #include <dev/mlx5/driver.h> +enum mlx5_flow_destination_type { + MLX5_FLOW_DESTINATION_TYPE_NONE, + MLX5_FLOW_DESTINATION_TYPE_VPORT, + MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE, + MLX5_FLOW_DESTINATION_TYPE_TIR, + MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER, + MLX5_FLOW_DESTINATION_TYPE_UPLINK, + MLX5_FLOW_DESTINATION_TYPE_PORT, + MLX5_FLOW_DESTINATION_TYPE_COUNTER, + MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE_NUM, + MLX5_FLOW_DESTINATION_TYPE_RANGE, + MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE, +}; + +enum { + MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO = 1 << 16, + MLX5_FLOW_CONTEXT_ACTION_ENCRYPT = 1 << 17, + MLX5_FLOW_CONTEXT_ACTION_DECRYPT = 1 << 18, + MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_NS = 1 << 19, +}; + enum { - MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO = 1 << 16, + MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT = BIT(0), + MLX5_FLOW_TABLE_TUNNEL_EN_DECAP = BIT(1), + MLX5_FLOW_TABLE_TERMINATION = BIT(2), + MLX5_FLOW_TABLE_UNMANAGED = BIT(3), + MLX5_FLOW_TABLE_OTHER_VPORT = BIT(4), }; /*Flow tag*/ @@ -61,54 +86,149 @@ enum { enum mlx5_flow_namespace_type { MLX5_FLOW_NAMESPACE_BYPASS, + MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, + MLX5_FLOW_NAMESPACE_LAG, MLX5_FLOW_NAMESPACE_OFFLOADS, + MLX5_FLOW_NAMESPACE_ETHTOOL, MLX5_FLOW_NAMESPACE_KERNEL, MLX5_FLOW_NAMESPACE_LEFTOVERS, - MLX5_FLOW_NAMESPACE_SNIFFER_RX, - MLX5_FLOW_NAMESPACE_SNIFFER_TX, + MLX5_FLOW_NAMESPACE_ANCHOR, + MLX5_FLOW_NAMESPACE_FDB_BYPASS, MLX5_FLOW_NAMESPACE_FDB, MLX5_FLOW_NAMESPACE_ESW_EGRESS, MLX5_FLOW_NAMESPACE_ESW_INGRESS, + MLX5_FLOW_NAMESPACE_SNIFFER_RX, + MLX5_FLOW_NAMESPACE_SNIFFER_TX, + MLX5_FLOW_NAMESPACE_EGRESS, + MLX5_FLOW_NAMESPACE_EGRESS_IPSEC, + MLX5_FLOW_NAMESPACE_EGRESS_MACSEC, + MLX5_FLOW_NAMESPACE_RDMA_RX, + MLX5_FLOW_NAMESPACE_RDMA_RX_KERNEL, + MLX5_FLOW_NAMESPACE_RDMA_TX, + MLX5_FLOW_NAMESPACE_PORT_SEL, + MLX5_FLOW_NAMESPACE_RDMA_RX_COUNTERS, + MLX5_FLOW_NAMESPACE_RDMA_TX_COUNTERS, + MLX5_FLOW_NAMESPACE_RDMA_RX_IPSEC, + MLX5_FLOW_NAMESPACE_RDMA_TX_IPSEC, +}; + +enum { + FDB_BYPASS_PATH, + FDB_TC_OFFLOAD, + FDB_FT_OFFLOAD, + FDB_TC_MISS, + FDB_BR_OFFLOAD, + FDB_SLOW_PATH, + FDB_PER_VPORT, }; struct mlx5_flow_table; struct mlx5_flow_group; struct mlx5_flow_rule; struct mlx5_flow_namespace; +struct mlx5_flow_handle; + +enum { + FLOW_CONTEXT_HAS_TAG = BIT(0), +}; + +struct mlx5_flow_context { + u32 flags; + u32 flow_tag; + u32 flow_source; +}; struct mlx5_flow_spec { u8 match_criteria_enable; u32 match_criteria[MLX5_ST_SZ_DW(fte_match_param)]; u32 match_value[MLX5_ST_SZ_DW(fte_match_param)]; + struct mlx5_flow_context flow_context; +}; + +enum { + MLX5_FLOW_DEST_VPORT_VHCA_ID = BIT(0), + MLX5_FLOW_DEST_VPORT_REFORMAT_ID = BIT(1), +}; + +enum mlx5_flow_dest_range_field { + MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN = 0, }; struct mlx5_flow_destination { - u32 type; + enum mlx5_flow_destination_type type; union { - u32 tir_num; - struct mlx5_flow_table *ft; - u32 vport_num; + u32 tir_num; + u32 ft_num; + struct mlx5_flow_table *ft; + u32 counter_id; + struct { + u16 num; + u16 vhca_id; + struct mlx5_pkt_reformat *pkt_reformat; + u8 flags; + } vport; + struct { + struct mlx5_flow_table *hit_ft; + struct mlx5_flow_table *miss_ft; + enum mlx5_flow_dest_range_field field; + u32 min; + u32 max; + } range; + u32 sampler_id; }; }; -enum mlx5_flow_act_actions { - MLX5_FLOW_ACT_ACTIONS_FLOW_TAG = 1 << 0, - MLX5_FLOW_ACT_ACTIONS_MODIFY_HDR = 1 << 1, - MLX5_FLOW_ACT_ACTIONS_PACKET_REFORMAT = 1 << 2, - MLX5_FLOW_ACT_ACTIONS_COUNT = 1 << 3, +struct mlx5_exe_aso { + u32 object_id; + u8 type; + u8 return_reg_id; + union { + u32 ctrl_data; + struct { + u8 meter_idx; + u8 init_color; + } flow_meter; + }; +}; + +enum { + FLOW_ACT_NO_APPEND = BIT(0), + FLOW_ACT_IGNORE_FLOW_LEVEL = BIT(1), +}; + +struct mlx5_fs_vlan { + u16 ethtype; + u16 vid; + u8 prio; +}; + +#define MLX5_FS_VLAN_DEPTH 2 + +enum mlx5_flow_act_crypto_type { + MLX5_FLOW_ACT_CRYPTO_TYPE_IPSEC, +}; + +enum mlx5_flow_act_crypto_op { + MLX5_FLOW_ACT_CRYPTO_OP_ENCRYPT, + MLX5_FLOW_ACT_CRYPTO_OP_DECRYPT, }; -enum MLX5_FLOW_ACT_FLAGS { - MLX5_FLOW_ACT_NO_APPEND = 1 << 0, +struct mlx5_flow_act_crypto_params { + u32 obj_id; + u8 type; /* see enum mlx5_flow_act_crypto_type */ + u8 op; /* see enum mlx5_flow_act_crypto_op */ }; struct mlx5_flow_act { - u32 actions; /* See enum mlx5_flow_act_actions */ - u32 flags; - u32 flow_tag; - struct mlx5_modify_hdr *modify_hdr; + u32 action; + struct mlx5_modify_hdr *modify_hdr; struct mlx5_pkt_reformat *pkt_reformat; - struct mlx5_fc *counter; + struct mlx5_flow_act_crypto_params crypto; + u32 flags; + struct mlx5_fs_vlan vlan[MLX5_FS_VLAN_DEPTH]; + struct ib_counters *counters; + struct mlx5_flow_group *fg; + struct mlx5_exe_aso exe_aso; }; #define FT_NAME_STR_SZ 20 @@ -135,6 +255,28 @@ static inline bool outer_header_zero(u32 *match_criteria) size - 1); } +struct mlx5_flow_namespace * +mlx5_get_flow_vport_acl_namespace(struct mlx5_core_dev *dev, + enum mlx5_flow_namespace_type type, + int vport); + +struct mlx5_flow_table_attr { + int prio; + int max_fte; + u32 level; + u32 flags; + u16 uid; + struct mlx5_flow_table *next_ft; + + struct { + int max_num_groups; + int num_reserved_entries; + } autogroup; +}; + +struct mlx5_flow_namespace * +mlx5_get_fdb_sub_ns(struct mlx5_core_dev *dev, int n); + struct mlx5_flow_namespace * mlx5_get_flow_namespace(struct mlx5_core_dev *dev, enum mlx5_flow_namespace_type type); @@ -145,24 +287,19 @@ mlx5_get_flow_namespace(struct mlx5_core_dev *dev, */ struct mlx5_flow_table * mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns, - int prio, - const char *name, - int num_flow_table_entries, - int max_num_groups, - int num_reserved_entries); + struct mlx5_flow_table_attr *ft_attr); struct mlx5_flow_table * mlx5_create_vport_flow_table(struct mlx5_flow_namespace *ns, - u16 vport, - int prio, - const char *name, - int num_flow_table_entries); + struct mlx5_flow_table_attr *ft_attr, u16 vport); + +struct mlx5_flow_table *mlx5_create_lag_demux_flow_table( + struct mlx5_flow_namespace *ns, + int prio, u32 level); struct mlx5_flow_table * mlx5_create_flow_table(struct mlx5_flow_namespace *ns, - int prio, - const char *name, - int num_flow_table_entries); + struct mlx5_flow_table_attr *ft_attr); int mlx5_destroy_flow_table(struct mlx5_flow_table *ft); /* inbox should be set with the following values: @@ -175,18 +312,17 @@ struct mlx5_flow_group * mlx5_create_flow_group(struct mlx5_flow_table *ft, u32 *in); void mlx5_destroy_flow_group(struct mlx5_flow_group *fg); -/* Single destination per rule. - * Group ID is implied by the match criteria. - */ -struct mlx5_flow_rule * -mlx5_add_flow_rule(struct mlx5_flow_table *ft, - u8 match_criteria_enable, - u32 *match_criteria, - u32 *match_value, - u32 sw_action, - struct mlx5_flow_act *flow_act, - struct mlx5_flow_destination *dest); -void mlx5_del_flow_rule(struct mlx5_flow_rule **); +struct mlx5_flow_handle * +mlx5_add_flow_rules(struct mlx5_flow_table *ft, + const struct mlx5_flow_spec *spec, + struct mlx5_flow_act *flow_act, + struct mlx5_flow_destination *dest, + int num_dest); +void mlx5_del_flow_rules(struct mlx5_flow_handle **pp); + +int mlx5_modify_rule_destination(struct mlx5_flow_handle *handler, + struct mlx5_flow_destination *new_dest, + struct mlx5_flow_destination *old_dest); /*The following API is for sniffer*/ typedef int (*rule_event_fn)(struct mlx5_flow_rule *rule, @@ -292,4 +428,8 @@ int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter, u64 *packets, u64 *bytes); u32 mlx5_fc_id(struct mlx5_fc *counter); /******* End of Flow counters API ******/ + +u32 mlx5_flow_table_id(struct mlx5_flow_table *ft); +int mlx5_fs_add_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn); +int mlx5_fs_remove_rx_underlay_qpn(struct mlx5_core_dev *dev, u32 underlay_qpn); #endif diff --git a/sys/dev/mlx5/mlx5_accel/ipsec.h b/sys/dev/mlx5/mlx5_accel/ipsec.h index c020d41cd875..1658542fc9c6 100644 --- a/sys/dev/mlx5/mlx5_accel/ipsec.h +++ b/sys/dev/mlx5/mlx5_accel/ipsec.h @@ -1,137 +1,277 @@ /*- - * Copyright (c) 2017 Mellanox Technologies. All rights reserved. + * Copyright (c) 2023 NVIDIA corporation & affiliates. * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: + * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. */ #ifndef __MLX5_ACCEL_IPSEC_H__ #define __MLX5_ACCEL_IPSEC_H__ -#ifdef CONFIG_MLX5_ACCEL - +#include <sys/mbuf.h> #include <dev/mlx5/driver.h> +#include <dev/mlx5/qp.h> +#include <dev/mlx5/mlx5_core/mlx5_core.h> +#include <dev/mlx5/mlx5_en/en.h> +#include <dev/mlx5/mlx5_lib/aso.h> + +#define MLX5E_IPSEC_SADB_RX_BITS 10 +#define MLX5_IPSEC_METADATA_MARKER(ipsec_metadata) ((ipsec_metadata >> 31) & 0x1) + +struct mlx5e_priv; +struct mlx5e_tx_wqe; +struct mlx5e_ipsec_tx; +struct mlx5e_ipsec_rx; + +struct aes_gcm_keymat { + u64 seq_iv; + + u32 salt; + u32 icv_len; -enum { - MLX5_ACCEL_IPSEC_DEVICE = BIT(1), - MLX5_ACCEL_IPSEC_IPV6 = BIT(2), - MLX5_ACCEL_IPSEC_ESP = BIT(3), - MLX5_ACCEL_IPSEC_LSO = BIT(4), + u32 key_len; + u32 aes_key[256 / 32]; }; -#define MLX5_IPSEC_SADB_IP_AH BIT(7) -#define MLX5_IPSEC_SADB_IP_ESP BIT(6) -#define MLX5_IPSEC_SADB_SA_VALID BIT(5) -#define MLX5_IPSEC_SADB_SPI_EN BIT(4) -#define MLX5_IPSEC_SADB_DIR_SX BIT(3) -#define MLX5_IPSEC_SADB_IPV6 BIT(2) +struct mlx5e_ipsec_priv_bothdir { + struct mlx5e_ipsec_sa_entry *priv_in; + struct mlx5e_ipsec_sa_entry *priv_out; +}; + +struct mlx5e_ipsec_work { + struct work_struct work; + struct mlx5e_ipsec_sa_entry *sa_entry; + void *data; +}; -enum { - MLX5_IPSEC_CMD_ADD_SA = 0, - MLX5_IPSEC_CMD_DEL_SA = 1, +struct mlx5e_ipsec_dwork { + struct delayed_work dwork; + struct mlx5e_ipsec_sa_entry *sa_entry; + struct mlx5e_ipsec_priv_bothdir *pb; }; -enum mlx5_accel_ipsec_enc_mode { - MLX5_IPSEC_SADB_MODE_NONE = 0, - MLX5_IPSEC_SADB_MODE_AES_GCM_128_AUTH_128 = 1, - MLX5_IPSEC_SADB_MODE_AES_GCM_256_AUTH_128 = 3, +struct mlx5e_ipsec_aso { + u8 __aligned(64) ctx[MLX5_ST_SZ_BYTES(ipsec_aso)]; + dma_addr_t dma_addr; + struct mlx5_aso *aso; + /* Protect ASO WQ access, as it is global to whole IPsec */ + spinlock_t lock; }; -#define MLX5_IPSEC_DEV(mdev) (mlx5_accel_ipsec_device_caps(mdev) & \ - MLX5_ACCEL_IPSEC_DEVICE) +struct mlx5_replay_esn { + u32 replay_window; + u32 esn; + u32 esn_msb; + u8 overlap : 1; + u8 trigger : 1; +}; + +struct mlx5_accel_esp_xfrm_attrs { + u32 spi; + struct aes_gcm_keymat aes_gcm; -struct mlx5_accel_ipsec_sa { - __be32 cmd; - u8 key_enc[32]; - u8 key_auth[32]; - __be32 sip[4]; - __be32 dip[4]; union { - struct { - __be32 reserved; - u8 salt_iv[8]; - __be32 salt; - } __packed gcm; - struct { - u8 salt[16]; - } __packed cbc; - }; - __be32 spi; - __be32 sw_sa_handle; - __be16 tfclen; - u8 enc_mode; - u8 sip_masklen; - u8 dip_masklen; - u8 flags; - u8 reserved[2]; -} __packed; - -/** - * mlx5_accel_ipsec_sa_cmd_exec - Execute an IPSec SADB command - * @mdev: mlx5 device - * @cmd: command to execute - * May be called from atomic context. Returns context pointer, or error - * Caller must eventually call mlx5_accel_ipsec_sa_cmd_wait from non-atomic - * context, to cleanup the context pointer - */ -void *mlx5_accel_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev, - struct mlx5_accel_ipsec_sa *cmd); - -/** - * mlx5_accel_ipsec_sa_cmd_wait - Wait for command execution completion - * @context: Context pointer returned from call to mlx5_accel_ipsec_sa_cmd_exec - * Sleeps (killable) until command execution is complete. - * Returns the command result, or -EINTR if killed - */ -int mlx5_accel_ipsec_sa_cmd_wait(void *context); + __be32 a4; + __be32 a6[4]; + } saddr; + + union { + __be32 a4; + __be32 a6[4]; + } daddr; + + u8 dir : 2; + u8 encap : 1; + u8 drop : 1; + u8 family; + struct mlx5_replay_esn replay_esn; + u32 authsize; + u32 reqid; + u16 sport; + u16 dport; +}; + +enum mlx5_ipsec_cap { + MLX5_IPSEC_CAP_CRYPTO = 1 << 0, + MLX5_IPSEC_CAP_ESN = 1 << 1, + MLX5_IPSEC_CAP_PACKET_OFFLOAD = 1 << 2, + MLX5_IPSEC_CAP_ROCE = 1 << 3, + MLX5_IPSEC_CAP_PRIO = 1 << 4, + MLX5_IPSEC_CAP_TUNNEL = 1 << 5, + MLX5_IPSEC_CAP_ESPINUDP = 1 << 6, +}; + +struct mlx5e_ipsec { + struct mlx5_core_dev *mdev; *** 16872 LINES SKIPPED ***
