The branch main has been updated by markj: URL: https://cgit.FreeBSD.org/src/commit/?id=682f135f5de39cfc24cfd529ea8a161e94c76c8e
commit 682f135f5de39cfc24cfd529ea8a161e94c76c8e Author: Hareshx Sankar Raj <[email protected]> AuthorDate: 2026-06-24 15:08:16 +0000 Commit: Mark Johnston <[email protected]> CommitDate: 2026-06-24 16:17:01 +0000 qat: driver updates to enhance qat infrastructure - Updated QAT infrastructure FW version/AE mask/num_banks fields to facilitate integration of future QAT products. - Exposed service as sym;asym instead of cy for gen4 - Enhanced cpaGetInstances() for accurate instance retrieval - Added 57-bit virtual address support to lac_lock_free_stack - Minor bug fixes and improvements Signed-off-by: Hareshx Sankar Raj <[email protected]> Reviewed by: markj MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D57746 --- sys/contrib/dev/qat/LICENSE | 11 ++-- sys/dev/qat/include/adf_gen4vf_hw_csr_data.h | 12 ++--- sys/dev/qat/include/common/adf_accel_devices.h | 34 ++++++------ sys/dev/qat/include/common/adf_common_drv.h | 4 +- sys/dev/qat/include/common/adf_gen2_hw_data.h | 25 ++++----- sys/dev/qat/include/common/adf_gen4_hw_data.h | 14 +++-- sys/dev/qat/include/common/adf_uio.h | 4 +- sys/dev/qat/include/common/adf_uio_control.h | 4 +- .../crypto/sym/include/lac_sym_cipher_defs.h | 8 ++- .../qat_api/common/crypto/sym/lac_sym_alg_chain.c | 41 +++++++++------ sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c | 58 ++++++++++++--------- .../qat/qat_api/common/ctrl/sal_get_instances.c | 22 +++++++- .../qat/qat_api/common/utils/lac_lock_free_stack.h | 60 ++++++++++++---------- sys/dev/qat/qat_api/device/dev_info.c | 19 +------ sys/dev/qat/qat_api/include/icp_sal_versions.h | 4 +- .../qat_kernel/src/lac_adf_interface_freebsd.c | 3 +- sys/dev/qat/qat_api/qat_kernel/src/lac_symbols.c | 6 ++- sys/dev/qat/qat_common/adf_accel_engine.c | 6 +-- sys/dev/qat/qat_common/adf_cfg_device.c | 7 +-- sys/dev/qat/qat_common/adf_ctl_drv.c | 13 +++-- sys/dev/qat/qat_common/adf_freebsd_admin.c | 35 ++++++------- .../qat/qat_common/adf_freebsd_cnvnr_ctrs_dbg.c | 6 +-- sys/dev/qat/qat_common/adf_freebsd_uio.c | 4 +- sys/dev/qat/qat_common/adf_freebsd_ver_dbg.c | 10 +--- sys/dev/qat/qat_common/adf_fw_counters.c | 4 +- sys/dev/qat/qat_common/adf_gen2_hw_data.c | 19 +++++-- sys/dev/qat/qat_common/adf_gen4_hw_data.c | 16 ++++-- sys/dev/qat/qat_common/adf_gen4_timer.c | 7 +-- sys/dev/qat/qat_common/adf_gen4vf_hw_csr_data.c | 9 ++-- sys/dev/qat/qat_common/adf_heartbeat.c | 4 +- sys/dev/qat/qat_common/adf_transport.c | 12 +++-- sys/dev/qat/qat_common/qat_hal.c | 8 +-- sys/dev/qat/qat_hw/qat_200xx/adf_200xx_hw_data.c | 18 +++---- sys/dev/qat/qat_hw/qat_200xx/adf_drv.c | 4 +- sys/dev/qat/qat_hw/qat_4xxx/adf_4xxx_hw_data.c | 50 +++++++++--------- sys/dev/qat/qat_hw/qat_4xxx/adf_drv.c | 4 +- sys/dev/qat/qat_hw/qat_4xxxvf/adf_4xxxvf_hw_data.c | 26 +++++----- sys/dev/qat/qat_hw/qat_c3xxx/adf_c3xxx_hw_data.c | 18 +++---- sys/dev/qat/qat_hw/qat_c3xxx/adf_drv.c | 4 +- sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_ae_config.c | 8 +-- sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_hw_data.c | 29 ++++++----- sys/dev/qat/qat_hw/qat_c4xxx/adf_c4xxx_res_part.c | 12 ++--- sys/dev/qat/qat_hw/qat_c4xxx/adf_drv.c | 4 +- sys/dev/qat/qat_hw/qat_c62x/adf_c62x_hw_data.c | 18 +++---- sys/dev/qat/qat_hw/qat_c62x/adf_drv.c | 4 +- .../qat/qat_hw/qat_dh895xcc/adf_dh895xcc_hw_data.c | 16 +++--- sys/dev/qat/qat_hw/qat_dh895xcc/adf_drv.c | 4 +- 47 files changed, 391 insertions(+), 317 deletions(-) diff --git a/sys/contrib/dev/qat/LICENSE b/sys/contrib/dev/qat/LICENSE index 2d9af4268f0f..7ff00ec63c24 100644 --- a/sys/contrib/dev/qat/LICENSE +++ b/sys/contrib/dev/qat/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2021 Intel Corporation +Copyright (c) 2026 Intel Corporation Redistribution. Redistribution and use in binary form, without modification, are permitted provided that the following conditions are @@ -17,12 +17,9 @@ Limited patent license. Intel Corporation grants a world-wide, royalty-free, non-exclusive license under patents it now or hereafter owns or controls to make, have made, use, import, offer to sell and sell ("Utilize") this software, but solely to the extent that any -such patent is necessary to Utilize the software alone, or in -combination with an operating system licensed under an approved Open -Source license as listed by the Open Source Initiative at -http://opensource.org/licenses. The patent license shall not apply to -any other combinations which include this software. No hardware per -se is licensed hereunder. +such patent is necessary to Utilize the software alone. The patent +license shall not apply to any other combinations which include this +software. No hardware per se is licensed hereunder. DISCLAIMER. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, diff --git a/sys/dev/qat/include/adf_gen4vf_hw_csr_data.h b/sys/dev/qat/include/adf_gen4vf_hw_csr_data.h index 5143b88907ba..83841e1b919c 100644 --- a/sys/dev/qat/include/adf_gen4vf_hw_csr_data.h +++ b/sys/dev/qat/include/adf_gen4vf_hw_csr_data.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef ADF_GEN4VF_HW_CSR_DATA_H_ #define ADF_GEN4VF_HW_CSR_DATA_H_ @@ -86,12 +86,12 @@ read_base_gen4vf(struct resource *csr_base_addr, u32 bank, u32 ring) return addr; } -#define WRITE_CSR_INT_SRCSEL_GEN4VF(csr_base_addr, bank) \ +#define WRITE_CSR_INT_SRCSEL_GEN4VF(csr_base_addr, bank, idx, value) \ ADF_CSR_WR((csr_base_addr), \ - ADF_RING_CSR_ADDR_OFFSET_GEN4VF + \ - ADF_RING_BUNDLE_SIZE_GEN4 * (bank) + \ - ADF_RING_CSR_INT_SRCSEL, \ - ADF_BANK_INT_SRC_SEL_MASK_GEN4) + (ADF_RING_BUNDLE_SIZE_GEN4 * (bank)) + \ + (ADF_RING_CSR_INT_SRCSEL + \ + ((idx)*ADF_RING_CSR_NEXT_INT_SRCSEL)), \ + (value)) #define READ_CSR_RING_BASE_GEN4VF(csr_base_addr, bank, ring) \ read_base_gen4vf((csr_base_addr), (bank), (ring)) diff --git a/sys/dev/qat/include/common/adf_accel_devices.h b/sys/dev/qat/include/common/adf_accel_devices.h index eeffc6a9132c..1da5c8a7afce 100644 --- a/sys/dev/qat/include/common/adf_accel_devices.h +++ b/sys/dev/qat/include/common/adf_accel_devices.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef ADF_ACCEL_DEVICES_H_ #define ADF_ACCEL_DEVICES_H_ @@ -199,11 +199,11 @@ struct adf_accel_unit { } __packed; struct adf_accel_unit_info { - u32 inline_ingress_msk; - u32 inline_egress_msk; - u32 sym_ae_msk; - u32 asym_ae_msk; - u32 dc_ae_msk; + u64 inline_ingress_msk; + u64 inline_egress_msk; + u64 sym_ae_msk; + u64 asym_ae_msk; + u64 dc_ae_msk; u8 num_cy_au; u8 num_dc_au; u8 num_asym_au; @@ -287,7 +287,10 @@ struct adf_hw_csr_ops { void (*write_csr_int_flag)(struct resource *csr_base_addr, u32 bank, u32 value); - void (*write_csr_int_srcsel)(struct resource *csr_base_addr, u32 bank); + void (*write_csr_int_srcsel)(struct resource *csr_base_addr, + u32 bank, + u32 idx, + u32 value); void (*write_csr_int_col_en)(struct resource *csr_base_addr, u32 bank, u32 value); @@ -342,7 +345,7 @@ struct adf_hw_csr_info { struct adf_hw_device_data { struct adf_hw_device_class *dev_class; uint32_t (*get_accel_mask)(struct adf_accel_dev *accel_dev); - uint32_t (*get_ae_mask)(struct adf_accel_dev *accel_dev); + uint64_t (*get_ae_mask)(struct adf_accel_dev *accel_dev); uint32_t (*get_sram_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_misc_bar_id)(struct adf_hw_device_data *self); uint32_t (*get_etr_bar_id)(struct adf_hw_device_data *self); @@ -402,7 +405,7 @@ struct adf_hw_device_data { void (*reset_hw_units)(struct adf_accel_dev *accel_dev); int (*measure_clock)(struct adf_accel_dev *accel_dev); void (*restore_device)(struct adf_accel_dev *accel_dev); - uint32_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev, + uint64_t (*get_obj_cfg_ae_mask)(struct adf_accel_dev *accel_dev, enum adf_accel_unit_services services); enum adf_accel_unit_services ( *get_service_type)(struct adf_accel_dev *accel_dev, s32 obj_num); @@ -440,14 +443,14 @@ struct adf_hw_device_data { uint32_t instance_id; uint16_t accel_mask; u32 aerucm_mask; - u32 ae_mask; - u32 admin_ae_mask; + u64 ae_mask; + u64 admin_ae_mask; u32 service_mask; u32 service_to_load_mask; u32 heartbeat_ctr_num; uint16_t tx_rings_mask; uint8_t tx_rx_gap; - uint8_t num_banks; + uint16_t num_banks; u8 num_rings_per_bank; uint8_t num_accel; uint8_t num_logical_accel; @@ -614,6 +617,7 @@ struct adf_admin_comms { bus_dmamap_t hb_map; char *virt_addr; char *virt_hb_addr; + uint32_t mailbox_offset; struct resource *mailbox_addr; struct sx lock; struct bus_dmamem dma_mem; @@ -637,9 +641,9 @@ struct adf_accel_vf_info { }; struct adf_fw_versions { - u8 fw_version_major; - u8 fw_version_minor; - u8 fw_version_patch; + u16 fw_version_major; + u16 fw_version_minor; + u32 fw_version_patch; u8 mmp_version_major; u8 mmp_version_minor; u8 mmp_version_patch; diff --git a/sys/dev/qat/include/common/adf_common_drv.h b/sys/dev/qat/include/common/adf_common_drv.h index f9f4463f69c3..3d4d0530866e 100644 --- a/sys/dev/qat/include/common/adf_common_drv.h +++ b/sys/dev/qat/include/common/adf_common_drv.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef ADF_DRV_H #define ADF_DRV_H @@ -145,7 +145,7 @@ struct icp_qat_fw_init_admin_resp; int adf_send_admin(struct adf_accel_dev *accel_dev, struct icp_qat_fw_init_admin_req *req, struct icp_qat_fw_init_admin_resp *resp, - u32 ae_mask); + u64 ae_mask); int adf_config_device(struct adf_accel_dev *accel_dev); int adf_init_admin_comms(struct adf_accel_dev *accel_dev); diff --git a/sys/dev/qat/include/common/adf_gen2_hw_data.h b/sys/dev/qat/include/common/adf_gen2_hw_data.h index 3e62431931d0..761cb69d1711 100644 --- a/sys/dev/qat/include/common/adf_gen2_hw_data.h +++ b/sys/dev/qat/include/common/adf_gen2_hw_data.h @@ -1,5 +1,5 @@ -/* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2021 Intel Corporation */ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef ADF_GEN2_HW_DATA_H_ #define ADF_GEN2_HW_DATA_H_ @@ -7,8 +7,10 @@ #include "adf_cfg_common.h" /* Transport access */ +#define ADF_RINGS_PER_INT_SRCSEL 8 +#define ADF_RING_CSR_NEXT_INT_SRCSEL 0x4 #define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL -#define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_BANK_INT_SRC_SEL_MASK 0x44444444UL #define ADF_RING_CSR_RING_CONFIG 0x000 #define ADF_RING_CSR_RING_LBASE 0x040 #define ADF_RING_CSR_RING_UBASE 0x080 @@ -97,17 +99,12 @@ read_base(struct resource *csr_base_addr, u32 bank, u32 ring) ADF_CSR_WR(csr_base_addr, \ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_FLAG, \ value) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ - do { \ - ADF_CSR_WR(csr_base_addr, \ - (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_SRCSEL, \ - ADF_BANK_INT_SRC_SEL_MASK_0); \ - ADF_CSR_WR(csr_base_addr, \ - (ADF_RING_BUNDLE_SIZE * (bank)) + \ - ADF_RING_CSR_INT_SRCSEL_2, \ - ADF_BANK_INT_SRC_SEL_MASK_X); \ - } while (0) +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank, idx, value) \ + ADF_CSR_WR(csr_base_addr, \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + (ADF_RING_CSR_INT_SRCSEL + \ + ((idx)*ADF_RING_CSR_NEXT_INT_SRCSEL)), \ + (value)) #define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ ADF_CSR_WR(csr_base_addr, \ (ADF_RING_BUNDLE_SIZE * (bank)) + ADF_RING_CSR_INT_COL_EN, \ diff --git a/sys/dev/qat/include/common/adf_gen4_hw_data.h b/sys/dev/qat/include/common/adf_gen4_hw_data.h index cde5ae1f4e10..c93b7215635e 100644 --- a/sys/dev/qat/include/common/adf_gen4_hw_data.h +++ b/sys/dev/qat/include/common/adf_gen4_hw_data.h @@ -1,11 +1,13 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef ADF_GEN4_HW_CSR_DATA_H_ #define ADF_GEN4_HW_CSR_DATA_H_ #include "adf_accel_devices.h" /* Transport access */ +#define ADF_RINGS_PER_INT_SRCSEL 2 +#define ADF_RING_CSR_NEXT_INT_SRCSEL 0x4 #define ADF_BANK_INT_SRC_SEL_MASK 0x44UL #define ADF_RING_CSR_RING_CONFIG 0x1000 #define ADF_RING_CSR_RING_LBASE 0x1040 @@ -117,11 +119,13 @@ read_base_gen4(struct resource *csr_base_addr, u32 bank, u32 ring) ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ ADF_RING_CSR_INT_FLAG, \ (value)) -#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ +#define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank, idx, value) \ ADF_CSR_WR((csr_base_addr), \ - ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ - ADF_RING_CSR_INT_SRCSEL, \ - ADF_BANK_INT_SRC_SEL_MASK) + ADF_RING_CSR_ADDR_OFFSET + \ + (ADF_RING_BUNDLE_SIZE * (bank)) + \ + (ADF_RING_CSR_INT_SRCSEL + \ + ((idx)*ADF_RING_CSR_NEXT_INT_SRCSEL)), \ + (value)) #define WRITE_CSR_INT_COL_EN(csr_base_addr, bank, value) \ ADF_CSR_WR((csr_base_addr), \ ADF_RING_CSR_ADDR_OFFSET + ADF_RING_BUNDLE_SIZE * (bank) + \ diff --git a/sys/dev/qat/include/common/adf_uio.h b/sys/dev/qat/include/common/adf_uio.h index 9e32787ce128..0565da7a488d 100644 --- a/sys/dev/qat/include/common/adf_uio.h +++ b/sys/dev/qat/include/common/adf_uio.h @@ -1,11 +1,11 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2023 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef ADF_UIO_H #define ADF_UIO_H #include "adf_accel_devices.h" struct qat_uio_bundle_dev { - u8 hardware_bundle_number; + u16 hardware_bundle_number; struct adf_uio_control_bundle *bundle; struct adf_uio_control_accel *accel; }; diff --git a/sys/dev/qat/include/common/adf_uio_control.h b/sys/dev/qat/include/common/adf_uio_control.h index 032baa9b54c2..da1b57ef12c6 100644 --- a/sys/dev/qat/include/common/adf_uio_control.h +++ b/sys/dev/qat/include/common/adf_uio_control.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef QAT_UIO_CONTROL_H #define QAT_UIO_CONTROL_H #include <sys/condvar.h> @@ -11,7 +11,7 @@ struct adf_uio_instance_rings { }; struct adf_uio_control_bundle { - uint8_t hardware_bundle_number; + uint16_t hardware_bundle_number; bool used; struct list_head list; struct mutex list_lock; /* protects list struct */ diff --git a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h index 0873e56031db..a94a8b2b9702 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h +++ b/sys/dev/qat/qat_api/common/crypto/sym/include/lac_sym_cipher_defs.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /** ***************************************************************************** @@ -52,6 +52,12 @@ #define LAC_CIPHER_CCM_AAD_OFFSET \ (LAC_CIPHER_CCM_B0_SIZE + LAC_CIPHER_CCM_ENCODED_AAD_LEN_SIZE) +/* The hardware requires the AAD buffer to be padded out to a multiple + * of 16 for these algorithms: + * AES_CCM, AES_GCM, CHACHA + */ +#define LAC_SYM_CIPHER_AAD_PADLEN 16 + #define LAC_SYM_SNOW3G_CIPHER_CONFIG_FOR_HASH_SZ 40 /* Snow3g cipher config required for performing a Snow3g hash operation. * It contains 8 Bytes of config for hardware, 16 Bytes of Key and requires diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c index 56f211025103..cd383844b3ed 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_alg_chain.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /** *************************************************************************** @@ -1567,7 +1567,6 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, CpaCySymCipherAlgorithm cipher; CpaCySymHashAlgorithm hash; Cpa8U paddingLen = 0; - Cpa8U blockLen = 0; CpaBoolean digestIsAppended = CPA_FALSE; Cpa32U aadLenInBytes = 0; Cpa64U srcPktSize = 0; @@ -2000,13 +1999,17 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, if ((SPC == pSessionDesc->singlePassState) && CPA_STATUS_SUCCESS == status) { Cpa64U aadBufferPhysAddr = 0; - - /* For CHACHA and AES-GCM there is an AAD buffer - * if aadLenInBytes is nonzero In case of - * AES-GMAC, AAD buffer passed in the src + /* Pad the AAD buffer to a blocklen multiple and + * get the physical address of the buffer. For + * CHACHA and AES-GCM there is an AAD buffer if + * aadLenInBytes is nonzero. For CCM there is + * always an AAD buffer, even if aadLenInBytes + * is zero, as the first 16 bytes are needed for + * B0. In case of AES-GMAC, no AAD buffer is + * needed as the AAD is passed in the src * buffer. */ - if ((0 != aadLenInBytes && + if ((0 < aadLenInBytes && CPA_CY_SYM_HASH_AES_GMAC != hash) || isSpCcm) { LAC_CHECK_NULL_PARAM( @@ -2014,19 +2017,23 @@ LacAlgChain_Perform(const CpaInstanceHandle instanceHandle, Cpa32U aadDataLen = pSessionDesc->aadLenInBytes; - /* In case of AES_CCM, B0 block size and - * 2 bytes of AAD len encoding need to - * be added to total AAD data len */ - if (isSpCcm) + /* In case of AES_CCM, start padding + * after B0 block size plus 2 bytes of + * AAD len encoding plus AAD data len. + * Padding is only needed for aadLen > + * 0. + */ + if (isSpCcm && 0 < aadLenInBytes) aadDataLen += LAC_CIPHER_CCM_AAD_OFFSET; - blockLen = - LacSymQat_CipherBlockSizeBytesGet( - cipher); - if ((aadDataLen % blockLen) != 0) { - paddingLen = blockLen - - (aadDataLen % blockLen); + if (aadDataLen % + LAC_SYM_CIPHER_AAD_PADLEN != + 0) { + paddingLen = + LAC_SYM_CIPHER_AAD_PADLEN - + (aadDataLen % + LAC_SYM_CIPHER_AAD_PADLEN); memset( &pOpData ->pAdditionalAuthData diff --git a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c index 65a0d17d307f..882dc43d7edf 100644 --- a/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c +++ b/sys/dev/qat/qat_api/common/crypto/sym/lac_sym_dp.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /** *************************************************************************** @@ -261,11 +261,14 @@ LacDp_EnqueueParamCheck(const CpaCySymDpOpData *pRequest) ->generic_service_info.capabilitiesMask; if (LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask) && (LAC_CIPHER_SPC_IV_SIZE == pRequest->ivLenInBytes)) { - /* For CHACHA and AES_GCM single pass there is an AAD - * buffer if aadLenInBytes is nonzero. AES_GMAC AAD is - * stored in source buffer, therefore there is no - * separate AAD buffer. */ - if ((0 != pSessionDesc->aadLenInBytes) && + /* For CHACHA and AES-GCM there is an AAD buffer if + * aadLenInBytes is nonzero. For CCM there is always an + * AAD buffer, even if aadLenInBytes is zero, as the + * first 16 bytes are needed for B0. In case of + * AES-GMAC, no AAD buffer is needed as the AAD is + * passed in the src buffer. + */ + if ((0 < pSessionDesc->aadLenInBytes) && (CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm)) { LAC_CHECK_NULL_PARAM( @@ -593,8 +596,7 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, LAC_CIPHER_IS_SPC(cipher, hash, capabilitiesMask); Cpa8U paddingLen = 0; - Cpa8U blockLen = 0; - Cpa32U aadDataLen = 0; + Cpa32U aadLenInBytes = 0; pMsgDummy = (Cpa8U *)pCurrentQatMsg; /* Write Request */ @@ -832,29 +834,35 @@ LacDp_WriteRingMsgFull(CpaCySymDpOpData *pRequest, (Cpa8U)pSessionDesc->hashResultSize; } - /* For CHACHA and AES_GCM single pass AAD buffer needs - * alignment if aadLenInBytes is nonzero. In case of - * AES-GMAC, AAD buffer passed in the src buffer. + /* Pad the AAD buffer to a blocklen multiple. + * For ChaChaPoly and AES-GCM there is an AAD buffer if + * aadLenInBytes is nonzero. For CCM there is always an + * AAD buffer, even if aadLenInBytes is zero, as the + * first 16 bytes are needed for B0, but padding is only + * needed if aadLen > 0. In case of AES-GMAC, there's no + * AAD buffer as the AAD is passed in the src buffer, so + * no padding needed. */ - if (0 != pSessionDesc->aadLenInBytes && + if (0 < pSessionDesc->aadLenInBytes && CPA_CY_SYM_HASH_AES_GMAC != pSessionDesc->hashAlgorithm) { - blockLen = LacSymQat_CipherBlockSizeBytesGet( - pSessionDesc->cipherAlgorithm); - aadDataLen = pSessionDesc->aadLenInBytes; - - /* In case of AES_CCM, B0 block size and 2 bytes - * of AAD len - * encoding need to be added to total AAD data - * len */ + aadLenInBytes = pSessionDesc->aadLenInBytes; + + /* In case of AES_CCM, start padding after B0 + * block size plus 2 bytes of AAD len encoding + * plus AAD data len. + */ if (isSpCcm) - aadDataLen += LAC_CIPHER_CCM_AAD_OFFSET; + aadLenInBytes += + LAC_CIPHER_CCM_AAD_OFFSET; - if ((aadDataLen % blockLen) != 0) { - paddingLen = - blockLen - (aadDataLen % blockLen); + if (aadLenInBytes % LAC_SYM_CIPHER_AAD_PADLEN != + 0) { + paddingLen = LAC_SYM_CIPHER_AAD_PADLEN - + (aadLenInBytes % + LAC_SYM_CIPHER_AAD_PADLEN); memset(&pRequest->pAdditionalAuthData - [aadDataLen], + [aadLenInBytes], 0, paddingLen); } diff --git a/sys/dev/qat/qat_api/common/ctrl/sal_get_instances.c b/sys/dev/qat/qat_api/common/ctrl/sal_get_instances.c index f68853dc43a8..c09d75608bb4 100644 --- a/sys/dev/qat/qat_api/common/ctrl/sal_get_instances.c +++ b/sys/dev/qat/qat_api/common/ctrl/sal_get_instances.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /** ***************************************************************************** @@ -134,6 +134,11 @@ Lac_GetCyNumInstancesByType( if (CPA_ACC_SVC_TYPE_CRYPTO_ASYM == accelerationServiceType || CPA_ACC_SVC_TYPE_CRYPTO == accelerationServiceType) { list_temp = base_addr->asym_services; + if ((NULL == list_temp) && + (CPA_ACC_SVC_TYPE_CRYPTO != + accelerationServiceType)) { + list_temp = base_addr->crypto_services; + } while (NULL != list_temp) { instanceHandle = SalList_getObject(list_temp); status = cpaCyInstanceGetInfo2(instanceHandle, @@ -149,6 +154,11 @@ Lac_GetCyNumInstancesByType( if (CPA_ACC_SVC_TYPE_CRYPTO_SYM == accelerationServiceType || CPA_ACC_SVC_TYPE_CRYPTO == accelerationServiceType) { list_temp = base_addr->sym_services; + if ((NULL == list_temp) && + (CPA_ACC_SVC_TYPE_CRYPTO != + accelerationServiceType)) { + list_temp = base_addr->crypto_services; + } while (NULL != list_temp) { instanceHandle = SalList_getObject(list_temp); status = cpaCyInstanceGetInfo2(instanceHandle, @@ -292,6 +302,11 @@ Lac_GetCyInstancesByType( if (CPA_ACC_SVC_TYPE_CRYPTO_ASYM == accelerationServiceType || CPA_ACC_SVC_TYPE_CRYPTO == accelerationServiceType) { list_temp = base_addr->asym_services; + if ((NULL == list_temp) && + (CPA_ACC_SVC_TYPE_CRYPTO != + accelerationServiceType)) { + list_temp = base_addr->crypto_services; + } while (NULL != list_temp) { if (index > (numInstances - 1)) break; @@ -312,6 +327,11 @@ Lac_GetCyInstancesByType( if (CPA_ACC_SVC_TYPE_CRYPTO_SYM == accelerationServiceType || CPA_ACC_SVC_TYPE_CRYPTO == accelerationServiceType) { list_temp = base_addr->sym_services; + if ((NULL == list_temp) && + (CPA_ACC_SVC_TYPE_CRYPTO != + accelerationServiceType)) { + list_temp = base_addr->crypto_services; + } while (NULL != list_temp) { if (index > (numInstances - 1)) break; diff --git a/sys/dev/qat/qat_api/common/utils/lac_lock_free_stack.h b/sys/dev/qat/qat_api/common/utils/lac_lock_free_stack.h index 49e2329c25a7..f1e65d428bff 100644 --- a/sys/dev/qat/qat_api/common/utils/lac_lock_free_stack.h +++ b/sys/dev/qat/qat_api/common/utils/lac_lock_free_stack.h @@ -1,33 +1,45 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #ifndef LAC_LOCK_FREE_STACK_H_1 #define LAC_LOCK_FREE_STACK_H_1 #include "lac_mem_pools.h" +#ifdef __LP64__ +typedef unsigned int atomic_int __attribute__((mode(TI))); +#else +typedef unsigned int atomic_int __attribute__((mode(DI))); +#endif + typedef union { struct { - uint64_t ctr : 16; - uint64_t ptr : 48; + unsigned long ctr; + void *ptr; }; - uint64_t atomic; + atomic_int atomic; } pointer_t; typedef struct { volatile pointer_t top; } lock_free_stack_t; -static inline void * -PTR(const uintptr_t addr48) +static inline char +lac_atomic_cmp_swap(volatile pointer_t *ptr, + pointer_t old_val, + pointer_t new_val) { -#ifdef __x86_64__ - const int64_t addr64 = addr48 << 16; + uint64_t new_high, new_low, old_high, old_low; + char res; - /* Do arithmetic shift to restore kernel canonical address (if not NULL) - */ - return (void *)(addr64 >> 16); -#else - return (void *)(addr48); -#endif + old_low = old_val.ctr; + old_high = (uintptr_t)old_val.ptr; + new_low = new_val.ctr; + new_high = (uintptr_t)new_val.ptr; + + __asm volatile("lock;cmpxchg16b\t%1" + : "=@cce"(res), "+m"(*ptr), "+a"(old_low), "+d"(old_high) + : "b"(new_low), "c"(new_high) + : "memory", "cc"); + return (res); } static inline lac_mem_blk_t * @@ -39,15 +51,13 @@ pop(lock_free_stack_t *stack) do { old_top.atomic = stack->top.atomic; - next = PTR(old_top.ptr); + next = old_top.ptr; if (NULL == next) return next; - new_top.ptr = (uintptr_t)next->pNext; + new_top.ptr = next->pNext; new_top.ctr = old_top.ctr + 1; - } while (!__sync_bool_compare_and_swap(&stack->top.atomic, - old_top.atomic, - new_top.atomic)); + } while (!lac_atomic_cmp_swap(&stack->top, old_top, new_top)); return next; } @@ -60,18 +70,16 @@ push(lock_free_stack_t *stack, lac_mem_blk_t *val) do { old_top.atomic = stack->top.atomic; - val->pNext = PTR(old_top.ptr); - new_top.ptr = (uintptr_t)val; + val->pNext = old_top.ptr; + new_top.ptr = val; new_top.ctr = old_top.ctr + 1; - } while (!__sync_bool_compare_and_swap(&stack->top.atomic, - old_top.atomic, - new_top.atomic)); + } while (!lac_atomic_cmp_swap(&stack->top, old_top, new_top)); } static inline lock_free_stack_t _init_stack(void) { - lock_free_stack_t stack = { { { 0 } } }; + lock_free_stack_t stack = { .top.atomic = 0 }; return stack; } @@ -79,7 +87,7 @@ static inline lac_mem_blk_t * top(lock_free_stack_t *stack) { pointer_t old_top = stack->top; - lac_mem_blk_t *next = PTR(old_top.ptr); + lac_mem_blk_t *next = old_top.ptr; return next; } diff --git a/sys/dev/qat/qat_api/device/dev_info.c b/sys/dev/qat/qat_api/device/dev_info.c index d4eb9c3b6e0c..d88cd603ed02 100644 --- a/sys/dev/qat/qat_api/device/dev_info.c +++ b/sys/dev/qat/qat_api/device/dev_info.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /** ***************************************************************************** * @file dev_info.c @@ -34,25 +34,10 @@ cpaGetDeviceInfo(Cpa16U device, CpaDeviceInfo *deviceInfo) { CpaStatus status = CPA_STATUS_SUCCESS; icp_accel_dev_t *pDevice = NULL; - Cpa16U numDevicesAvail = 0; Cpa32U capabilitiesMask = 0; Cpa32U enabledServices = 0; LAC_CHECK_NULL_PARAM(deviceInfo); - status = icp_amgr_getNumInstances(&numDevicesAvail); - /* Check if the application is not attempting to access a - * device that does not exist. - */ - if (0 == numDevicesAvail) { - QAT_UTILS_LOG("Failed to retrieve number of devices!\n"); - return CPA_STATUS_FAIL; - } - if (device >= numDevicesAvail) { - QAT_UTILS_LOG( - "Invalid device access! Number of devices available: %d.\n", - numDevicesAvail); - return CPA_STATUS_FAIL; - } /* Clear the entire capability structure before initialising it */ memset(deviceInfo, 0x00, sizeof(CpaDeviceInfo)); @@ -62,7 +47,7 @@ cpaGetDeviceInfo(Cpa16U device, CpaDeviceInfo *deviceInfo) pDevice = icp_adf_getAccelDevByAccelId(device); if (NULL == pDevice) { QAT_UTILS_LOG("Failed to retrieve device.\n"); - return status; + return CPA_STATUS_FAIL; } /* Device of interest is found, retrieve the information for it */ diff --git a/sys/dev/qat/qat_api/include/icp_sal_versions.h b/sys/dev/qat/qat_api/include/icp_sal_versions.h index 0eb227ade09c..f1c63ad85cc9 100644 --- a/sys/dev/qat/qat_api/include/icp_sal_versions.h +++ b/sys/dev/qat/qat_api/include/icp_sal_versions.h @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /** *************************************************************************** * @file icp_sal_versions.h @@ -26,7 +26,7 @@ /* Part name and number of the accelerator device */ #define SAL_INFO2_DRIVER_SW_VERSION_MAJ_NUMBER 3 -#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 16 +#define SAL_INFO2_DRIVER_SW_VERSION_MIN_NUMBER 17 #define SAL_INFO2_DRIVER_SW_VERSION_PATCH_NUMBER 0 /** diff --git a/sys/dev/qat/qat_api/qat_kernel/src/lac_adf_interface_freebsd.c b/sys/dev/qat/qat_api/qat_kernel/src/lac_adf_interface_freebsd.c index 12cce62d7806..e20cec22901c 100644 --- a/sys/dev/qat/qat_api/qat_kernel/src/lac_adf_interface_freebsd.c +++ b/sys/dev/qat/qat_api/qat_kernel/src/lac_adf_interface_freebsd.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #include "adf_cfg.h" #include "cpa.h" #include "icp_accel_devices.h" @@ -49,6 +49,7 @@ create_adf_dev_structure(struct adf_accel_dev *accel_dev) adf->accelCapabilitiesMask = hw_data->accel_capabilities_mask; adf->sku = hw_data->get_sku(hw_data); adf->accel_dev = accel_dev; + adf->pciDevId = pci_get_device(accel_to_pci_dev(adf->accel_dev)); accel_dev->lac_dev = adf; return adf; diff --git a/sys/dev/qat/qat_api/qat_kernel/src/lac_symbols.c b/sys/dev/qat/qat_api/qat_kernel/src/lac_symbols.c index d89ac14d0731..143afa76b6f9 100644 --- a/sys/dev/qat/qat_api/qat_kernel/src/lac_symbols.c +++ b/sys/dev/qat/qat_api/qat_kernel/src/lac_symbols.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ /****************************************************************************** * @file lac_symbols.c * @@ -70,4 +70,8 @@ EXPORT_SYMBOL(icp_sal_iommu_get_remap_size); EXPORT_SYMBOL(icp_sal_iommu_map); EXPORT_SYMBOL(icp_sal_iommu_unmap); +/* Device symbols */ +EXPORT_SYMBOL(cpaGetNumDevices); +EXPORT_SYMBOL(cpaGetDeviceInfo); + EXPORT_SYMBOL(icp_sal_get_dc_error); diff --git a/sys/dev/qat/qat_common/adf_accel_engine.c b/sys/dev/qat/qat_common/adf_accel_engine.c index e712f750501c..4aeb32dc57f2 100644 --- a/sys/dev/qat/qat_common/adf_accel_engine.c +++ b/sys/dev/qat/qat_common/adf_accel_engine.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #include "qat_freebsd.h" #include "adf_cfg.h" #include "adf_common_drv.h" @@ -49,7 +49,7 @@ adf_ae_fw_load(struct adf_accel_dev *accel_dev) u32 max_objs = 1; const char *obj_name = NULL; struct adf_mmp_version_s mmp_ver = { { 0 } }; - unsigned int cfg_ae_mask = 0; + u64 cfg_ae_mask = 0; if (!hw_device->fw_name) return 0; @@ -206,7 +206,7 @@ adf_ae_stop(struct adf_accel_dev *accel_dev) return 0; for (ae = 0, ae_ctr = 0; ae < max_aes; ae++) { - if (hw_data->ae_mask & (1 << ae)) { + if (hw_data->ae_mask & (1ULL << ae)) { qat_hal_stop(loader_data->fw_loader, ae, 0xFF); ae_ctr++; } diff --git a/sys/dev/qat/qat_common/adf_cfg_device.c b/sys/dev/qat/qat_common/adf_cfg_device.c index 4860a4064b97..4c07434817ad 100644 --- a/sys/dev/qat/qat_common/adf_cfg_device.c +++ b/sys/dev/qat/qat_common/adf_cfg_device.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2025 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #include "adf_cfg_instance.h" #include "adf_cfg_section.h" #include "adf_cfg_device.h" @@ -373,7 +373,7 @@ adf_cfg_gen_dispatch_arbiter(struct adf_accel_dev *accel_dev, u16 ena_srv_mask = GET_HW_DATA(accel_dev)->ring_to_svc_map; for (engine = 0; engine < total_engines; engine++) { - if (!(GET_HW_DATA(accel_dev)->ae_mask & (1 << engine))) + if (!(GET_HW_DATA(accel_dev)->ae_mask & (1ULL << engine))) continue; bits = 0; /* ability_map is used to indicate the threads ability */ @@ -1015,7 +1015,8 @@ adf_cfg_static_conf(struct adf_accel_dev *accel_dev) ret |= adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC); snprintf(key, ADF_CFG_MAX_KEY_LEN_IN_BYTES, ADF_SERVICES_ENABLED); - if (strcmp(ADF_CFG_SYM_ASYM, accel_dev->cfg->cfg_services) == 0) { + if (strcmp(ADF_CFG_SYM_ASYM, accel_dev->cfg->cfg_services) == 0 && + !(IS_QAT_GEN4(pci_get_device(GET_DEV(accel_dev))))) { strncpy(value, ADF_CFG_CY, ADF_CFG_MAX_VAL_LEN_IN_BYTES); } else { strncpy(value, diff --git a/sys/dev/qat/qat_common/adf_ctl_drv.c b/sys/dev/qat/qat_common/adf_ctl_drv.c index 71b1e107cb5b..588a850b927a 100644 --- a/sys/dev/qat/qat_common/adf_ctl_drv.c +++ b/sys/dev/qat/qat_common/adf_ctl_drv.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #include "qat_freebsd.h" #include "adf_cfg.h" #include "adf_common_drv.h" @@ -135,6 +135,7 @@ static int adf_ctl_ioctl_get_status(unsigned int cmd, struct adf_hw_device_data *hw_data; struct adf_dev_status_info *dev_info; struct adf_accel_dev *accel_dev; + uint16_t banks_per_accel; dev_info = (struct adf_dev_status_info *)arg; @@ -143,12 +144,18 @@ static int adf_ctl_ioctl_get_status(unsigned int cmd, return ENODEV; hw_data = accel_dev->hw_device; + + /* If the number of banks per accel exceeds the max value of uint8_t, + * set it to 0 to indicate that it's not valid. + */ + banks_per_accel = hw_data->num_banks / hw_data->num_logical_accel; + banks_per_accel = banks_per_accel > UINT8_MAX ? 0 : banks_per_accel; + dev_info->state = adf_dev_started(accel_dev) ? DEV_UP : DEV_DOWN; dev_info->num_ae = hw_data->get_num_aes(hw_data); dev_info->num_accel = hw_data->get_num_accels(hw_data); dev_info->num_logical_accel = hw_data->num_logical_accel; - dev_info->banks_per_accel = hw_data->num_banks - / hw_data->num_logical_accel; + dev_info->banks_per_accel = (uint8_t)banks_per_accel; strlcpy(dev_info->name, hw_data->dev_class->name, sizeof(dev_info->name)); dev_info->instance_id = hw_data->instance_id; diff --git a/sys/dev/qat/qat_common/adf_freebsd_admin.c b/sys/dev/qat/qat_common/adf_freebsd_admin.c index 8a8c890908c5..a2213c032d70 100644 --- a/sys/dev/qat/qat_common/adf_freebsd_admin.c +++ b/sys/dev/qat/qat_common/adf_freebsd_admin.c @@ -1,5 +1,5 @@ /* SPDX-License-Identifier: BSD-3-Clause */ -/* Copyright(c) 2007-2022 Intel Corporation */ +/* Copyright(c) 2007-2026 Intel Corporation */ #include "qat_freebsd.h" #include "adf_cfg.h" #include "adf_common_drv.h" @@ -204,10 +204,10 @@ int adf_send_admin(struct adf_accel_dev *accel_dev, struct icp_qat_fw_init_admin_req *req, struct icp_qat_fw_init_admin_resp *resp, - u32 ae_mask) + u64 ae_mask) { int i; - unsigned int mask; + u64 mask; for (i = 0, mask = ae_mask; mask; i++, mask >>= 1) { if (!(mask & 1)) @@ -226,7 +226,7 @@ adf_init_me(struct adf_accel_dev *accel_dev) struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; + u64 ae_mask = hw_device->ae_mask; explicit_bzero(&req, sizeof(req)); explicit_bzero(&resp, sizeof(resp)); @@ -250,7 +250,7 @@ adf_set_heartbeat_timer(struct adf_accel_dev *accel_dev) struct icp_qat_fw_init_admin_req req; struct icp_qat_fw_init_admin_resp resp; struct adf_hw_device_data *hw_device = accel_dev->hw_device; - u32 ae_mask = hw_device->ae_mask; + u64 ae_mask = hw_device->ae_mask; u32 heartbeat_ticks; explicit_bzero(&req, sizeof(req)); @@ -271,7 +271,7 @@ adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, u32 *capabilities) { struct icp_qat_fw_init_admin_req req; *** 1185 LINES SKIPPED ***
