From: Alex Ousherovitch <[email protected]> Register AES algorithms using the CMH AES core (core ID 0x03): - skcipher: AES-ECB, AES-CBC, AES-CTR, AES-XTS, AES-CFB - aead: AES-GCM, AES-CCM - ahash: AES-CMAC
Supports 128, 192, and 256-bit keys. AEAD algorithms handle associated data, payload, and authentication tag with correct encrypt/decrypt separation. Co-developed-by: Saravanakrishnan Krishnamoorthy <[email protected]> Signed-off-by: Saravanakrishnan Krishnamoorthy <[email protected]> Signed-off-by: Alex Ousherovitch <[email protected]> Reviewed-by: Joel Wittenauer <[email protected]> Reviewed-by: Thi Nguyen <[email protected]> --- drivers/crypto/cmh/Makefile | 5 +- drivers/crypto/cmh/cmh_aes.c | 736 ++++++++++++++++++++ drivers/crypto/cmh/cmh_aes_aead.c | 987 +++++++++++++++++++++++++++ drivers/crypto/cmh/cmh_aes_cmac.c | 537 +++++++++++++++ drivers/crypto/cmh/cmh_main.c | 25 + drivers/crypto/cmh/include/cmh_aes.h | 24 + 6 files changed, 2313 insertions(+), 1 deletion(-) create mode 100644 drivers/crypto/cmh/cmh_aes.c create mode 100644 drivers/crypto/cmh/cmh_aes_aead.c create mode 100644 drivers/crypto/cmh/cmh_aes_cmac.c create mode 100644 drivers/crypto/cmh/include/cmh_aes.h diff --git a/drivers/crypto/cmh/Makefile b/drivers/crypto/cmh/Makefile index b3018fbcf211..ced8d1748e6c 100644 --- a/drivers/crypto/cmh/Makefile +++ b/drivers/crypto/cmh/Makefile @@ -19,7 +19,10 @@ cmh-y := \ cmh_hmac.o \ cmh_cshake.o \ cmh_kmac.o \ - cmh_sm3.o + cmh_sm3.o \ + cmh_aes.o \ + cmh_aes_aead.o \ + cmh_aes_cmac.o # Management ioctl device (/dev/cmh_mgmt): key lifecycle, PKE, PQC ioctls. cmh-$(CONFIG_CRYPTO_DEV_CMH_MGMT) += \ diff --git a/drivers/crypto/cmh/cmh_aes.c b/drivers/crypto/cmh/cmh_aes.c new file mode 100644 index 000000000000..b36295763e33 --- /dev/null +++ b/drivers/crypto/cmh/cmh_aes.c @@ -0,0 +1,736 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API AES (skcipher) Driver + * + * Registers skcipher algorithms with the Linux crypto subsystem: + * ecb(aes), cbc(aes), ctr(aes), cfb(aes), xts(aes) + * + * Uses the CMH AES Core via VCQ commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + [AES_CMD_UPDATE] + AES_CMD_FINAL + * + VCQ_CMD_FLUSH + * + * The AES core requires bidirectional DMA -- both input and output + * buffers are mapped and passed in a single AES_CMD_FINAL command. + * + * Raw-key atomicity: SYS_CMD_WRITE to SYS_REF_TEMP is packed into + * the same VCQ as AES commands (see cmh_key.h for details). + * + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/skcipher.h> +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <crypto/xts.h> +#include <crypto/scatterwalk.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/unaligned.h> + +#include "cmh_aes.h" +#include "cmh_vcq.h" +#include "cmh_aes_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* Algorithm Table */ + +struct cmh_aes_alg_info { + u32 aes_mode; /* AES_MODE_* */ + u32 ivsize; /* bytes (0 for ECB) */ + u32 min_keysize; /* minimum key bytes */ + u32 max_keysize; /* maximum key bytes */ + const char *alg_name; /* Linux crypto name: "ecb(aes)" */ + const char *drv_name; /* driver name: "cri-cmh-ecb-aes" */ +}; + +static const struct cmh_aes_alg_info aes_algs[] = { + { AES_MODE_ECB, 0, AES_KEYSIZE_128, AES_KEYSIZE_256, + "ecb(aes)", "cri-cmh-ecb-aes" }, + { AES_MODE_CBC, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256, + "cbc(aes)", "cri-cmh-cbc-aes" }, + { AES_MODE_CTR, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256, + "ctr(aes)", "cri-cmh-ctr-aes" }, + { AES_MODE_CFB, CMH_AES_IV_SIZE, AES_KEYSIZE_128, AES_KEYSIZE_256, + "cfb(aes)", "cri-cmh-cfb-aes" }, + { AES_MODE_XTS, CMH_AES_IV_SIZE, 2 * AES_KEYSIZE_128, 2 * AES_KEYSIZE_256, + "xts(aes)", "cri-cmh-xts-aes" }, +}; + +/* Per-transform context (allocated by crypto framework) */ + +struct cmh_aes_tfm_ctx { + struct cmh_key_ctx key; +}; + +/* Per-request context (lives in skcipher_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + [AES_CMD_UPDATE] + AES_CMD_FINAL + * + VCQ_CMD_FLUSH = 5 + * UPDATE is used for XTS data > 2 blocks (see cmh_aes_crypt). + */ +#define CMH_AES_MAX_PAYLOAD 5 +#define CMH_AES_MAX_PACKED (CMH_AES_MAX_PAYLOAD * 2) + +struct cmh_aes_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t iv2_dma; + dma_addr_t key_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *iv2_buf; + u32 cryptlen; + u32 ivsize; + u32 keylen; + u32 aes_mode; + u32 aes_op; + /* CTR counter-wrap split state */ + u32 ctr_chunk1_len; + u32 core_id; + s32 target_mbx; + u64 key_ref; + struct vcq_cmd packed[CMH_AES_MAX_PACKED]; +}; + +/* VCQ Builders -- AES-specific */ + +static void vcq_add_aes_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, u64 iv_dma, + u32 keylen, u32 ivlen, u32 mode, u32 op, + u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = iv_dma; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = ivlen; + slot->hwc.aes.cmd_init.mode = mode; + slot->hwc.aes.cmd_init.op = op; + slot->hwc.aes.cmd_init.aadlen = 0; + slot->hwc.aes.cmd_init.iolen = iolen; + slot->hwc.aes.cmd_init.taglen = 0; +} + +static void vcq_add_aes_update(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_UPDATE); + slot->hwc.aes.cmd_update.input = input_dma; + slot->hwc.aes.cmd_update.output = output_dma; + slot->hwc.aes.cmd_update.iolen = iolen; +} + +static void vcq_add_aes_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u32 iolen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_FINAL); + slot->hwc.aes.cmd_final.input = input_dma; + slot->hwc.aes.cmd_final.output = output_dma; + slot->hwc.aes.cmd_final.iolen = iolen; + slot->hwc.aes.cmd_final.tag = 0; + slot->hwc.aes.cmd_final.taglen = 0; +} + +/* + * We wrap each skcipher_alg with its info pointer in a compound struct, + * then use container_of() in cmh_aes_get_info() to recover it. + * This is the same pattern used by hash, hmac, cshake, kmac. + */ +struct cmh_aes_alg_drv { + struct skcipher_alg alg; + const struct cmh_aes_alg_info *info; +}; + +static bool aes_is_stream_mode(u32 mode) +{ + return mode == AES_MODE_CTR || mode == AES_MODE_CFB; +} + +/* + * Update req->iv after a successful encrypt/decrypt. + * + * The Linux skcipher API contract requires that req->iv is updated to + * reflect the state needed to continue processing in a chained call: + * CBC encrypt: IV <- last ciphertext block + * CBC decrypt: IV <- last ciphertext block of the *input* + * CTR: IV <- counter incremented by ceil(cryptlen / blocksize) + * CFB: IV <- last ciphertext block + */ +static void cmh_aes_update_iv(struct skcipher_request *req, u32 mode, + u32 op, const u8 *in_buf, const u8 *out_buf) +{ + u32 bs = CMH_AES_BLOCK_SIZE; + u32 nblocks; + + switch (mode) { + case AES_MODE_CBC: + if (op == AES_OP_ENCRYPT) + memcpy(req->iv, out_buf + req->cryptlen - bs, bs); + else + memcpy(req->iv, in_buf + req->cryptlen - bs, bs); + break; + case AES_MODE_CTR: + /* + * Arithmetic big-endian 128-bit counter increment. + * Process from the least-significant byte (index 15) + * upward, carrying as needed. + */ + nblocks = DIV_ROUND_UP(req->cryptlen, bs); + { + u8 *iv = req->iv; + int i; + + for (i = bs - 1; i >= 0 && nblocks; i--) { + u32 sum = (u32)iv[i] + (nblocks & 0xff); + + iv[i] = (u8)sum; + nblocks = (nblocks >> 8) + (sum >> 8); + } + } + break; + case AES_MODE_CFB: + /* + * CFB-128 chains on the last ciphertext block. On encrypt, + * that is out_buf; on decrypt, it is in_buf. + * + * For sub-block requests (cryptlen < 16), there is no + * complete ciphertext block to chain, so the IV is left + * unchanged -- CFB-128 has no defined chaining semantic + * for partial blocks (shift-register CFB-n is a different + * mode). Without this guard the pointer arithmetic + * underflows and reads before the buffer. + */ + if (req->cryptlen >= bs) { + if (op == AES_OP_ENCRYPT) + memcpy(req->iv, out_buf + req->cryptlen - bs, + bs); + else + memcpy(req->iv, in_buf + req->cryptlen - bs, + bs); + } + break; + default: + break; + } +} + +/* skcipher Operations */ + +static const struct cmh_aes_alg_info * +cmh_aes_get_info(struct crypto_skcipher *tfm) +{ + struct skcipher_alg *alg = crypto_skcipher_alg(tfm); + + return container_of(alg, struct cmh_aes_alg_drv, alg)->info; +} + +static int cmh_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + const struct cmh_aes_alg_info *info = cmh_aes_get_info(tfm); + + if (info->aes_mode == AES_MODE_XTS) { + int err; + + /* XTS: double key (32, 48, or 64 bytes) */ + if (keylen != 2 * AES_KEYSIZE_128 && + keylen != 2 * AES_KEYSIZE_192 && + keylen != 2 * AES_KEYSIZE_256) + return -EINVAL; + err = xts_verify_key(tfm, key, keylen); + if (err) + return err; + } else { + /* Standard: 16, 24, or 32 bytes */ + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) + return -EINVAL; + } + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES); +} + +static int cmh_aes_init_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct cmh_aes_reqctx)); + return 0; +} + +static void cmh_aes_exit_tfm(struct crypto_skcipher *tfm) +{ + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + + cmh_key_destroy(&tctx->key); +} + +#define CMH_AES_MAX_CRYPTLEN SZ_32M + +/* DMA unmap helper */ +static void cmh_aes_unmap_dma(struct cmh_aes_reqctx *rctx) +{ + if (rctx->iv2_buf) + cmh_dma_unmap_single(rctx->iv2_dma, rctx->ivsize, + DMA_TO_DEVICE); + if (rctx->ivsize > 0) + cmh_dma_unmap_single(rctx->iv_dma, rctx->ivsize, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, DMA_TO_DEVICE); +} + +static void cmh_aes_free_bufs(struct cmh_aes_reqctx *rctx) +{ + kfree(rctx->iv2_buf); + rctx->iv2_buf = NULL; + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; +} + +/* + * Submit the second CTR chunk after the first completes. + * Called from cmh_aes_complete when ctr_chunk1_len > 0. + */ +static void cmh_aes_complete(void *data, int error); + +static int cmh_aes_ctr_submit_chunk2(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_MAX_PAYLOAD]; + u32 chunk1 = rctx->ctr_chunk1_len; + u32 chunk2 = rctx->cryptlen - chunk1; + u64 key_ref; + u32 keylen; + u32 idx = 0; + + /* Clear split flag so next completion is final */ + rctx->ctr_chunk1_len = 0; + + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + + vcq_add_aes_init(&cmds[idx++], rctx->core_id, key_ref, + (u64)rctx->iv2_dma, keylen, rctx->ivsize, + rctx->aes_mode, rctx->aes_op, 0); + vcq_add_aes_final(&cmds[idx++], rctx->core_id, + (u64)(rctx->in_dma + chunk1), + (u64)(rctx->out_dma + chunk1), chunk2); + vcq_add_flush(&cmds[idx++], rctx->core_id); + + return cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_MAX_PACKED, + rctx->target_mbx, + cmh_aes_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); +} + +/* + * Async completion callback -- fires from RH threaded IRQ context. + * + * Unmaps DMA buffers, copies output to req->dst scatterlist, + * updates the IV state, frees temporaries, and completes the request. + * + * For CTR counter-wrap splits, the first chunk completion chains + * into a second VCQ submission rather than finalizing immediately. + */ +static void cmh_aes_complete(void *data, int error) +{ + struct skcipher_request *req = data; + struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* + * CTR counter-wrap: first chunk completed, submit second. + * DMA mappings remain valid (they cover the full buffer). + * + * Recursion depth bounded: chunk2 clears ctr_chunk1_len before + * submission, so the second cmh_aes_complete invocation sees 0 + * and finalizes (max depth = 2). + */ + if (rctx->ctr_chunk1_len && !error) { + int ret; + + ret = cmh_aes_ctr_submit_chunk2(req); + + if (!ret || ret == -EBUSY) + return; + /* Submission failed; clean up below */ + error = ret; + } + + cmh_aes_unmap_dma(rctx); + + if (!error) { + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + 0, rctx->cryptlen, 1); + cmh_aes_update_iv(req, rctx->aes_mode, rctx->aes_op, + rctx->in_buf, rctx->out_buf); + } + + cmh_aes_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * Core encrypt/decrypt -- builds a VCQ transaction and submits async. + * + * Returns -EINPROGRESS on successful submission (completion callback + * will fire later). Returns 0 for trivial cases (zero-length). + * Returns negative errno on pre-submission errors. + */ +static int cmh_aes_crypt(struct skcipher_request *req, u32 aes_op) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct cmh_aes_tfm_ctx *tctx = crypto_skcipher_ctx(tfm); + const struct cmh_aes_alg_info *info = cmh_aes_get_info(tfm); + struct cmh_aes_reqctx *rctx = skcipher_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + if (!req->cryptlen) + return 0; + + if (req->cryptlen > CMH_AES_MAX_CRYPTLEN) + return -EINVAL; + + switch (info->aes_mode) { + case AES_MODE_CTR: + case AES_MODE_CFB: + break; + case AES_MODE_XTS: + if (req->cryptlen < CMH_AES_BLOCK_SIZE) + return -EINVAL; + break; + default: + if (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1)) + return -EINVAL; + break; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Initialise reqctx */ + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = req->cryptlen; + rctx->ivsize = info->ivsize; + rctx->aes_mode = info->aes_mode; + rctx->aes_op = aes_op; + rctx->iv2_buf = NULL; + + /* Linearise input from scatterlist */ + rctx->in_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->in_buf) + return -ENOMEM; + + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, req->cryptlen, 0); + + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, req->cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + + /* Allocate and map output buffer */ + rctx->out_buf = kmalloc(req->cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, req->cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + + /* Map IV if required */ + if (info->ivsize > 0) { + rctx->iv_buf = kmemdup(req->iv, info->ivsize, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, info->ivsize, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* + * iolen in INIT: XTS needs total length upfront for tweak + * computation; all other modes use 0 (streaming). + */ + vcq_add_aes_init(&cmds[idx++], core_id, key_ref, (u64)rctx->iv_dma, + keylen, info->ivsize, info->aes_mode, aes_op, + info->aes_mode == AES_MODE_XTS ? + req->cryptlen : 0); + + if (info->aes_mode == AES_MODE_XTS && + req->cryptlen > 2 * CMH_AES_BLOCK_SIZE) { + u32 final_len, update_len; + + if (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1)) + final_len = CMH_AES_BLOCK_SIZE + + (req->cryptlen & (CMH_AES_BLOCK_SIZE - 1)); + else + final_len = 2 * CMH_AES_BLOCK_SIZE; + + update_len = req->cryptlen - final_len; + + vcq_add_aes_update(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, update_len); + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)(rctx->in_dma + update_len), + (u64)(rctx->out_dma + update_len), + final_len); + } else if (info->aes_mode == AES_MODE_CTR) { + /* + * CTR counter-wrap workaround: + * The AES-SCA hardware uses a 64-bit block counter. + * If the lower 64 bits of the IV would wrap during + * this operation, split into two separate VCQ + * transactions -- the completion callback for the + * first chunk submits the second. + */ + u64 lower64 = get_unaligned_be64(rctx->iv_buf + 8); + u32 nblocks = DIV_ROUND_UP(req->cryptlen, + CMH_AES_BLOCK_SIZE); + u64 bwrap = lower64 ? (~lower64 + 1ULL) : U64_MAX; + + if (nblocks > bwrap) { + u32 chunk1 = (u32)bwrap * CMH_AES_BLOCK_SIZE; + u64 upper64; + + /* Prepare second IV for chained submission */ + rctx->iv2_buf = kmalloc(info->ivsize, gfp); + if (!rctx->iv2_buf) { + ret = -ENOMEM; + goto out_unmap_iv; + } + upper64 = get_unaligned_be64(rctx->iv_buf); + put_unaligned_be64(upper64 + 1, rctx->iv2_buf); + put_unaligned_be64(0, rctx->iv2_buf + 8); + + rctx->iv2_dma = + cmh_dma_map_single(rctx->iv2_buf, + info->ivsize, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv2_dma)) { + ret = -ENOMEM; + goto out_free_iv2; + } + + /* Store state for the chained second submission */ + rctx->ctr_chunk1_len = chunk1; + rctx->core_id = core_id; + rctx->target_mbx = target_mbx; + rctx->key_ref = key_ref; + + /* First transaction: only chunk1 */ + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, chunk1); + } else { + /* No wrap: single FINAL with all data */ + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, + req->cryptlen); + } + } else { + vcq_add_aes_final(&cmds[idx++], core_id, + (u64)rctx->in_dma, + (u64)rctx->out_dma, req->cryptlen); + } + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_MAX_PACKED, target_mbx, + cmh_aes_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->iv2_buf) { + cmh_dma_unmap_single(rctx->iv2_dma, info->ivsize, + DMA_TO_DEVICE); + } +out_free_iv2: + kfree(rctx->iv2_buf); +out_unmap_iv: + if (info->ivsize > 0) + cmh_dma_unmap_single(rctx->iv_dma, info->ivsize, + DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_out: + cmh_dma_unmap_single(rctx->out_dma, req->cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + cmh_dma_unmap_single(rctx->in_dma, req->cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); + return ret; +} + +static int cmh_aes_encrypt(struct skcipher_request *req) +{ + return cmh_aes_crypt(req, AES_OP_ENCRYPT); +} + +static int cmh_aes_decrypt(struct skcipher_request *req) +{ + return cmh_aes_crypt(req, AES_OP_DECRYPT); +} + +/* Registration */ + +static struct cmh_aes_alg_drv aes_drv_algs[ARRAY_SIZE(aes_algs)]; + +/** + * cmh_aes_register() - Register AES-CBC/CTR/ECB/XTS skcipher algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_aes_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + const struct cmh_aes_alg_info *info = &aes_algs[i]; + struct cmh_aes_alg_drv *drv = &aes_drv_algs[i]; + struct skcipher_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->setkey = cmh_aes_setkey; + alg->encrypt = cmh_aes_encrypt; + alg->decrypt = cmh_aes_decrypt; + alg->init = cmh_aes_init_tfm; + alg->exit = cmh_aes_exit_tfm; + alg->min_keysize = info->min_keysize; + alg->max_keysize = info->max_keysize; + alg->ivsize = info->ivsize; + + strscpy(alg->base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + alg->base.cra_blocksize = aes_is_stream_mode(info->aes_mode) + ? 1 : CMH_AES_BLOCK_SIZE; + alg->base.cra_ctxsize = sizeof(struct cmh_aes_tfm_ctx); + alg->base.cra_module = THIS_MODULE; + + ret = crypto_register_skcipher(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_aes: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_aes: registered %s\n", info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_skcipher(&aes_drv_algs[i].alg); + return ret; +} + +/** + * cmh_aes_unregister() - Unregister AES skcipher algorithms from the crypto framework + */ +void cmh_aes_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) { + crypto_unregister_skcipher(&aes_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_aes: unregistered %s\n", aes_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/cmh_aes_aead.c b/drivers/crypto/cmh/cmh_aes_aead.c new file mode 100644 index 000000000000..0b59c5f7d474 --- /dev/null +++ b/drivers/crypto/cmh/cmh_aes_aead.c @@ -0,0 +1,987 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API AES AEAD Driver (GCM/CCM) + * + * Registers AEAD algorithms with the Linux crypto subsystem: + * gcm(aes), ccm(aes) + * + * GCM: AES_CMD_INIT(mode=GCM) + [AAD_FINAL] + AES_CMD_FINAL + FLUSH + * - Standard 12-byte IV (nonce), 16-byte tag + * - AES_CMD_INIT carries aadlen/iolen/taglen + * - AES_CMD_FINAL carries tag DMA for encrypt (produce) / decrypt (verify) + * + * CCM: AES_CMD_CCM_INIT + [AAD_FINAL] + AES_CMD_FINAL + FLUSH + * - Variable nonce (7--13 bytes), variable tag (4--16 bytes) + * - Uses AES_CMD_CCM_INIT (0x0A) with aes_cmd_init struct + * - Nonce passed via IV field, taglen in init + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/aead.h> +#include <crypto/internal/cipher.h> +#include <crypto/scatterwalk.h> +#include <crypto/utils.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "cmh_aes.h" +#include "cmh_vcq.h" +#include "cmh_aes_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +/* + * GCM IV contract: + * + * The AES core requires exactly 16 bytes loaded into its IV register. + * For standard 96-bit nonce GCM, the driver passes: + * + * IV[0..11] = user-supplied 12-byte nonce + * IV[12..15] = 0x00000000 + * + * The hardware internally sets the last 32 bits to the big-endian + * counter value 1 (forming J0 = nonce || 0x00000001) before + * processing AAD. The driver must NOT pre-set the counter. + * + * If the IV format is incorrect, GCM authentication will fail + * (encrypt produces wrong ciphertext/tag, decrypt rejects). + */ +#define AES_GCM_IV_SIZE 12U /* GCM nonce size (standard) */ +#define AES_GCM_HW_IV_SIZE 16U /* HW requires 16-byte IV buffer */ +#define AES_GCM_TAG_SIZE 16U + +/* CCM: callers pass a 16-byte IV in RFC 3610 format: + * iv[0] = L-1, iv[1..14-iv[0]] = nonce, rest = counter (zeroed). + * Nonce length = 14 - iv[0], range 7..13. + */ +#define AES_CCM_IV_SIZE 16U + +enum cmh_aes_aead_type { + CMH_AES_AEAD_GCM, + CMH_AES_AEAD_CCM, +}; + +struct cmh_aes_aead_info { + enum cmh_aes_aead_type type; + u32 aes_mode; /* AES_MODE_GCM or AES_MODE_CCM */ + u32 ivsize; + u32 maxauthsize; + const char *alg_name; + const char *drv_name; +}; + +static const struct cmh_aes_aead_info aes_aead_algs[] = { + { CMH_AES_AEAD_GCM, AES_MODE_GCM, AES_GCM_IV_SIZE, + AES_GCM_TAG_SIZE, "gcm(aes)", "cri-cmh-gcm-aes" }, + { CMH_AES_AEAD_CCM, AES_MODE_CCM, AES_CCM_IV_SIZE, + AES_GCM_TAG_SIZE, "ccm(aes)", "cri-cmh-ccm-aes" }, +}; + +struct cmh_aes_aead_tfm_ctx { + struct cmh_key_ctx key; + u32 authsize; /* tag length set by setauthsize */ + struct crypto_cipher *sw_cipher; /* CCM empty-input fallback */ + struct crypto_aead *fallback; /* CCM authsize=10 fallback */ +}; + +/* Per-request context (lives in aead_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + AAD_FINAL + AES_CMD_FINAL + FLUSH = 5 + */ +#define CMH_AES_AEAD_MAX_PAYLOAD 5 +#define CMH_AES_AEAD_MAX_PACKED (CMH_AES_AEAD_MAX_PAYLOAD * 2) + +struct cmh_aes_aead_reqctx { + dma_addr_t in_dma; + dma_addr_t out_dma; + dma_addr_t iv_dma; + dma_addr_t key_dma; + dma_addr_t aad_dma; + dma_addr_t tag_dma; + u8 *in_buf; + u8 *out_buf; + u8 *iv_buf; + u8 *aad_buf; + u8 *tag_buf; + u32 cryptlen; + u32 assoclen; + u32 authsize; + u32 iv_map_len; + u32 keylen; + bool encrypting; + bool empty_gcm_fallback; + struct vcq_cmd packed[CMH_AES_AEAD_MAX_PACKED]; +}; + +struct cmh_aes_aead_drv { + struct aead_alg alg; + const struct cmh_aes_aead_info *info; +}; + +static const struct cmh_aes_aead_info * +cmh_aes_aead_get_info(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + + return container_of(alg, struct cmh_aes_aead_drv, alg)->info; +} + +/* VCQ Builders -- AEAD-specific */ + +static void vcq_add_aes_aead_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 iv_dma, u32 keylen, u32 ivlen, + u32 mode, u32 op, u32 aadlen, u32 iolen, + u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = iv_dma; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = ivlen; + slot->hwc.aes.cmd_init.mode = mode; + slot->hwc.aes.cmd_init.op = op; + slot->hwc.aes.cmd_init.aadlen = aadlen; + slot->hwc.aes.cmd_init.iolen = iolen; + slot->hwc.aes.cmd_init.taglen = taglen; +} + +static void vcq_add_aes_ccm_init(struct vcq_cmd *slot, u32 core_id, u64 key_ref, + u64 nonce_dma, u32 keylen, u32 noncelen, + u32 op, u32 aadlen, u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_CCM_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = nonce_dma; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = noncelen; + slot->hwc.aes.cmd_init.mode = AES_MODE_CCM; + slot->hwc.aes.cmd_init.op = op; + slot->hwc.aes.cmd_init.aadlen = aadlen; + slot->hwc.aes.cmd_init.iolen = iolen; + slot->hwc.aes.cmd_init.taglen = taglen; +} + +static void vcq_add_aes_aad_final(struct vcq_cmd *slot, u32 core_id, u64 aad_dma, + u32 aadlen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_AAD_FINAL); + slot->hwc.aes.cmd_aad_final.data = aad_dma; + slot->hwc.aes.cmd_aad_final.datalen = aadlen; +} + +static void vcq_add_aes_aead_final(struct vcq_cmd *slot, u32 core_id, u64 input_dma, + u64 output_dma, u64 tag_dma, + u32 iolen, u32 taglen) +{ + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_FINAL); + slot->hwc.aes.cmd_final.input = input_dma; + slot->hwc.aes.cmd_final.output = output_dma; + slot->hwc.aes.cmd_final.tag = tag_dma; + slot->hwc.aes.cmd_final.iolen = iolen; + slot->hwc.aes.cmd_final.taglen = taglen; +} + +/* setkey */ +static int cmh_aes_aead_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + int ret; + + if (keylen != 16 && keylen != 24 && keylen != 32) + return -EINVAL; + + /* Keep SW fallback ciphers in sync for CCM edge cases */ + if (tctx->sw_cipher) { + ret = crypto_cipher_setkey(tctx->sw_cipher, key, keylen); + if (ret) + return ret; + } + if (tctx->fallback) { + ret = crypto_aead_setkey(tctx->fallback, key, keylen); + if (ret) + return ret; + } + + ret = cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES); + + return ret; +} + +static int cmh_aes_aead_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm); + int ret; + + if (info->type == CMH_AES_AEAD_GCM) { + /* GCM: accept 4, 8, 12, 13, 14, 15, 16 per NIST SP 800-38D */ + if (authsize < 4 || authsize > 16 || + (authsize > 4 && authsize < 8) || + (authsize > 8 && authsize < 12)) + return -EINVAL; + } else { + /* CCM: accept all RFC 3610 values {4,6,8,10,12,14,16} */ + if (authsize < 4 || authsize > 16 || (authsize & 1)) + return -EINVAL; + /* Forward to SW fallback for authsize=10 (HW unsupported) */ + if (tctx->fallback) { + ret = crypto_aead_setauthsize(tctx->fallback, + authsize); + if (ret) + return ret; + } + } + + tctx->authsize = authsize; + return 0; +} + +static int cmh_aes_aead_init_tfm(struct crypto_aead *tfm) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm); + + memset(tctx, 0, sizeof(*tctx)); + tctx->authsize = info->maxauthsize; + + if (info->type == CMH_AES_AEAD_CCM) { + struct crypto_aead *fb; + struct crypto_cipher *ci; + + ci = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ci)) + return PTR_ERR(ci); + tctx->sw_cipher = ci; + + fb = crypto_alloc_aead("ccm(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(fb)) { + crypto_free_cipher(ci); + tctx->sw_cipher = NULL; + return PTR_ERR(fb); + } + tctx->fallback = fb; + + /* + * Subreq lives at (rctx + 1). Alignment is guaranteed + * by the crypto framework's __ctx ALIGN mechanism. + */ + crypto_aead_set_reqsize(tfm, + sizeof(struct cmh_aes_aead_reqctx) + + sizeof(struct aead_request) + + crypto_aead_reqsize(fb)); + } else { + crypto_aead_set_reqsize(tfm, + sizeof(struct cmh_aes_aead_reqctx)); + } + + return 0; +} + +static void cmh_aes_aead_exit_tfm(struct crypto_aead *tfm) +{ + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + + if (tctx->fallback) + crypto_free_aead(tctx->fallback); + if (tctx->sw_cipher) + crypto_free_cipher(tctx->sw_cipher); + cmh_key_destroy(&tctx->key); +} + +/* DMA unmap helper */ +static void cmh_aes_aead_unmap_dma(struct cmh_aes_aead_reqctx *rctx) +{ + u32 tag_map_len; + + cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE); + /* + * The empty-GCM fallback maps a full AES block (16 bytes) for the + * ECB output regardless of authsize, so unmap with the mapped size. + */ + tag_map_len = rctx->empty_gcm_fallback ? + AES_GCM_HW_IV_SIZE : rctx->authsize; + cmh_dma_unmap_single(rctx->tag_dma, tag_map_len, + (rctx->encrypting || rctx->empty_gcm_fallback) ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (rctx->cryptlen > 0) { + cmh_dma_unmap_single(rctx->out_dma, rctx->cryptlen, + DMA_FROM_DEVICE); + cmh_dma_unmap_single(rctx->in_dma, rctx->cryptlen, + DMA_TO_DEVICE); + } + if (rctx->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, rctx->assoclen, + DMA_TO_DEVICE); +} + +static void cmh_aes_aead_free_bufs(struct cmh_aes_aead_reqctx *rctx) +{ + kfree(rctx->iv_buf); + rctx->iv_buf = NULL; + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->out_buf); + rctx->out_buf = NULL; + kfree_sensitive(rctx->in_buf); + rctx->in_buf = NULL; + kfree(rctx->aad_buf); + rctx->aad_buf = NULL; +} + +static void cmh_aes_aead_complete(void *data, int error) +{ + struct aead_request *req = data; + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + cmh_aes_aead_unmap_dma(rctx); + + /* + * Map HW error on decrypt to -EBADMSG. The eSW AES core uses a + * single error code (-EIO) for both authentication failures and + * other core errors (e.g. DMA timeout), so we cannot distinguish + * them from the MBX_STATUS alone. In practice the only error + * during a well-formed AEAD decrypt is auth-tag mismatch; a DMA + * timeout would indicate a fatal HW problem where -EBADMSG vs + * -EIO is moot. The kernel crypto API requires -EBADMSG for + * AEAD authentication failures. + */ + if (error == -EIO && !rctx->encrypting) + error = -EBADMSG; + + if (!error) { + /* GCM empty-input decrypt: compare computed tag with expected */ + if (rctx->empty_gcm_fallback && !rctx->encrypting) { + if (crypto_memneq(rctx->tag_buf, rctx->in_buf, + rctx->authsize)) + error = -EBADMSG; + } + if (!error && rctx->cryptlen > 0) + scatterwalk_map_and_copy(rctx->out_buf, req->dst, + req->assoclen, + rctx->cryptlen, 1); + if (!error && rctx->encrypting) + scatterwalk_map_and_copy(rctx->tag_buf, req->dst, + req->assoclen + + rctx->cryptlen, + rctx->authsize, 1); + } + + cmh_aes_aead_free_bufs(rctx); + cmh_complete(&req->base, error); +} + +/* + * GCM empty-input fallback. + * + * When both AAD and plaintext are empty, GCM reduces to: + * tag = E(K, J0) where J0 = nonce || 0x00000001 + * + * The eSW GCM engine rejects this degenerate case, so we compute it + * via a single ECB block encryption of J0. + * + * VCQ: [SYS_CMD_WRITE] + AES_CMD_INIT(ECB) + AES_CMD_FINAL + FLUSH + */ +static int cmh_aes_gcm_empty(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + authsize = tctx->authsize; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = 0; + rctx->assoclen = 0; + rctx->authsize = authsize; + rctx->encrypting = (aes_op == AES_OP_ENCRYPT); + rctx->empty_gcm_fallback = true; + + /* Build J0 = nonce || 0x00000001 in iv_buf */ + rctx->iv_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp); + if (!rctx->iv_buf) + return -ENOMEM; + memcpy(rctx->iv_buf, req->iv, AES_GCM_IV_SIZE); + rctx->iv_buf[15] = 0x01; /* big-endian counter = 1 */ + rctx->iv_map_len = AES_GCM_HW_IV_SIZE; + + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, AES_GCM_HW_IV_SIZE, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Tag buffer -- receives E(K, J0) output */ + rctx->tag_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_iv; + } + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, AES_GCM_HW_IV_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* For decrypt: read expected tag from request for later comparison */ + if (!rctx->encrypting) { + rctx->in_buf = kmalloc(authsize, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, 0, + authsize, 0); + } + + /* Resolve key */ + idx = 0; + rctx->key_dma = tctx->key.raw.dma; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* ECB INIT: single block encryption of J0 */ + vcq_add_aes_aead_init(&cmds[idx++], core_id, key_ref, + 0, keylen, 0, AES_MODE_ECB, AES_OP_ENCRYPT, + 0, AES_GCM_HW_IV_SIZE, 0); + + /* FINAL: J0 in, E(K,J0) out */ + vcq_add_aes_aead_final(&cmds[idx++], core_id, + (u64)rctx->iv_dma, (u64)rctx->tag_dma, + 0, AES_GCM_HW_IV_SIZE, 0); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_AEAD_MAX_PACKED, + target_mbx, + cmh_aes_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_free_in; + + return -EINPROGRESS; + +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, AES_GCM_HW_IV_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_iv: + cmh_dma_unmap_single(rctx->iv_dma, AES_GCM_HW_IV_SIZE, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); + return ret; +} + +/* + * CCM empty-input fallback. + * + * When both AAD and plaintext are empty, CCM reduces to: + * T = E(K, B0) -- CBC-MAC of the single formatting block + * S0 = E(K, A0) -- CTR block zero + * tag = (T XOR S0)[0..authsize-1] + * + * The eSW rejects this degenerate case, so the driver computes it + * synchronously via two crypto_cipher single-block encryptions. + */ +static int cmh_aes_ccm_empty(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + u32 authsize = tctx->authsize; + u8 b0[CMH_AES_BLOCK_SIZE], a0[CMH_AES_BLOCK_SIZE]; + u8 t[CMH_AES_BLOCK_SIZE], s0[CMH_AES_BLOCK_SIZE]; + u8 tag[CMH_AES_BLOCK_SIZE]; + u8 L; + u32 i; + + /* Defense-in-depth: iv[0] = L-1, valid L is 2..8 per RFC 3610 S2.1 */ + if (WARN_ON_ONCE(req->iv[0] < 1 || req->iv[0] > 7)) + return -EINVAL; + + L = req->iv[0] + 1; + + if (tctx->key.mode != CMH_KEY_RAW) + return -EOPNOTSUPP; + + /* B0: flags || nonce || Q(=0). Adata=0, t=authsize, q=L. */ + memset(b0, 0, CMH_AES_BLOCK_SIZE); + b0[0] = (u8)(8 * ((authsize - 2) / 2) + (L - 1)); + memcpy(&b0[1], &req->iv[1], 15 - L); + + /* A0: (L-1) || nonce || counter(=0) */ + memset(a0, 0, CMH_AES_BLOCK_SIZE); + a0[0] = (u8)(L - 1); + memcpy(&a0[1], &req->iv[1], 15 - L); + + crypto_cipher_encrypt_one(tctx->sw_cipher, t, b0); + crypto_cipher_encrypt_one(tctx->sw_cipher, s0, a0); + + for (i = 0; i < authsize; i++) + tag[i] = t[i] ^ s0[i]; + + if (aes_op == AES_OP_ENCRYPT) { + scatterwalk_map_and_copy(tag, req->dst, + req->assoclen, authsize, 1); + } else { + u8 expected[CMH_AES_BLOCK_SIZE]; + + scatterwalk_map_and_copy(expected, req->src, + req->assoclen, authsize, 0); + if (crypto_memneq(tag, expected, authsize)) + return -EBADMSG; + } + + return 0; +} + +/* + * CCM authsize=10 fallback. + * + * The eSW AES CCM core does not support authsize=10 (valid per RFC 3610). + * Forward the entire request to the generic CCM implementation. + */ +static void cmh_aes_ccm_fb_done(void *data, int err) +{ + struct aead_request *req = data; + + cmh_complete(&req->base, err); +} + +static int cmh_aes_ccm_fallback(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + struct aead_request *subreq = (void *)(rctx + 1); + + aead_request_set_tfm(subreq, tctx->fallback); + aead_request_set_callback(subreq, req->base.flags, + cmh_aes_ccm_fb_done, req); + aead_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_ad(subreq, req->assoclen); + + return (aes_op == AES_OP_ENCRYPT) ? + crypto_aead_encrypt(subreq) : crypto_aead_decrypt(subreq); +} + +/* + * Core AEAD encrypt/decrypt -- async path. + * + * Encrypt: plaintext -> ciphertext + tag appended + * Decrypt: ciphertext + tag -> plaintext (tag verified by eSW) + * + * VCQ: [SYS_CMD_WRITE] + INIT/CCM_INIT + [AAD_FINAL] + FINAL + FLUSH + */ +static int cmh_aes_aead_crypt(struct aead_request *req, u32 aes_op) +{ + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cmh_aes_aead_tfm_ctx *tctx = crypto_aead_ctx(tfm); + const struct cmh_aes_aead_info *info = cmh_aes_aead_get_info(tfm); + struct cmh_aes_aead_reqctx *rctx = aead_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_AEAD_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen, authsize, cryptlen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) + return -ENOKEY; + + authsize = tctx->authsize; + + if (aes_op == AES_OP_ENCRYPT) { + cryptlen = req->cryptlen; + } else { + if (req->cryptlen < authsize) + return -EINVAL; + cryptlen = req->cryptlen - authsize; + } + + /* + * Validate CCM IV format early -- the empty-input fallback and + * nonce extraction both depend on iv[0] being in range [1,7]. + */ + if (info->type == CMH_AES_AEAD_CCM) { + if (req->iv[0] < 1 || req->iv[0] > 7) + return -EINVAL; + } + + /* + * The CMH eSW rejects GCM/CCM when both aadlen and iolen are zero. + * For GCM, the tag is simply E(K, J0) -- handle with ECB fallback. + * For CCM, compute tag = E(K,B0) XOR E(K,A0) in software. + */ + if (cryptlen == 0 && req->assoclen == 0) { + if (info->type == CMH_AES_AEAD_GCM) + return cmh_aes_gcm_empty(req, aes_op); + return cmh_aes_ccm_empty(req, aes_op); + } + + /* + * HW does not support authsize=10 for CCM. Forward the entire + * request to the generic CCM implementation. + */ + if (info->type == CMH_AES_AEAD_CCM && authsize == 10) + return cmh_aes_ccm_fallback(req, aes_op); + + /* + * HW uses a proprietary LLI scatter-gather format that is + * incompatible with struct scatterlist, so the payload is + * linearised into contiguous buffers for DMA. Cap total + * size to prevent excessive memory consumption. + */ + if ((u64)cryptlen + req->assoclen > SZ_1M) + return -EINVAL; + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + memset(rctx, 0, sizeof(*rctx)); + rctx->cryptlen = cryptlen; + rctx->assoclen = req->assoclen; + rctx->authsize = authsize; + rctx->encrypting = (aes_op == AES_OP_ENCRYPT); + + /* Linearise AAD */ + if (req->assoclen > 0) { + rctx->aad_buf = kmalloc(req->assoclen, gfp); + if (!rctx->aad_buf) + return -ENOMEM; + scatterwalk_map_and_copy(rctx->aad_buf, req->src, + 0, req->assoclen, 0); + rctx->aad_dma = cmh_dma_map_single(rctx->aad_buf, + req->assoclen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->aad_dma)) { + ret = -ENOMEM; + goto out_free_aad; + } + } + + /* Linearise input */ + if (cryptlen > 0) { + rctx->in_buf = kmalloc(cryptlen, gfp); + if (!rctx->in_buf) { + ret = -ENOMEM; + goto out_unmap_aad; + } + scatterwalk_map_and_copy(rctx->in_buf, req->src, + req->assoclen, cryptlen, 0); + rctx->in_dma = cmh_dma_map_single(rctx->in_buf, cryptlen, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_free_in; + } + } + + /* Allocate output buffer */ + if (cryptlen > 0) { + rctx->out_buf = kmalloc(cryptlen, gfp); + if (!rctx->out_buf) { + ret = -ENOMEM; + goto out_unmap_in; + } + rctx->out_dma = cmh_dma_map_single(rctx->out_buf, cryptlen, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->out_dma)) { + ret = -ENOMEM; + goto out_free_out; + } + } + + /* Tag buffer */ + rctx->tag_buf = kmalloc(authsize, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_unmap_out; + } + + if (!rctx->encrypting) { + scatterwalk_map_and_copy(rctx->tag_buf, req->src, + req->assoclen + cryptlen, + authsize, 0); + } else { + memset(rctx->tag_buf, 0, authsize); + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, authsize, + rctx->encrypting ? + DMA_FROM_DEVICE : DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map IV/nonce */ + if (info->type == CMH_AES_AEAD_GCM) { + rctx->iv_buf = kzalloc(AES_GCM_HW_IV_SIZE, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + memcpy(rctx->iv_buf, req->iv, AES_GCM_IV_SIZE); + rctx->iv_map_len = AES_GCM_HW_IV_SIZE; + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, + rctx->iv_map_len, + DMA_TO_DEVICE); + } else { + u32 noncelen; + + if (req->iv[0] < 1 || req->iv[0] > 7) { + ret = -EINVAL; + goto out_unmap_tag; + } + noncelen = 14 - req->iv[0]; + + rctx->iv_buf = kmemdup(req->iv + 1, noncelen, gfp); + if (!rctx->iv_buf) { + ret = -ENOMEM; + goto out_unmap_tag; + } + rctx->iv_map_len = noncelen; + rctx->iv_dma = cmh_dma_map_single(rctx->iv_buf, + rctx->iv_map_len, + DMA_TO_DEVICE); + } + if (cmh_dma_map_error(rctx->iv_dma)) { + ret = -ENOMEM; + goto out_free_iv; + } + + /* Resolve key reference */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* Build INIT command */ + if (info->type == CMH_AES_AEAD_CCM) { + vcq_add_aes_ccm_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, + rctx->iv_map_len, aes_op, + req->assoclen, cryptlen, authsize); + } else { + vcq_add_aes_aead_init(&cmds[idx++], core_id, key_ref, + (u64)rctx->iv_dma, keylen, + AES_GCM_HW_IV_SIZE, info->aes_mode, + aes_op, req->assoclen, cryptlen, + authsize); + } + + if (req->assoclen > 0) + vcq_add_aes_aad_final(&cmds[idx++], core_id, + (u64)rctx->aad_dma, req->assoclen); + + vcq_add_aes_aead_final(&cmds[idx++], core_id, + cryptlen > 0 ? (u64)rctx->in_dma : 0, + cryptlen > 0 ? (u64)rctx->out_dma : 0, + (u64)rctx->tag_dma, cryptlen, authsize); + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_AEAD_MAX_PACKED, + target_mbx, + cmh_aes_aead_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + cmh_dma_unmap_single(rctx->iv_dma, rctx->iv_map_len, DMA_TO_DEVICE); +out_free_iv: + kfree(rctx->iv_buf); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, authsize, + rctx->encrypting ? DMA_FROM_DEVICE : + DMA_TO_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_unmap_out: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->out_dma, cryptlen, DMA_FROM_DEVICE); +out_free_out: + kfree_sensitive(rctx->out_buf); +out_unmap_in: + if (cryptlen > 0) + cmh_dma_unmap_single(rctx->in_dma, cryptlen, DMA_TO_DEVICE); +out_free_in: + kfree_sensitive(rctx->in_buf); +out_unmap_aad: + if (req->assoclen > 0) + cmh_dma_unmap_single(rctx->aad_dma, req->assoclen, + DMA_TO_DEVICE); +out_free_aad: + kfree(rctx->aad_buf); + return ret; +} + +static int cmh_aes_aead_encrypt(struct aead_request *req) +{ + return cmh_aes_aead_crypt(req, AES_OP_ENCRYPT); +} + +static int cmh_aes_aead_decrypt(struct aead_request *req) +{ + return cmh_aes_aead_crypt(req, AES_OP_DECRYPT); +} + +/* Registration */ + +static struct cmh_aes_aead_drv aes_aead_drv_algs[ARRAY_SIZE(aes_aead_algs)]; + +/** + * cmh_aes_aead_register() - Register AES-GCM/CCM AEAD algorithms with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_aes_aead_register(void) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) { + const struct cmh_aes_aead_info *info = &aes_aead_algs[i]; + struct cmh_aes_aead_drv *drv = &aes_aead_drv_algs[i]; + struct aead_alg *alg = &drv->alg; + + drv->info = info; + + memset(alg, 0, sizeof(*alg)); + + alg->setkey = cmh_aes_aead_setkey; + alg->setauthsize = cmh_aes_aead_setauthsize; + alg->encrypt = cmh_aes_aead_encrypt; + alg->decrypt = cmh_aes_aead_decrypt; + alg->init = cmh_aes_aead_init_tfm; + alg->exit = cmh_aes_aead_exit_tfm; + alg->ivsize = info->ivsize; + alg->maxauthsize = info->maxauthsize; + + strscpy(alg->base.cra_name, info->alg_name, + CRYPTO_MAX_ALG_NAME); + strscpy(alg->base.cra_driver_name, info->drv_name, + CRYPTO_MAX_ALG_NAME); + alg->base.cra_priority = 300; + alg->base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC; + if (info->type == CMH_AES_AEAD_CCM) { + alg->base.cra_flags |= CRYPTO_ALG_NEED_FALLBACK; + /* + * Bump priority above 300 so we beat the generic + * ccm_base template instance. That template inherits + * priority (ctr + cbcmac) / 2 = 300 when both + * constituents are at 300, and list ordering would + * otherwise let it shadow our driver. + */ + alg->base.cra_priority = 301; + } + alg->base.cra_blocksize = 1; + alg->base.cra_ctxsize = sizeof(struct cmh_aes_aead_tfm_ctx); + alg->base.cra_module = THIS_MODULE; + + ret = crypto_register_aead(alg); + if (ret) { + dev_err(cmh_dev(), "cmh_aes_aead: failed to register %s (rc=%d)\n", + info->alg_name, ret); + goto err_unregister; + } + + dev_dbg(cmh_dev(), "cmh_aes_aead: registered %s\n", info->alg_name); + } + + return 0; + +err_unregister: + while (i--) + crypto_unregister_aead(&aes_aead_drv_algs[i].alg); + return ret; +} + +/** + * cmh_aes_aead_unregister() - Unregister AES AEAD algorithms from the crypto framework + */ +void cmh_aes_aead_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(aes_aead_algs); i++) { + crypto_unregister_aead(&aes_aead_drv_algs[i].alg); + dev_dbg(cmh_dev(), "cmh_aes_aead: unregistered %s\n", + aes_aead_algs[i].alg_name); + } +} diff --git a/drivers/crypto/cmh/cmh_aes_cmac.c b/drivers/crypto/cmh/cmh_aes_cmac.c new file mode 100644 index 000000000000..a711c575398d --- /dev/null +++ b/drivers/crypto/cmh/cmh_aes_cmac.c @@ -0,0 +1,537 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- Kernel Crypto API AES-CMAC (ahash) Driver + * + * Registers cmac(aes) as an ahash algorithm. + * + * CMAC produces a 16-byte tag (MAC) from a key and message. + * VCQ sequence: [SYS_CMD_WRITE] + AES_CMD_INIT(CMAC) + + * AES_CMD_AAD_FINAL_AUTH + FLUSH + * + * The ahash interface accumulates data in a kernel buffer via .update(), + * then .final() builds and submits the VCQ asynchronously. + */ + +#include <linux/module.h> +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <crypto/internal/hash.h> +#include <crypto/scatterwalk.h> +#include <linux/slab.h> +#include <linux/string.h> + +#include "cmh_aes.h" +#include "cmh_vcq.h" +#include "cmh_aes_abi.h" +#include "cmh_sys_abi.h" +#include "cmh_sys.h" +#include "cmh_txn.h" +#include "cmh_dma.h" +#include "cmh_key.h" + +#define AES_CMAC_DIGEST_SIZE 16U +#define AES_CMAC_BLOCK_SIZE 16U + +/* + * Maximum accumulated data for CMAC -- driver-imposed, not HW. + * + * The AES core does not expose external save/restore VCQ commands, + * so the driver must accumulate all data in kernel memory via + * .update() and submit it atomically in .final(). This cap limits + * the per-request kernel allocation. + */ +#define AES_CMAC_MAX_DATA (64 * 1024) + +/* Per-transform context */ +struct cmh_aes_cmac_tfm_ctx { + struct cmh_key_ctx key; + spinlock_t chunk_lock; /* protects all_chunks */ + struct list_head all_chunks; /* orphan-safe chunk tracking */ +}; + +/* One chunk per .update() call -- data is embedded via flexible array */ +struct cmh_aes_cmac_chunk { + struct list_head list; + struct list_head tfm_node; /* per-tfm orphan tracking */ + u32 len; + u8 data[]; +}; + +/* Per-request context (lives in ahash_request::__ctx) */ + +/* + * Maximum payload commands: + * [SYS_CMD_WRITE] + AES_CMD_INIT + AES_CMD_AAD_FINAL_AUTH + FLUSH = 4 + */ +#define CMH_AES_CMAC_MAX_PAYLOAD 4 +#define CMH_AES_CMAC_MAX_PACKED (CMH_AES_CMAC_MAX_PAYLOAD * 2) + +struct cmh_aes_cmac_reqctx { + struct list_head chunks; + u32 total_len; + u8 *buf; /* linearised in final() for DMA */ + /* DMA state for async final */ + dma_addr_t key_dma; + dma_addr_t in_dma; + dma_addr_t tag_dma; + u8 *tag_buf; + u32 keylen; + struct vcq_cmd packed[CMH_AES_CMAC_MAX_PACKED]; +}; + +/* Flat state for export/import -- holds accumulated input data only */ +struct cmh_aes_cmac_export_state { + u32 total_len; + u8 data[]; +}; + +/* + * Flat state buffer for export/import. The CMH AES core does not + * support save/restore of intermediate CMAC state, so this driver + * accumulates input in SW and serialises the buffer on export. + * + * PAGE_SIZE (4096) caps the exportable accumulated-data window. + * Full-range export is not feasible because the crypto subsystem + * pre-allocates statesize bytes per request. Export returns -EINVAL + * if the caller has accumulated more than CMH_AES_CMAC_EXPORT_MAX. + */ +#define CMH_AES_CMAC_STATE_SIZE 4096 +#define CMH_AES_CMAC_EXPORT_MAX \ + (CMH_AES_CMAC_STATE_SIZE - sizeof(struct cmh_aes_cmac_export_state)) + +/* + * Export/import: not supported. + * + * The AES core lacks external save/restore VCQ commands, so there is + * no way to checkpoint intermediate CMAC state to host memory. + * Pending eSW ABI extension to add save/restore for the AES core. + */ + +static int cmh_aes_cmac_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen) +{ + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + if (keylen != 16 && keylen != 24 && keylen != 32) + return -EINVAL; + + return cmh_key_setkey_raw(&tctx->key, key, keylen, CORE_ID_AES); +} + +static void cmh_aes_cmac_free_chunks(struct cmh_aes_cmac_reqctx *rctx, + struct cmh_aes_cmac_tfm_ctx *tctx) +{ + struct cmh_aes_cmac_chunk *c, *tmp; + + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &rctx->chunks, list) { + list_del(&c->list); + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len = 0; +} + +static int cmh_aes_cmac_init(struct ahash_request *req) +{ + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + return 0; +} + +static int cmh_aes_cmac_update(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_aes_cmac_chunk *chunk; + gfp_t gfp; + int ret; + + if (!req->nbytes) + return 0; + + if (req->nbytes > AES_CMAC_MAX_DATA - rctx->total_len) { + ret = -EINVAL; + goto err_free_chunks; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + chunk = kmalloc(sizeof(*chunk) + req->nbytes, gfp); + if (!chunk) { + ret = -ENOMEM; + goto err_free_chunks; + } + + chunk->len = req->nbytes; + if (req->base.flags & CRYPTO_AHASH_REQ_VIRT) + memcpy(chunk->data, req->svirt, req->nbytes); + else + scatterwalk_map_and_copy(chunk->data, req->src, + 0, req->nbytes, 0); + + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len += req->nbytes; + return 0; + +err_free_chunks: + /* + * Terminal error -- free all previously accumulated chunks. + * callers may not call .final() on error, so they would leak. + */ + cmh_aes_cmac_free_chunks(rctx, tctx); + return ret; +} + +static void cmh_aes_cmac_complete(void *data, int error) +{ + struct ahash_request *req = data; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + + if (error == -EINPROGRESS) { + cmh_complete(&req->base, error); + return; + } + + /* Unmap DMA */ + if (rctx->total_len > 0) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); + cmh_dma_unmap_single(rctx->tag_dma, AES_CMAC_DIGEST_SIZE, + DMA_FROM_DEVICE); + + if (!error) + memcpy(req->result, rctx->tag_buf, AES_CMAC_DIGEST_SIZE); + + kfree(rctx->tag_buf); + rctx->tag_buf = NULL; + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + cmh_aes_cmac_free_chunks(rctx, tctx); + cmh_complete(&req->base, error); +} + +static int cmh_aes_cmac_final(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + struct vcq_cmd cmds[CMH_AES_CMAC_MAX_PAYLOAD]; + u64 key_ref; + u32 keylen; + struct core_dispatch d; + s32 target_mbx; + u32 core_id; + u32 idx; + int ret; + gfp_t gfp; + + if (tctx->key.mode == CMH_KEY_NONE) { + ret = -ENOKEY; + goto out_free_buf; + } + + gfp = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? + GFP_KERNEL : GFP_ATOMIC; + + /* Linearise accumulated chunks into a contiguous buffer for DMA */ + if (rctx->total_len > 0) { + struct cmh_aes_cmac_chunk *c; + u32 off = 0; + + rctx->buf = kmalloc(rctx->total_len, gfp); + if (!rctx->buf) { + ret = -ENOMEM; + goto out_free_chunks; + } + list_for_each_entry(c, &rctx->chunks, list) { + memcpy(rctx->buf + off, c->data, c->len); + off += c->len; + } + } + + /* Tag output buffer */ + rctx->tag_buf = kzalloc(AES_CMAC_DIGEST_SIZE, gfp); + if (!rctx->tag_buf) { + ret = -ENOMEM; + goto out_free_buf; + } + + rctx->tag_dma = cmh_dma_map_single(rctx->tag_buf, + AES_CMAC_DIGEST_SIZE, + DMA_FROM_DEVICE); + if (cmh_dma_map_error(rctx->tag_dma)) { + ret = -ENOMEM; + goto out_free_tag; + } + + /* Map input data (may be zero-length for empty CMAC) */ + if (rctx->total_len > 0) { + rctx->in_dma = cmh_dma_map_single(rctx->buf, rctx->total_len, + DMA_TO_DEVICE); + if (cmh_dma_map_error(rctx->in_dma)) { + ret = -ENOMEM; + goto out_unmap_tag; + } + } + + /* Resolve key */ + idx = 0; + + rctx->key_dma = tctx->key.raw.dma; + rctx->keylen = tctx->key.raw.len; + vcq_add_sys_write(&cmds[idx++], SYS_REF_TEMP, + (u64)rctx->key_dma, SYS_REF_NONE, + tctx->key.raw.len, + tctx->key.raw.sys_type); + key_ref = SYS_REF_TEMP; + keylen = tctx->key.raw.len; + d = cmh_core_select_instance(CMH_CORE_AES); + target_mbx = d.mbx_idx; + core_id = d.core_id; + + /* + * INIT: mode=CMAC, op=ENCRYPT (CMAC always "encrypts") + * CMAC data goes through the AAD path: + * aadlen = total data length, iolen = 0 + */ + { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_INIT); + slot->hwc.aes.cmd_init.key = key_ref; + slot->hwc.aes.cmd_init.iv = 0; + slot->hwc.aes.cmd_init.keylen = keylen; + slot->hwc.aes.cmd_init.ivlen = 0; + slot->hwc.aes.cmd_init.mode = AES_MODE_CMAC; + slot->hwc.aes.cmd_init.op = AES_OP_ENCRYPT; + slot->hwc.aes.cmd_init.aadlen = rctx->total_len; + slot->hwc.aes.cmd_init.iolen = 0; + slot->hwc.aes.cmd_init.taglen = AES_CMAC_DIGEST_SIZE; + } + + /* AAD_FINAL_AUTH: final AAD + tag extraction in one atomic step */ + { + struct vcq_cmd *slot = &cmds[idx++]; + + memset(slot, 0, sizeof(*slot)); + slot->magic = VCQ_CMD_MAGIC; + slot->id = VCQ_CMD_ID(core_id, 0, 1, AES_CMD_AAD_FINAL_AUTH); + slot->hwc.aes.cmd_aad_final_auth.data = + rctx->total_len > 0 ? (u64)rctx->in_dma : 0; + slot->hwc.aes.cmd_aad_final_auth.datalen = rctx->total_len; + slot->hwc.aes.cmd_aad_final_auth.tag = (u64)rctx->tag_dma; + slot->hwc.aes.cmd_aad_final_auth.taglen = AES_CMAC_DIGEST_SIZE; + } + + vcq_add_flush(&cmds[idx++], core_id); + + ret = cmh_vcq_pack_and_submit_async(cmds, idx, rctx->packed, + CMH_AES_CMAC_MAX_PACKED, + target_mbx, + cmh_aes_cmac_complete, req, + !!(req->base.flags & + CRYPTO_TFM_REQ_MAY_BACKLOG), + cmh_tm_async_timeout_jiffies()); + /* -EBUSY = backlogged; ownership transferred to callback. */ + if (ret == -EBUSY) + return -EBUSY; + if (ret) + goto out_cleanup_all; + + return -EINPROGRESS; + +out_cleanup_all: + if (rctx->total_len > 0 && !cmh_dma_map_error(rctx->in_dma)) + cmh_dma_unmap_single(rctx->in_dma, rctx->total_len, + DMA_TO_DEVICE); +out_unmap_tag: + cmh_dma_unmap_single(rctx->tag_dma, AES_CMAC_DIGEST_SIZE, + DMA_FROM_DEVICE); +out_free_tag: + kfree(rctx->tag_buf); +out_free_buf: +out_free_chunks: + cmh_aes_cmac_free_chunks(rctx, tctx); + kfree_sensitive(rctx->buf); + rctx->buf = NULL; + rctx->total_len = 0; + return ret; +} + +/* + * ahash .export()/.import(): serialize/deserialize the software + * accumulation buffer. No HW state is involved -- the AES core + * does not support save/restore, but we only export the input queue. + */ + +static int cmh_aes_cmac_export(struct ahash_request *req, void *out) +{ + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + struct cmh_aes_cmac_export_state *state = out; + struct cmh_aes_cmac_chunk *chunk; + u32 offset = 0; + + if (rctx->total_len > CMH_AES_CMAC_EXPORT_MAX) + return -ENOSPC; + + state->total_len = rctx->total_len; + list_for_each_entry(chunk, &rctx->chunks, list) { + memcpy(state->data + offset, chunk->data, chunk->len); + offset += chunk->len; + } + return 0; +} + +static int cmh_aes_cmac_import(struct ahash_request *req, const void *in) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_reqctx *rctx = ahash_request_ctx(req); + const struct cmh_aes_cmac_export_state *state = in; + struct cmh_aes_cmac_chunk *chunk; + + /* + * Do NOT call free_chunks() here: the crypto API does not + * guarantee the request context is in a valid state before + * import(), so the list pointers may be stale or invalid. + * Re-initialize from scratch instead. Any pre-existing chunks + * are tracked on tctx->all_chunks and freed in exit_tfm. + */ + memset(rctx, 0, sizeof(*rctx)); + INIT_LIST_HEAD(&rctx->chunks); + + if (state->total_len > CMH_AES_CMAC_EXPORT_MAX) + return -EINVAL; + + if (state->total_len) { + chunk = kmalloc(sizeof(*chunk) + state->total_len, GFP_KERNEL); + if (!chunk) + return -ENOMEM; + chunk->len = state->total_len; + memcpy(chunk->data, state->data, state->total_len); + list_add_tail(&chunk->list, &rctx->chunks); + spin_lock_bh(&tctx->chunk_lock); + list_add_tail(&chunk->tfm_node, &tctx->all_chunks); + spin_unlock_bh(&tctx->chunk_lock); + rctx->total_len = state->total_len; + } + return 0; +} + +static int cmh_aes_cmac_finup(struct ahash_request *req) +{ + int err; + + err = cmh_aes_cmac_update(req); + if (err) + return err; + return cmh_aes_cmac_final(req); +} + +static int cmh_aes_cmac_digest(struct ahash_request *req) +{ + int err; + + err = cmh_aes_cmac_init(req); + if (err) + return err; + return cmh_aes_cmac_finup(req); +} + +static int cmh_aes_cmac_init_tfm(struct crypto_ahash *tfm) +{ + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + + memset(tctx, 0, sizeof(*tctx)); + spin_lock_init(&tctx->chunk_lock); + INIT_LIST_HEAD(&tctx->all_chunks); + crypto_ahash_set_reqsize(tfm, sizeof(struct cmh_aes_cmac_reqctx)); + return 0; +} + +static void cmh_aes_cmac_exit_tfm(struct crypto_ahash *tfm) +{ + struct cmh_aes_cmac_tfm_ctx *tctx = crypto_ahash_ctx(tfm); + struct cmh_aes_cmac_chunk *c, *tmp; + + /* Free any orphaned chunks (e.g. testmgr export/reimport poison) */ + spin_lock_bh(&tctx->chunk_lock); + list_for_each_entry_safe(c, tmp, &tctx->all_chunks, tfm_node) { + list_del(&c->tfm_node); + kfree_sensitive(c); + } + spin_unlock_bh(&tctx->chunk_lock); + + cmh_key_destroy(&tctx->key); +} + +static struct ahash_alg cmh_aes_cmac_alg = { + .init = cmh_aes_cmac_init, + .update = cmh_aes_cmac_update, + .final = cmh_aes_cmac_final, + .finup = cmh_aes_cmac_finup, + .digest = cmh_aes_cmac_digest, + .export = cmh_aes_cmac_export, + .import = cmh_aes_cmac_import, + .setkey = cmh_aes_cmac_setkey, + .init_tfm = cmh_aes_cmac_init_tfm, + .exit_tfm = cmh_aes_cmac_exit_tfm, + .halg = { + .digestsize = AES_CMAC_DIGEST_SIZE, + .statesize = CMH_AES_CMAC_STATE_SIZE, + .base = { + .cra_name = "cmac(aes)", + .cra_driver_name = "cri-cmh-cmac-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NO_FALLBACK | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_REQ_VIRT, + .cra_blocksize = AES_CMAC_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct cmh_aes_cmac_tfm_ctx), + .cra_module = THIS_MODULE, + }, + }, +}; + +/** + * cmh_aes_cmac_register() - Register AES-CMAC hash algorithm with the crypto framework + * + * Return: 0 on success, negative errno on failure. + */ +int cmh_aes_cmac_register(void) +{ + int ret; + + ret = crypto_register_ahash(&cmh_aes_cmac_alg); + if (ret) + dev_err(cmh_dev(), "cmh_aes_cmac: failed to register cmac(aes) (rc=%d)\n", + ret); + else + dev_dbg(cmh_dev(), "cmh_aes_cmac: registered cmac(aes)\n"); + + return ret; +} + +/** + * cmh_aes_cmac_unregister() - Unregister AES-CMAC hash algorithm from the crypto framework + */ +void cmh_aes_cmac_unregister(void) +{ + crypto_unregister_ahash(&cmh_aes_cmac_alg); + dev_dbg(cmh_dev(), "cmh_aes_cmac: unregistered cmac(aes)\n"); +} diff --git a/drivers/crypto/cmh/cmh_main.c b/drivers/crypto/cmh/cmh_main.c index 56541e0d4219..1edd8d14c666 100644 --- a/drivers/crypto/cmh/cmh_main.c +++ b/drivers/crypto/cmh/cmh_main.c @@ -34,6 +34,7 @@ #include "cmh_cshake.h" #include "cmh_kmac.h" #include "cmh_sm3.h" +#include "cmh_aes.h" #include "cmh_mgmt.h" #include "cmh_registers.h" #include "cmh_debugfs.h" @@ -221,6 +222,21 @@ static int cmh_probe(struct platform_device *pdev) if (ret) goto err_sm3_register; + /* Register AES skcipher algorithms */ + ret = cmh_aes_register(); + if (ret) + goto err_aes_register; + + /* Register AES AEAD algorithms (GCM, CCM) */ + ret = cmh_aes_aead_register(); + if (ret) + goto err_aes_aead_register; + + /* Register AES CMAC algorithm */ + ret = cmh_aes_cmac_register(); + if (ret) + goto err_aes_cmac_register; + /* Register key management device (/dev/cmh_mgmt) */ ret = cmh_mgmt_register(); if (ret) @@ -233,6 +249,12 @@ static int cmh_probe(struct platform_device *pdev) return 0; err_mgmt_register: + cmh_aes_cmac_unregister(); +err_aes_cmac_register: + cmh_aes_aead_unregister(); +err_aes_aead_register: + cmh_aes_unregister(); +err_aes_register: cmh_sm3_unregister(); err_sm3_register: cmh_kmac_unregister(); @@ -269,6 +291,9 @@ static void cmh_remove(struct platform_device *pdev) cfg = &dev->config; cmh_mgmt_unregister(); + cmh_aes_cmac_unregister(); + cmh_aes_aead_unregister(); + cmh_aes_unregister(); cmh_sm3_unregister(); cmh_kmac_unregister(); cmh_cshake_unregister(); diff --git a/drivers/crypto/cmh/include/cmh_aes.h b/drivers/crypto/cmh/include/cmh_aes.h new file mode 100644 index 000000000000..591afaa36f85 --- /dev/null +++ b/drivers/crypto/cmh/include/cmh_aes.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2026 Cryptography Research, Inc. (CRI). + * CMH LKM -- AES Crypto API Drivers + * + * Registers AES algorithms with the Linux crypto subsystem: + * skcipher: ecb/cbc/ctr/cfb/xts(aes) + * aead: gcm/ccm(aes) + * shash: cmac(aes) + */ + +#ifndef CMH_AES_H +#define CMH_AES_H + +int cmh_aes_register(void); +void cmh_aes_unregister(void); + +int cmh_aes_aead_register(void); +void cmh_aes_aead_unregister(void); + +int cmh_aes_cmac_register(void); +void cmh_aes_cmac_unregister(void); + +#endif /* CMH_AES_H */ -- 2.43.7 ** This message and any attachments are for the sole use of the intended recipient(s). It may contain information that is confidential and privileged. If you are not the intended recipient of this message, you are prohibited from printing, copying, forwarding or saving it. Please delete the message and attachments and notify the sender immediately. ** Rambus Inc.<http://www.rambus.com>
