On Tue, 28 Jul 2020 at 10:19, Herbert Xu <[email protected]> wrote:
>
> As it stands chacha cannot do chaining. That is, it has to handle
> each request as a whole. This patch adds support for chaining when
> the CRYPTO_TFM_REQ_MORE flag is set.
>
> Signed-off-by: Herbert Xu <[email protected]>
Only state[12] needs to be preserved, since it contains the block
counter. Everything else in the state can be derived from the IV.
So by doing the init unconditionally, and overriding state[12] to the
captured value (if it exists), we can get rid of the redundant copy of
state, which also avoids inconsistencies if IV and state are out of
sync.
> ---
>
> arch/arm64/crypto/chacha-neon-glue.c | 43
> ++++++++++++++++++++++-------------
> 1 file changed, 28 insertions(+), 15 deletions(-)
>
> diff --git a/arch/arm64/crypto/chacha-neon-glue.c
> b/arch/arm64/crypto/chacha-neon-glue.c
> index af2bbca38e70f..d82c574ddcc00 100644
> --- a/arch/arm64/crypto/chacha-neon-glue.c
> +++ b/arch/arm64/crypto/chacha-neon-glue.c
> @@ -19,10 +19,8 @@
> * (at your option) any later version.
> */
>
> -#include <crypto/algapi.h>
> #include <crypto/internal/chacha.h>
> #include <crypto/internal/simd.h>
> -#include <crypto/internal/skcipher.h>
> #include <linux/jump_label.h>
> #include <linux/kernel.h>
> #include <linux/module.h>
> @@ -101,16 +99,16 @@ void chacha_crypt_arch(u32 *state, u8 *dst, const u8
> *src, unsigned int bytes,
> }
> EXPORT_SYMBOL(chacha_crypt_arch);
>
> -static int chacha_neon_stream_xor(struct skcipher_request *req,
> - const struct chacha_ctx *ctx, const u8 *iv)
> +static int chacha_neon_stream_xor(struct skcipher_request *req, int nrounds)
> {
> + struct chacha_reqctx *rctx = skcipher_request_ctx(req);
> struct skcipher_walk walk;
> - u32 state[16];
> + u32 *state = rctx->state;
> int err;
>
> - err = skcipher_walk_virt(&walk, req, false);
> + rctx->init = req->base.flags & CRYPTO_TFM_REQ_MORE;
>
> - chacha_init_generic(state, ctx->key, iv);
> + err = skcipher_walk_virt(&walk, req, false);
>
> while (walk.nbytes > 0) {
> unsigned int nbytes = walk.nbytes;
> @@ -122,11 +120,11 @@ static int chacha_neon_stream_xor(struct
> skcipher_request *req,
> !crypto_simd_usable()) {
> chacha_crypt_generic(state, walk.dst.virt.addr,
> walk.src.virt.addr, nbytes,
> - ctx->nrounds);
> + nrounds);
> } else {
> kernel_neon_begin();
> chacha_doneon(state, walk.dst.virt.addr,
> - walk.src.virt.addr, nbytes,
> ctx->nrounds);
> + walk.src.virt.addr, nbytes, nrounds);
> kernel_neon_end();
> }
> err = skcipher_walk_done(&walk, walk.nbytes - nbytes);
> @@ -138,26 +136,38 @@ static int chacha_neon_stream_xor(struct
> skcipher_request *req,
> static int chacha_neon(struct skcipher_request *req)
> {
> struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> + struct chacha_reqctx *rctx = skcipher_request_ctx(req);
> struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
>
> - return chacha_neon_stream_xor(req, ctx, req->iv);
> + if (!rctx->init)
> + chacha_init_generic(rctx->state, ctx->key, req->iv);
> +
> + return chacha_neon_stream_xor(req, ctx->nrounds);
> }
>
> static int xchacha_neon(struct skcipher_request *req)
> {
> struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
> + struct chacha_reqctx *rctx = skcipher_request_ctx(req);
> struct chacha_ctx *ctx = crypto_skcipher_ctx(tfm);
> - struct chacha_ctx subctx;
> - u32 state[16];
> + int nrounds = ctx->nrounds;
> + u32 *state = rctx->state;
> u8 real_iv[16];
> + u32 key[8];
> +
> + if (rctx->init)
> + goto skip_init;
>
> chacha_init_generic(state, ctx->key, req->iv);
> - hchacha_block_arch(state, subctx.key, ctx->nrounds);
> - subctx.nrounds = ctx->nrounds;
> + hchacha_block_arch(state, key, nrounds);
>
> memcpy(&real_iv[0], req->iv + 24, 8);
> memcpy(&real_iv[8], req->iv + 16, 8);
> - return chacha_neon_stream_xor(req, &subctx, real_iv);
> +
> + chacha_init_generic(state, key, real_iv);
> +
> +skip_init:
> + return chacha_neon_stream_xor(req, nrounds);
> }
>
> static struct skcipher_alg algs[] = {
> @@ -174,6 +184,7 @@ static struct skcipher_alg algs[] = {
> .ivsize = CHACHA_IV_SIZE,
> .chunksize = CHACHA_BLOCK_SIZE,
> .walksize = 5 * CHACHA_BLOCK_SIZE,
> + .reqsize = sizeof(struct chacha_reqctx),
> .setkey = chacha20_setkey,
> .encrypt = chacha_neon,
> .decrypt = chacha_neon,
> @@ -190,6 +201,7 @@ static struct skcipher_alg algs[] = {
> .ivsize = XCHACHA_IV_SIZE,
> .chunksize = CHACHA_BLOCK_SIZE,
> .walksize = 5 * CHACHA_BLOCK_SIZE,
> + .reqsize = sizeof(struct chacha_reqctx),
> .setkey = chacha20_setkey,
> .encrypt = xchacha_neon,
> .decrypt = xchacha_neon,
> @@ -206,6 +218,7 @@ static struct skcipher_alg algs[] = {
> .ivsize = XCHACHA_IV_SIZE,
> .chunksize = CHACHA_BLOCK_SIZE,
> .walksize = 5 * CHACHA_BLOCK_SIZE,
> + .reqsize = sizeof(struct chacha_reqctx),
> .setkey = chacha12_setkey,
> .encrypt = xchacha_neon,
> .decrypt = xchacha_neon,
