From: Eric Biggers <[email protected]> blk-crypto-fallback.c has many static variables with inconsistent names, e.g. "blk_crypto_*", "crypto_*", and some unprefixed names. This is confusing. Consolidate them all into a struct named "blk_crypto_fallback" so that it's clear what they are.
Signed-off-by: Eric Biggers <[email protected]> --- block/blk-crypto-fallback.c | 251 +++++++++++++++++++----------------- 1 file changed, 136 insertions(+), 115 deletions(-) diff --git a/block/blk-crypto-fallback.c b/block/blk-crypto-fallback.c index 82b302597b474..a4a444c83fb3c 100644 --- a/block/blk-crypto-fallback.c +++ b/block/blk-crypto-fallback.c @@ -21,20 +21,10 @@ #include "blk-crypto-internal.h" -static unsigned int num_prealloc_bounce_pg = 32; -module_param(num_prealloc_bounce_pg, uint, 0); -MODULE_PARM_DESC(num_prealloc_bounce_pg, - "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); - -static unsigned int blk_crypto_num_keyslots = 100; -module_param_named(num_keyslots, blk_crypto_num_keyslots, uint, 0); -MODULE_PARM_DESC(num_keyslots, - "Number of keyslots for the blk-crypto crypto API fallback"); - -static unsigned int num_prealloc_fallback_crypt_ctxs = 128; -module_param(num_prealloc_fallback_crypt_ctxs, uint, 0); -MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, - "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); +struct blk_crypto_fallback_keyslot { + enum blk_crypto_mode_num crypto_mode; + struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; +}; struct bio_fallback_crypt_ctx { struct bio_crypt_ctx crypt_ctx; @@ -57,47 +47,80 @@ struct bio_fallback_crypt_ctx { }; }; -static struct kmem_cache *bio_fallback_crypt_ctx_cache; -static mempool_t *bio_fallback_crypt_ctx_pool; +/* All state for blk-crypto-fallback */ +static struct blk_crypto_fallback { + unsigned int num_prealloc_bounce_pg; + unsigned int num_keyslots; + unsigned int num_prealloc_crypt_ctxs; -/* - * Allocating a crypto tfm during I/O can deadlock, so we have to preallocate - * all of a mode's tfms when that mode starts being used. Since each mode may - * need all the keyslots at some point, each mode needs its own tfm for each - * keyslot; thus, a keyslot may contain tfms for multiple modes. However, to - * match the behavior of real inline encryption hardware (which only supports a - * single encryption context per keyslot), we only allow one tfm per keyslot to - * be used at a time - the rest of the unused tfms have their keys cleared. - */ -static DEFINE_MUTEX(tfms_init_lock); -static bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; + bool initialized; -static struct blk_crypto_fallback_keyslot { - enum blk_crypto_mode_num crypto_mode; - struct crypto_skcipher *tfms[BLK_ENCRYPTION_MODE_MAX]; -} *blk_crypto_keyslots; + /* + * This is the key we set when evicting a keyslot. This *should* be the + * all 0's key, but AES-XTS rejects that key, so we use some random + * bytes instead. + */ + u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; -static struct blk_keyslot_manager blk_crypto_ksm; -static struct workqueue_struct *blk_crypto_wq; -static mempool_t *blk_crypto_bounce_page_pool; -static struct bio_set crypto_bio_split; + struct bio_set bio_split; -/* - * This is the key we set when evicting a keyslot. This *should* be the all 0's - * key, but AES-XTS rejects that key, so we use some random bytes instead. - */ -static u8 blank_key[BLK_CRYPTO_MAX_KEY_SIZE]; + struct blk_keyslot_manager ksm; + + struct workqueue_struct *decrypt_wq; + + struct blk_crypto_fallback_keyslot *keyslots; + + mempool_t *bounce_page_pool; + struct kmem_cache *crypt_ctx_cache; + mempool_t *crypt_ctx_pool; + + /* + * Allocating a crypto tfm during I/O can deadlock, so we have to + * preallocate all of a mode's tfms when that mode starts being used. + * Since each mode may need all the keyslots at some point, each mode + * needs its own tfm for each keyslot; thus, a keyslot may contain tfms + * for multiple modes. However, to match the behavior of real inline + * encryption hardware (which only supports a single encryption context + * per keyslot), we only allow one tfm per keyslot to be used at a time + * - the rest of the unused tfms have their keys cleared. + */ + struct mutex tfms_init_lock; + bool tfms_inited[BLK_ENCRYPTION_MODE_MAX]; + +} blk_crypto_fallback = { + .num_prealloc_bounce_pg = 32, + .num_keyslots = 100, + .num_prealloc_crypt_ctxs = 128, + .tfms_init_lock = + __MUTEX_INITIALIZER(blk_crypto_fallback.tfms_init_lock), +}; + +module_param_named(num_prealloc_bounce_pg, + blk_crypto_fallback.num_prealloc_bounce_pg, uint, 0); +MODULE_PARM_DESC(num_prealloc_bounce_pg, + "Number of preallocated bounce pages for the blk-crypto crypto API fallback"); + +module_param_named(num_keyslots, blk_crypto_fallback.num_keyslots, uint, 0); +MODULE_PARM_DESC(num_keyslots, + "Number of keyslots for the blk-crypto crypto API fallback"); + +module_param_named(num_prealloc_fallback_crypt_ctxs, + blk_crypto_fallback.num_prealloc_crypt_ctxs, uint, 0); +MODULE_PARM_DESC(num_prealloc_crypt_fallback_ctxs, + "Number of preallocated bio fallback crypto contexts for blk-crypto to use during crypto API fallback"); static void blk_crypto_fallback_evict_keyslot(unsigned int slot) { - struct blk_crypto_fallback_keyslot *slotp = &blk_crypto_keyslots[slot]; + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; + struct blk_crypto_fallback_keyslot *slotp = &fallback->keyslots[slot]; enum blk_crypto_mode_num crypto_mode = slotp->crypto_mode; int err; WARN_ON(slotp->crypto_mode == BLK_ENCRYPTION_MODE_INVALID); /* Clear the key in the skcipher */ - err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], blank_key, + err = crypto_skcipher_setkey(slotp->tfms[crypto_mode], + fallback->blank_key, blk_crypto_modes[crypto_mode].keysize); WARN_ON(err); slotp->crypto_mode = BLK_ENCRYPTION_MODE_INVALID; @@ -107,7 +130,8 @@ static int blk_crypto_fallback_keyslot_program(struct blk_keyslot_manager *ksm, const struct blk_crypto_key *key, unsigned int slot) { - struct blk_crypto_fallback_keyslot *slotp = &blk_crypto_keyslots[slot]; + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; + struct blk_crypto_fallback_keyslot *slotp = &fallback->keyslots[slot]; const enum blk_crypto_mode_num crypto_mode = key->crypto_cfg.crypto_mode; int err; @@ -134,16 +158,6 @@ static int blk_crypto_fallback_keyslot_evict(struct blk_keyslot_manager *ksm, return 0; } -/* - * The crypto API fallback KSM ops - only used for a bio when it specifies a - * blk_crypto_key that was not supported by the device's inline encryption - * hardware. - */ -static const struct blk_ksm_ll_ops blk_crypto_ksm_ll_ops = { - .keyslot_program = blk_crypto_fallback_keyslot_program, - .keyslot_evict = blk_crypto_fallback_keyslot_evict, -}; - static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) { struct bio *src_bio = enc_bio->bi_private; @@ -151,7 +165,7 @@ static void blk_crypto_fallback_encrypt_endio(struct bio *enc_bio) for (i = 0; i < enc_bio->bi_vcnt; i++) mempool_free(enc_bio->bi_io_vec[i].bv_page, - blk_crypto_bounce_page_pool); + blk_crypto_fallback.bounce_page_pool); src_bio->bi_status = enc_bio->bi_status; @@ -195,7 +209,7 @@ blk_crypto_fallback_alloc_cipher_req(struct blk_ksm_keyslot *slot, const struct blk_crypto_fallback_keyslot *slotp; int keyslot_idx = blk_ksm_get_slot_idx(slot); - slotp = &blk_crypto_keyslots[keyslot_idx]; + slotp = &blk_crypto_fallback.keyslots[keyslot_idx]; ciph_req = skcipher_request_alloc(slotp->tfms[slotp->crypto_mode], GFP_NOIO); if (!ciph_req) @@ -227,7 +241,7 @@ static bool blk_crypto_fallback_split_bio_if_needed(struct bio **bio_ptr) struct bio *split_bio; split_bio = bio_split(bio, num_sectors, GFP_NOIO, - &crypto_bio_split); + &blk_crypto_fallback.bio_split); if (!split_bio) { bio->bi_status = BLK_STS_RESOURCE; return false; @@ -263,6 +277,7 @@ static void blk_crypto_dun_to_iv(const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], */ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) { + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; struct bio *src_bio, *enc_bio; struct bio_crypt_ctx *bc; struct blk_ksm_keyslot *slot; @@ -295,7 +310,7 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) * Use the crypto API fallback keyslot manager to get a crypto_skcipher * for the algorithm and key specified for this bio. */ - blk_st = blk_ksm_get_slot_for_key(&blk_crypto_ksm, bc->bc_key, &slot); + blk_st = blk_ksm_get_slot_for_key(&fallback->ksm, bc->bc_key, &slot); if (blk_st != BLK_STS_OK) { src_bio->bi_status = blk_st; goto out_put_enc_bio; @@ -319,7 +334,7 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) struct bio_vec *enc_bvec = &enc_bio->bi_io_vec[i]; struct page *plaintext_page = enc_bvec->bv_page; struct page *ciphertext_page = - mempool_alloc(blk_crypto_bounce_page_pool, GFP_NOIO); + mempool_alloc(fallback->bounce_page_pool, GFP_NOIO); enc_bvec->bv_page = ciphertext_page; @@ -359,7 +374,7 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) out_free_bounce_pages: while (i > 0) mempool_free(enc_bio->bi_io_vec[--i].bv_page, - blk_crypto_bounce_page_pool); + fallback->bounce_page_pool); out_free_ciph_req: skcipher_request_free(ciph_req); out_release_keyslot: @@ -377,6 +392,7 @@ static bool blk_crypto_fallback_encrypt_bio(struct bio **bio_ptr) */ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) { + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; struct bio_fallback_crypt_ctx *f_ctx = container_of(work, struct bio_fallback_crypt_ctx, work); struct bio *bio = f_ctx->bio; @@ -397,7 +413,7 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) * Use the crypto API fallback keyslot manager to get a crypto_skcipher * for the algorithm and key specified for this bio. */ - blk_st = blk_ksm_get_slot_for_key(&blk_crypto_ksm, bc->bc_key, &slot); + blk_st = blk_ksm_get_slot_for_key(&fallback->ksm, bc->bc_key, &slot); if (blk_st != BLK_STS_OK) { bio->bi_status = blk_st; goto out_no_keyslot; @@ -437,7 +453,7 @@ static void blk_crypto_fallback_decrypt_bio(struct work_struct *work) skcipher_request_free(ciph_req); blk_ksm_put_slot(slot); out_no_keyslot: - mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); + mempool_free(f_ctx, fallback->crypt_ctx_pool); bio_endio(bio); } @@ -458,14 +474,14 @@ static void blk_crypto_fallback_decrypt_endio(struct bio *bio) /* If there was an IO error, don't queue for decrypt. */ if (bio->bi_status) { - mempool_free(f_ctx, bio_fallback_crypt_ctx_pool); + mempool_free(f_ctx, blk_crypto_fallback.crypt_ctx_pool); bio_endio(bio); return; } INIT_WORK(&f_ctx->work, blk_crypto_fallback_decrypt_bio); f_ctx->bio = bio; - queue_work(blk_crypto_wq, &f_ctx->work); + queue_work(blk_crypto_fallback.decrypt_wq, &f_ctx->work); } /** @@ -490,17 +506,19 @@ static void blk_crypto_fallback_decrypt_endio(struct bio *bio) */ bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) { + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; struct bio *bio = *bio_ptr; struct bio_crypt_ctx *bc = bio->bi_crypt_context; struct bio_fallback_crypt_ctx *f_ctx; - if (WARN_ON_ONCE(!tfms_inited[bc->bc_key->crypto_cfg.crypto_mode])) { + if (WARN_ON_ONCE(!fallback->tfms_inited[ + bc->bc_key->crypto_cfg.crypto_mode])) { /* User didn't call blk_crypto_start_using_key() first */ bio->bi_status = BLK_STS_IOERR; return false; } - if (!blk_ksm_crypto_cfg_supported(&blk_crypto_ksm, + if (!blk_ksm_crypto_cfg_supported(&fallback->ksm, &bc->bc_key->crypto_cfg)) { bio->bi_status = BLK_STS_NOTSUPP; return false; @@ -513,7 +531,7 @@ bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) * bio READ case: Set up a f_ctx in the bio's bi_private and set the * bi_end_io appropriately to trigger decryption when the bio is ended. */ - f_ctx = mempool_alloc(bio_fallback_crypt_ctx_pool, GFP_NOIO); + f_ctx = mempool_alloc(fallback->crypt_ctx_pool, GFP_NOIO); f_ctx->crypt_ctx = *bc; f_ctx->crypt_iter = bio->bi_iter; f_ctx->bi_private_orig = bio->bi_private; @@ -527,79 +545,82 @@ bool blk_crypto_fallback_bio_prep(struct bio **bio_ptr) int blk_crypto_fallback_evict_key(const struct blk_crypto_key *key) { - return blk_ksm_evict_key(&blk_crypto_ksm, key); + return blk_ksm_evict_key(&blk_crypto_fallback.ksm, key); } -static bool blk_crypto_fallback_inited; static int blk_crypto_fallback_init(void) { + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; int i; int err; - if (blk_crypto_fallback_inited) + if (fallback->initialized) return 0; - prandom_bytes(blank_key, BLK_CRYPTO_MAX_KEY_SIZE); + prandom_bytes(fallback->blank_key, BLK_CRYPTO_MAX_KEY_SIZE); - err = bioset_init(&crypto_bio_split, 64, 0, 0); + err = bioset_init(&fallback->bio_split, 64, 0, 0); if (err) goto out; - err = blk_ksm_init(&blk_crypto_ksm, blk_crypto_num_keyslots); + err = blk_ksm_init(&fallback->ksm, fallback->num_keyslots); if (err) goto fail_free_bioset; err = -ENOMEM; - blk_crypto_ksm.ksm_ll_ops = blk_crypto_ksm_ll_ops; - blk_crypto_ksm.max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE; + fallback->ksm.ksm_ll_ops.keyslot_program = + blk_crypto_fallback_keyslot_program; + fallback->ksm.ksm_ll_ops.keyslot_evict = + blk_crypto_fallback_keyslot_evict; + fallback->ksm.max_dun_bytes_supported = BLK_CRYPTO_MAX_IV_SIZE; /* All blk-crypto modes have a crypto API fallback. */ for (i = 0; i < BLK_ENCRYPTION_MODE_MAX; i++) - blk_crypto_ksm.crypto_modes_supported[i] = 0xFFFFFFFF; - blk_crypto_ksm.crypto_modes_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; - - blk_crypto_wq = alloc_workqueue("blk_crypto_wq", - WQ_UNBOUND | WQ_HIGHPRI | - WQ_MEM_RECLAIM, num_online_cpus()); - if (!blk_crypto_wq) + fallback->ksm.crypto_modes_supported[i] = 0xFFFFFFFF; + fallback->ksm.crypto_modes_supported[BLK_ENCRYPTION_MODE_INVALID] = 0; + + fallback->decrypt_wq = alloc_workqueue("blk_crypto_fallback_wq", + WQ_UNBOUND | WQ_HIGHPRI | + WQ_MEM_RECLAIM, + num_online_cpus()); + if (!fallback->decrypt_wq) goto fail_free_ksm; - blk_crypto_keyslots = kcalloc(blk_crypto_num_keyslots, - sizeof(blk_crypto_keyslots[0]), - GFP_KERNEL); - if (!blk_crypto_keyslots) + fallback->keyslots = kcalloc(fallback->num_keyslots, + sizeof(fallback->keyslots[0]), GFP_KERNEL); + if (!fallback->keyslots) goto fail_free_wq; - blk_crypto_bounce_page_pool = - mempool_create_page_pool(num_prealloc_bounce_pg, 0); - if (!blk_crypto_bounce_page_pool) + fallback->bounce_page_pool = + mempool_create_page_pool(fallback->num_prealloc_bounce_pg, 0); + if (!fallback->bounce_page_pool) goto fail_free_keyslots; - bio_fallback_crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); - if (!bio_fallback_crypt_ctx_cache) + fallback->crypt_ctx_cache = KMEM_CACHE(bio_fallback_crypt_ctx, 0); + if (!fallback->crypt_ctx_cache) goto fail_free_bounce_page_pool; - bio_fallback_crypt_ctx_pool = - mempool_create_slab_pool(num_prealloc_fallback_crypt_ctxs, - bio_fallback_crypt_ctx_cache); - if (!bio_fallback_crypt_ctx_pool) + fallback->crypt_ctx_pool = + mempool_create_slab_pool(fallback->num_prealloc_crypt_ctxs, + fallback->crypt_ctx_cache); + if (!fallback->crypt_ctx_pool) goto fail_free_crypt_ctx_cache; - blk_crypto_fallback_inited = true; + fallback->initialized = true; return 0; fail_free_crypt_ctx_cache: - kmem_cache_destroy(bio_fallback_crypt_ctx_cache); + kmem_cache_destroy(fallback->crypt_ctx_cache); fail_free_bounce_page_pool: - mempool_destroy(blk_crypto_bounce_page_pool); + mempool_destroy(fallback->bounce_page_pool); fail_free_keyslots: - kfree(blk_crypto_keyslots); + kfree(fallback->keyslots); fail_free_wq: - destroy_workqueue(blk_crypto_wq); + destroy_workqueue(fallback->decrypt_wq); fail_free_ksm: - blk_ksm_destroy(&blk_crypto_ksm); + blk_ksm_destroy(&fallback->ksm); fail_free_bioset: - bioset_exit(&crypto_bio_split); + bioset_exit(&fallback->bio_split); out: return err; } @@ -610,29 +631,29 @@ static int blk_crypto_fallback_init(void) */ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) { + struct blk_crypto_fallback *fallback = &blk_crypto_fallback; const char *cipher_str = blk_crypto_modes[mode_num].cipher_str; struct blk_crypto_fallback_keyslot *slotp; unsigned int i; int err = 0; /* - * Fast path - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we try to access them. + * Fast path. Ensure that updates to keyslots[i].tfms[mode_num] for + * each i are visible before we try to access them. */ - if (likely(smp_load_acquire(&tfms_inited[mode_num]))) + if (likely(smp_load_acquire(&fallback->tfms_inited[mode_num]))) return 0; - mutex_lock(&tfms_init_lock); - if (tfms_inited[mode_num]) + mutex_lock(&fallback->tfms_init_lock); + if (fallback->tfms_inited[mode_num]) goto out; err = blk_crypto_fallback_init(); if (err) goto out; - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; + for (i = 0; i < fallback->num_keyslots; i++) { + slotp = &fallback->keyslots[i]; slotp->tfms[mode_num] = crypto_alloc_skcipher(cipher_str, 0, 0); if (IS_ERR(slotp->tfms[mode_num])) { err = PTR_ERR(slotp->tfms[mode_num]); @@ -650,19 +671,19 @@ int blk_crypto_fallback_start_using_mode(enum blk_crypto_mode_num mode_num) } /* - * Ensure that updates to blk_crypto_keyslots[i].tfms[mode_num] - * for each i are visible before we set tfms_inited[mode_num]. + * Ensure that updates to keyslots[i].tfms[mode_num] for each i are + * visible before we set tfms_inited[mode_num]. */ - smp_store_release(&tfms_inited[mode_num], true); + smp_store_release(&fallback->tfms_inited[mode_num], true); goto out; out_free_tfms: - for (i = 0; i < blk_crypto_num_keyslots; i++) { - slotp = &blk_crypto_keyslots[i]; + for (i = 0; i < fallback->num_keyslots; i++) { + slotp = &fallback->keyslots[i]; crypto_free_skcipher(slotp->tfms[mode_num]); slotp->tfms[mode_num] = NULL; } out: - mutex_unlock(&tfms_init_lock); + mutex_unlock(&fallback->tfms_init_lock); return err; } -- 2.33.0 -- dm-devel mailing list [email protected] https://listman.redhat.com/mailman/listinfo/dm-devel
