Enhance ASYNC_TX subsystem to correctly update DMA channel reference counts, so it won't break DMA device hotplug logic.
Signed-off-by: Jiang Liu <[email protected]> --- crypto/async_tx/async_memcpy.c | 2 ++ crypto/async_tx/async_memset.c | 2 ++ crypto/async_tx/async_pq.c | 10 ++++++++-- crypto/async_tx/async_raid6_recov.c | 8 ++++++-- crypto/async_tx/async_tx.c | 6 +++--- crypto/async_tx/async_xor.c | 13 +++++++++---- include/linux/async_tx.h | 13 +++++++++++++ include/linux/dmaengine.h | 14 ++++++++++++++ 8 files changed, 57 insertions(+), 11 deletions(-) diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c index 361b5e8..0cbd90e 100644 --- a/crypto/async_tx/async_memcpy.c +++ b/crypto/async_tx/async_memcpy.c @@ -90,6 +90,8 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, async_tx_sync_epilog(submit); } + async_tx_put_channel(chan); + return tx; } EXPORT_SYMBOL_GPL(async_memcpy); diff --git a/crypto/async_tx/async_memset.c b/crypto/async_tx/async_memset.c index 58e4a87..ec568bb 100644 --- a/crypto/async_tx/async_memset.c +++ b/crypto/async_tx/async_memset.c @@ -79,6 +79,8 @@ async_memset(struct page *dest, int val, unsigned int offset, size_t len, async_tx_sync_epilog(submit); } + async_tx_put_channel(chan); + return tx; } EXPORT_SYMBOL_GPL(async_memset); diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c index 91d5d38..ae2070c 100644 --- a/crypto/async_tx/async_pq.c +++ b/crypto/async_tx/async_pq.c @@ -203,6 +203,7 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, blocks, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; dma_addr_t *dma_src = NULL; + struct dma_async_tx_descriptor *tx; BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks))); @@ -218,12 +219,15 @@ async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, /* run the p+q asynchronously */ pr_debug("%s: (async) disks: %d len: %zu\n", __func__, disks, len); - return do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset, - disks, len, dma_src, submit); + tx = do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset, + disks, len, dma_src, submit); + async_tx_put_channel(chan); + return tx; } /* run the pq synchronously */ pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); + async_tx_put_channel(chan); /* wait for any prerequisite operations */ async_tx_quiesce(&submit->depend_tx); @@ -331,6 +335,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, dma_async_issue_pending(chan); } async_tx_submit(chan, tx, submit); + async_tx_put_channel(chan); return tx; } else { @@ -344,6 +349,7 @@ async_syndrome_val(struct page **blocks, unsigned int offset, int disks, pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); + async_tx_put_channel(chan); /* caller must provide a temporary result buffer and * allow the input parameters to be preserved diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c index a9f08a6..0f54d7c 100644 --- a/crypto/async_tx/async_raid6_recov.c +++ b/crypto/async_tx/async_raid6_recov.c @@ -54,6 +54,7 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, len, dma_flags); if (tx) { async_tx_submit(chan, tx, submit); + async_tx_put_channel(chan); return tx; } @@ -66,6 +67,7 @@ async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, } /* run the operation synchronously */ + async_tx_put_channel(chan); async_tx_quiesce(&submit->depend_tx); amul = raid6_gfmul[coef[0]]; bmul = raid6_gfmul[coef[1]]; @@ -107,6 +109,7 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len, len, dma_flags); if (tx) { async_tx_submit(chan, tx, submit); + async_tx_put_channel(chan); return tx; } @@ -120,6 +123,7 @@ async_mult(struct page *dest, struct page *src, u8 coef, size_t len, /* no channel available, or failed to allocate a descriptor, so * perform the operation synchronously */ + async_tx_put_channel(chan); async_tx_quiesce(&submit->depend_tx); qmul = raid6_gfmul[coef]; d = page_address(dest); @@ -339,7 +343,7 @@ async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, * available' case be sure to use the scribble buffer to * preserve the content of 'blocks' as the caller intended. */ - if (!async_dma_find_channel(DMA_PQ) || !scribble) { + if (!async_tx_has_capability(DMA_PQ) || !scribble) { void **ptrs = scribble ? scribble : (void **) blocks; async_tx_quiesce(&submit->depend_tx); @@ -415,7 +419,7 @@ async_raid6_datap_recov(int disks, size_t bytes, int faila, * available' case be sure to use the scribble buffer to * preserve the content of 'blocks' as the caller intended. */ - if (!async_dma_find_channel(DMA_PQ) || !scribble) { + if (!async_tx_has_capability(DMA_PQ) || !scribble) { void **ptrs = scribble ? scribble : (void **) blocks; async_tx_quiesce(&submit->depend_tx); diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c index 8421209..6fe6561 100644 --- a/crypto/async_tx/async_tx.c +++ b/crypto/async_tx/async_tx.c @@ -47,8 +47,8 @@ module_init(async_tx_init); module_exit(async_tx_exit); /** - * __async_tx_find_channel - find a channel to carry out the operation or let - * the transaction execute synchronously + * __async_tx_find_channel - find and hold a channel to carry out the + * operation or let the transaction execute synchronously * @submit: transaction dependency and submission modifiers * @tx_type: transaction type */ @@ -61,7 +61,7 @@ __async_tx_find_channel(struct async_submit_ctl *submit, /* see if we can keep the chain on one channel */ if (depend_tx && dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) - return depend_tx->chan; + return async_dma_get_channel(depend_tx->chan); return async_dma_find_channel(tx_type); } EXPORT_SYMBOL_GPL(__async_tx_find_channel); diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index 154cc84..056e248 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c @@ -190,6 +190,7 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, &dest, 1, src_list, src_cnt, len); dma_addr_t *dma_src = NULL; + struct dma_async_tx_descriptor *tx = NULL; BUG_ON(src_cnt <= 1); @@ -202,8 +203,8 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, /* run the xor asynchronously */ pr_debug("%s (async): len: %zu\n", __func__, len); - return do_async_xor(chan, dest, src_list, offset, src_cnt, len, - dma_src, submit); + tx = do_async_xor(chan, dest, src_list, offset, src_cnt, len, + dma_src, submit); } else { /* run the xor synchronously */ pr_debug("%s (sync): len: %zu\n", __func__, len); @@ -222,9 +223,11 @@ async_xor(struct page *dest, struct page **src_list, unsigned int offset, async_tx_quiesce(&submit->depend_tx); do_sync_xor(dest, src_list, offset, src_cnt, len, submit); - - return NULL; } + + async_tx_put_channel(chan); + + return tx; } EXPORT_SYMBOL_GPL(async_xor); @@ -330,6 +333,8 @@ async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, submit->flags = flags_orig; } + async_tx_put_channel(chan); + return tx; } EXPORT_SYMBOL_GPL(async_xor_val); diff --git a/include/linux/async_tx.h b/include/linux/async_tx.h index a1c486a..35dea72 100644 --- a/include/linux/async_tx.h +++ b/include/linux/async_tx.h @@ -104,6 +104,10 @@ static inline void async_tx_issue_pending(struct dma_async_tx_descriptor *tx) dma->device_issue_pending(chan); } } + +#define async_tx_put_channel(c) async_dma_put_channel(c) +#define async_tx_has_capability(c) async_dma_has_capability(c) + #ifdef CONFIG_ARCH_HAS_ASYNC_TX_FIND_CHANNEL #include <asm/async_tx.h> #else @@ -132,6 +136,15 @@ async_tx_find_channel(struct async_submit_ctl *submit, { return NULL; } + +static inline void async_tx_put_channel(struct dma_chan *chan) +{ +} + +static inline bool async_tx_has_capability(enum dma_transaction_type type) +{ + return false; +} #endif /** diff --git a/include/linux/dmaengine.h b/include/linux/dmaengine.h index 874f8de..99762a2 100644 --- a/include/linux/dmaengine.h +++ b/include/linux/dmaengine.h @@ -789,6 +789,9 @@ static inline void net_dmaengine_put(void) #else #define async_dma_find_channel(type) dma_find_channel(type) #endif /* CONFIG_ASYNC_TX_ENABLE_CHANNEL_SWITCH */ +#define async_dma_get_channel(chan) dma_get_channel(chan) +#define async_dma_put_channel(chan) dma_put_channel(chan) +#define async_dma_has_capability(c) dma_has_capability(c) #else static inline void async_dmaengine_get(void) { @@ -801,6 +804,17 @@ async_dma_find_channel(enum dma_transaction_type type) { return NULL; } +static inline struct dma_chan *async_dma_get_channel(struct dma_chan *chan) +{ + return chan; +} +static inline void async_dma_put_channel(struct dma_chan *chan) +{ +} +static inline bool async_dma_has_capability(enum dma_transaction_type type) +{ + return false; +} #endif /* CONFIG_ASYNC_TX_DMA */ dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan, -- 1.7.5.4 -- To unsubscribe from this list: send the line "unsubscribe linux-crypto" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html
