PR #23805 opened by Lynne URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23805 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23805.patch
This commit implements GENERAL-only images (required to avoid locking by avoiding the need to change layouts), and a lockless submission mode (opt-in). >From 4a7623b56b407e8c10c084dc11c413fa511a1295 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Tue, 14 Jul 2026 15:03:29 +0900 Subject: [PATCH 1/3] vulkan: support always-GENERAL image layouts with lock-free frame usage Adds handling for AV_VK_FRAME_FLAG_GENERAL_LAYOUT to the execution framework. Frames with the flag set are kept in VK_IMAGE_LAYOUT_GENERAL permanently: image layout transitions and queue family ownership transfers between libavutil-side uses are elided (concurrent-sharing images are exempt from all ownership transfers; exclusive images still get release/acquire barriers for external handoff), and frames are no longer locked across command buffer recording. Semaphore values are instead assigned and published at submission time, with the frame lock only ever held within a single function call, one frame at a time. Frames transitioned to other layouts by API users are transitioned back upon the next use, using the layout tracking fields as before. This removes the lock-ordering and cross-thread mutex ownership hazards which made frame-threaded decoding unusable. --- libavutil/hwcontext_vulkan.h | 7 ++ libavutil/vulkan.c | 189 +++++++++++++++++++++++++++++++++-- libavutil/vulkan.h | 16 +++ libavutil/vulkan_functions.h | 2 + 4 files changed, 208 insertions(+), 6 deletions(-) diff --git a/libavutil/hwcontext_vulkan.h b/libavutil/hwcontext_vulkan.h index 87c2a2d28d..abfcbcb14e 100644 --- a/libavutil/hwcontext_vulkan.h +++ b/libavutil/hwcontext_vulkan.h @@ -163,6 +163,13 @@ typedef enum AVVkFrameFlags { /* Disables multiplane images. * This is required to export/import images from CUDA. */ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE = (1ULL << 2), + + /* Keeps frames in VK_IMAGE_LAYOUT_GENERAL, skipping all image layout + * transitions, queue family ownership transfers and frame locking. + * Automatically enabled if VK_KHR_unified_image_layouts is supported. + * AVVkFrame.layout must remain GENERAL at all times; AVVkFrame.access + * and AVVkFrame.queue_family are not maintained. */ + AV_VK_FRAME_FLAG_GENERAL_LAYOUT = (1ULL << 3), } AVVkFrameFlags; /** diff --git a/libavutil/vulkan.c b/libavutil/vulkan.c index 3b2cc35bf9..ab29741f55 100644 --- a/libavutil/vulkan.c +++ b/libavutil/vulkan.c @@ -319,6 +319,10 @@ void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool) av_free(e->access_dst); av_free(e->frame_update); av_free(e->frame_locked); + av_free(e->frame_wait_base); + av_free(e->frame_sig_base); + av_free(e->gen_mirror_dst); + av_free(e->gen_mirror_dep); av_free(e->sem_sig); av_free(e->sem_sig_val_dst); av_free(e->sem_wait); @@ -635,6 +639,7 @@ void ff_vk_exec_discard_deps(FFVulkanContext *s, FFVkExecContext *e) e->sem_wait_cnt = 0; e->sem_sig_cnt = 0; e->sem_sig_val_dst_cnt = 0; + e->gen_mirror_cnt = 0; } int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, @@ -714,6 +719,13 @@ static void destroy_tmp_semaphores(void *opaque, uint8_t *data) av_free(ts); } +int ff_vk_frame_is_general(AVFrame *f) +{ + AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data; + AVVulkanFramesContext *vkfc = hwfc->hwctx; + return !!(vkfc->flags & AV_VK_FRAME_FLAG_GENERAL_LAYOUT); +} + int ff_vk_exec_add_dep_wait_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore sem, uint64_t val, VkPipelineStageFlagBits2 stage) @@ -808,6 +820,8 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkImageLayout *layout_dst; uint32_t *queue_family_dst; VkAccessFlagBits *access_dst; + int *frame_wait_base; + int *frame_sig_base; AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data; AVVulkanFramesContext *vkfc = hwfc->hwctx; @@ -826,6 +840,8 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, ARR_REALLOC(e, frame_locked, &e->frame_locked_alloc_size, e->nb_frame_deps); ARR_REALLOC(e, frame_update, &e->frame_update_alloc_size, e->nb_frame_deps); ARR_REALLOC(e, frame_deps, &e->frame_deps_alloc_size, e->nb_frame_deps); + ARR_REALLOC(e, frame_wait_base, &e->frame_wait_base_alloc, e->nb_frame_deps); + ARR_REALLOC(e, frame_sig_base, &e->frame_sig_base_alloc, e->nb_frame_deps); /* prepare_frame in hwcontext_vulkan.c uses the regular frame management * code but has no frame yet, and it doesn't need to actually store a ref @@ -841,9 +857,17 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, } e->frame_deps[e->nb_frame_deps] = f; + e->frame_wait_base[e->nb_frame_deps] = e->sem_wait_cnt; + e->frame_sig_base[e->nb_frame_deps] = e->sem_sig_cnt; - vkfc->lock_frame(hwfc, vkf); - e->frame_locked[e->nb_frame_deps] = 1; + /* GENERAL_LAYOUT frames are not locked during recording; their + * semaphore values are assigned at submission time */ + if (ff_vk_frame_is_general(f)) { + e->frame_locked[e->nb_frame_deps] = 0; + } else { + vkfc->lock_frame(hwfc, vkf); + e->frame_locked[e->nb_frame_deps] = 1; + } e->frame_update[e->nb_frame_deps] = 0; e->nb_frame_deps++; @@ -859,19 +883,21 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, e->sem_wait[e->sem_wait_cnt++] = (VkSemaphoreSubmitInfo) { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = vkf->sem[i], - .value = vkf->sem_value[i], + .value = e->frame_locked[e->nb_frame_deps - 1] ? vkf->sem_value[i] : 0, .stageMask = wait_stage, }; e->sem_sig[e->sem_sig_cnt++] = (VkSemaphoreSubmitInfo) { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = vkf->sem[i], - .value = vkf->sem_value[i] + 1, + .value = e->frame_locked[e->nb_frame_deps - 1] ? vkf->sem_value[i] + 1 : 0, .stageMask = signal_stage, }; - e->sem_sig_val_dst[e->sem_sig_val_dst_cnt] = &vkf->sem_value[i]; - e->sem_sig_val_dst_cnt++; + if (e->frame_locked[e->nb_frame_deps - 1]) { + e->sem_sig_val_dst[e->sem_sig_val_dst_cnt] = &vkf->sem_value[i]; + e->sem_sig_val_dst_cnt++; + } } return 0; @@ -911,6 +937,22 @@ int ff_vk_exec_mirror_sem_value(FFVulkanContext *s, FFVkExecContext *e, if (i == e->nb_frame_deps) return AVERROR(EINVAL); + if (ff_vk_frame_is_general(f)) { + uint64_t **gen_mirror_dst; + int *gen_mirror_dep; + ARR_REALLOC(e, gen_mirror_dst, &e->gen_mirror_dst_alloc, e->gen_mirror_cnt); + ARR_REALLOC(e, gen_mirror_dep, &e->gen_mirror_dep_alloc, e->gen_mirror_cnt); + + *dst = vkf->sem[0]; + *dst_val = 0; /* Filled in at submission time */ + + e->gen_mirror_dst[e->gen_mirror_cnt] = dst_val; + e->gen_mirror_dep[e->gen_mirror_cnt] = i; + e->gen_mirror_cnt++; + + return 0; + } + ARR_REALLOC(e, sem_sig_val_dst, &e->sem_sig_val_dst_alloc, e->sem_sig_val_dst_cnt); *dst = vkf->sem[0]; @@ -948,6 +990,32 @@ int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e) return AVERROR_EXTERNAL; } + /* Assign and publish semaphore values for unlocked (GENERAL_LAYOUT) + * frames. The frame lock is only held within this loop, one frame at + * a time, and never across the submission itself. */ + for (int j = 0; j < e->nb_frame_deps; j++) { + AVFrame *f = e->frame_deps[j]; + AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data; + AVVulkanFramesContext *vkfc = hwfc->hwctx; + AVVkFrame *vkf = (AVVkFrame *)f->data[0]; + int nb_images = ff_vk_count_images(vkf); + + if (e->frame_locked[j]) + continue; + + vkfc->lock_frame(hwfc, vkf); + for (int i = 0; i < nb_images; i++) { + e->sem_wait[e->frame_wait_base[j] + i].value = vkf->sem_value[i]; + e->sem_sig[e->frame_sig_base[j] + i].value = vkf->sem_value[i] + 1; + vkf->sem_value[i]++; + } + vkfc->unlock_frame(hwfc, vkf); + } + + /* Fill in deferred semaphore value mirrors */ + for (int j = 0; j < e->gen_mirror_cnt; j++) + *e->gen_mirror_dst[j] = e->sem_sig[e->frame_sig_base[e->gen_mirror_dep[j]]].value; + #if FF_API_VULKAN_SYNC_QUEUES FF_DISABLE_DEPRECATION_WARNINGS s->hwctx->lock_queue(s->device, e->qf, e->qi); @@ -963,6 +1031,24 @@ FF_ENABLE_DEPRECATION_WARNINGS if (ret != VK_SUCCESS) { av_log(s, AV_LOG_ERROR, "Unable to submit command buffer: %s\n", ff_vk_ret2str(ret)); + + /* Signal already-published semaphore values from the host, to avoid + * deadlocking waiters which picked up values after us */ + for (int j = 0; j < e->nb_frame_deps; j++) { + AVVkFrame *vkf = (AVVkFrame *)e->frame_deps[j]->data[0]; + int nb_images = ff_vk_count_images(vkf); + if (e->frame_locked[j]) + continue; + for (int i = 0; i < nb_images; i++) { + VkSemaphoreSignalInfo sig_info = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SIGNAL_INFO, + .semaphore = vkf->sem[i], + .value = e->sem_sig[e->frame_sig_base[j] + i].value, + }; + vk->SignalSemaphore(s->hwctx->act_dev, &sig_info); + } + } + ff_vk_exec_discard_deps(s, e); return AVERROR_EXTERNAL; } @@ -2108,6 +2194,97 @@ void ff_vk_frame_barrier(FFVulkanContext *s, FFVkExecContext *e, break; } + if (ff_vk_frame_is_general(pic)) { + AVHWFramesContext *hwfc = (AVHWFramesContext *)pic->hw_frames_ctx->data; + AVVulkanFramesContext *vkfc = hwfc->hwctx; + VkImageLayout old_layout = VK_IMAGE_LAYOUT_GENERAL; + VkAccessFlagBits2 old_access = VK_ACCESS_2_MEMORY_READ_BIT | + VK_ACCESS_2_MEMORY_WRITE_BIT; + uint32_t src_qf = VK_QUEUE_FAMILY_IGNORED; + uint32_t dst_qf = VK_QUEUE_FAMILY_IGNORED; + int is_exclusive = 0; + int nb_qfs = 0; + + /* Concurrent-sharing images are exempt from all queue family + * ownership transfers, external ones included. Exclusive images + * (single queue family devices) still require transfers for + * external handoff. */ + for (int i = 0; i < s->hwctx->nb_qf; i++) { + int j; + for (j = 0; j < i; j++) + if (s->hwctx->qf[j].idx == s->hwctx->qf[i].idx) + break; + if (j == i) + nb_qfs++; + } + is_exclusive = nb_qfs <= 1; + + /* An external user may have moved the image out of GENERAL, or + * ownership may have been released to an external API; transition + * it back on first use. Reads/updates of the tracked state happen + * under the frame lock, but the lock is never held across + * recording or submission. */ + if (found < 0 && + (vkf->layout[0] != VK_IMAGE_LAYOUT_GENERAL || + (is_exclusive && + (vkf->queue_family[0] == VK_QUEUE_FAMILY_EXTERNAL_KHR || + vkf->queue_family[0] == VK_QUEUE_FAMILY_FOREIGN_EXT)))) { + vkfc->lock_frame(hwfc, vkf); + if (vkf->layout[0] != VK_IMAGE_LAYOUT_GENERAL) { + old_layout = vkf->layout[0]; + old_access = vkf->access[0]; + for (int i = 0; i < nb_images; i++) + vkf->layout[i] = VK_IMAGE_LAYOUT_GENERAL; + } + if (is_exclusive && + (vkf->queue_family[0] == VK_QUEUE_FAMILY_EXTERNAL_KHR || + vkf->queue_family[0] == VK_QUEUE_FAMILY_FOREIGN_EXT)) { + src_qf = vkf->queue_family[0]; + dst_qf = e->qf; + for (int i = 0; i < nb_images; i++) + vkf->queue_family[i] = e->qf; + } + vkfc->unlock_frame(hwfc, vkf); + } + + /* Release to an external API */ + if (is_exclusive && + (new_qf == VK_QUEUE_FAMILY_EXTERNAL_KHR || + new_qf == VK_QUEUE_FAMILY_FOREIGN_EXT)) { + src_qf = e->qf; + dst_qf = new_qf; + vkfc->lock_frame(hwfc, vkf); + for (int i = 0; i < nb_images; i++) + vkf->queue_family[i] = new_qf; + vkfc->unlock_frame(hwfc, vkf); + } + + for (int i = 0; i < nb_images; i++) { + bar[*nb_bar] = (VkImageMemoryBarrier2) { + .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, + .pNext = NULL, + .srcStageMask = src_stage, + .dstStageMask = dst_stage, + .srcAccessMask = found >= 0 ? e->access_dst[found] : old_access, + .dstAccessMask = new_access, + .oldLayout = found >= 0 ? VK_IMAGE_LAYOUT_GENERAL : old_layout, + .newLayout = VK_IMAGE_LAYOUT_GENERAL, + .srcQueueFamilyIndex = src_qf, + .dstQueueFamilyIndex = dst_qf, + .image = vkf->img[i], + .subresourceRange = (VkImageSubresourceRange) { + .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT, + .layerCount = 1, + .levelCount = 1, + }, + }; + *nb_bar += 1; + } + + ff_vk_exec_update_frame(s, e, pic, &bar[*nb_bar - nb_images], NULL); + return; + } + for (int i = 0; i < nb_images; i++) { bar[*nb_bar] = (VkImageMemoryBarrier2) { .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2, diff --git a/libavutil/vulkan.h b/libavutil/vulkan.h index b37d998d37..97407da417 100644 --- a/libavutil/vulkan.h +++ b/libavutil/vulkan.h @@ -160,6 +160,20 @@ typedef struct FFVkExecContext { uint8_t *frame_locked; unsigned int frame_locked_alloc_size; + /* Base indices into sem_wait/sem_sig for each frame dep, for + * submission-time semaphore value assignment of GENERAL_LAYOUT frames */ + int *frame_wait_base; + unsigned int frame_wait_base_alloc; + int *frame_sig_base; + unsigned int frame_sig_base_alloc; + + /* Deferred semaphore value mirrors for GENERAL_LAYOUT frames */ + uint64_t **gen_mirror_dst; + unsigned int gen_mirror_dst_alloc; + int *gen_mirror_dep; + unsigned int gen_mirror_dep_alloc; + int gen_mirror_cnt; + VkAccessFlagBits *access_dst; unsigned int access_dst_alloc; @@ -485,6 +499,8 @@ int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait); /* Ownership transferred if !wait */ +int ff_vk_frame_is_general(AVFrame *f); + int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage); diff --git a/libavutil/vulkan_functions.h b/libavutil/vulkan_functions.h index 86dff046b2..19d36b36a1 100644 --- a/libavutil/vulkan_functions.h +++ b/libavutil/vulkan_functions.h @@ -54,6 +54,7 @@ typedef uint64_t FFVulkanExtensions; #define FF_VK_EXT_REPLICATED_COMPOSITES (1ULL << 21) /* VK_EXT_shader_replicated_composites */ #define FF_VK_EXT_LONG_VECTOR (1ULL << 22) /* VK_EXT_shader_long_vector */ #define FF_VK_EXT_INTERNAL_QUEUE_SYNC (1ULL << 23) /* VK_KHR_internally_synchronized_queues */ +#define FF_VK_EXT_UNIFIED_IMAGE_LAYOUTS (1ULL << 24) /* VK_KHR_unified_image_layouts */ /* Video extensions */ #define FF_VK_EXT_VIDEO_QUEUE (1ULL << 36) /* VK_KHR_video_queue */ @@ -137,6 +138,7 @@ typedef uint64_t FFVulkanExtensions; MACRO(1, 1, FF_VK_EXT_EXTERNAL_FD_SEM, ImportSemaphoreFdKHR) \ MACRO(1, 1, FF_VK_EXT_NO_FLAG, CreateSemaphore) \ MACRO(1, 1, FF_VK_EXT_NO_FLAG, WaitSemaphores) \ + MACRO(1, 1, FF_VK_EXT_NO_FLAG, SignalSemaphore) \ MACRO(1, 1, FF_VK_EXT_NO_FLAG, DestroySemaphore) \ \ /* Memory */ \ -- 2.52.0 >From 193e8b789a71425fa9c96db856c1bb6c3cb21e41 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Tue, 14 Jul 2026 15:03:29 +0900 Subject: [PATCH 2/3] hwcontext_vulkan: enable unified image layouts when supported Enables VK_KHR_unified_image_layouts when available, and automatically sets AV_VK_FRAME_FLAG_GENERAL_LAYOUT on frame contexts when the unifiedImageLayouts feature is supported, the tiling is not DRM format modifiers, and, for contexts with video usage, when the unifiedImageLayoutsVideo feature is also supported. As with other autodetected flags, setting AV_VK_FRAME_FLAG_NONE disables this. --- libavutil/hwcontext_vulkan.c | 43 ++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c index 90d3d14dbb..b30926aca5 100644 --- a/libavutil/hwcontext_vulkan.c +++ b/libavutil/hwcontext_vulkan.c @@ -99,6 +99,10 @@ typedef struct VulkanDeviceFeatures { VkPhysicalDeviceShaderExpectAssumeFeaturesKHR expect_assume; #endif +#ifdef VK_KHR_unified_image_layouts + VkPhysicalDeviceUnifiedImageLayoutsFeaturesKHR unified_layouts; +#endif + VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1; #ifdef VK_KHR_video_maintenance2 VkPhysicalDeviceVideoMaintenance2FeaturesKHR video_maintenance_2; @@ -265,6 +269,11 @@ static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *f VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_EXPECT_ASSUME_FEATURES_KHR); #endif +#ifdef VK_KHR_unified_image_layouts + FF_VK_STRUCT_EXT(s, &feats->device, &feats->unified_layouts, FF_VK_EXT_UNIFIED_IMAGE_LAYOUTS, + VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_UNIFIED_IMAGE_LAYOUTS_FEATURES_KHR); +#endif + FF_VK_STRUCT_EXT(s, &feats->device, &feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR); #ifdef VK_KHR_video_maintenance2 @@ -403,6 +412,11 @@ static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceF COPY_VAL(expect_assume.shaderExpectAssume); #endif +#ifdef VK_KHR_unified_image_layouts + COPY_VAL(unified_layouts.unifiedImageLayouts); + COPY_VAL(unified_layouts.unifiedImageLayoutsVideo); +#endif + #ifdef VK_KHR_internally_synchronized_queues COPY_VAL(internal_queue_sync.internallySynchronizedQueues); #endif @@ -722,6 +736,9 @@ static const VulkanOptExtension optional_device_exts[] = { #endif #ifdef VK_KHR_shader_expect_assume { VK_KHR_SHADER_EXPECT_ASSUME_EXTENSION_NAME, FF_VK_EXT_EXPECT_ASSUME }, +#endif +#ifdef VK_KHR_unified_image_layouts + { VK_KHR_UNIFIED_IMAGE_LAYOUTS_EXTENSION_NAME, FF_VK_EXT_UNIFIED_IMAGE_LAYOUTS }, #endif { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 }, #ifdef VK_KHR_video_maintenance2 @@ -2887,8 +2904,10 @@ static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size) if (err) goto fail; - if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) && - !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)) + if (hwctx->flags & AV_VK_FRAME_FLAG_GENERAL_LAYOUT) + err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL); + else if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) && + !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)) err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB); else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR) err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST); @@ -3074,6 +3093,26 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc) } } + /* Keep frames in VK_IMAGE_LAYOUT_GENERAL and skip all transitions if + * the implementation guarantees this comes with no penalty. + * Imported/exported DRM modifier images are excluded, as they may + * require queue family ownership transfers. */ +#ifdef VK_KHR_unified_image_layouts + if (!(hwctx->flags & AV_VK_FRAME_FLAG_NONE) && + (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT) && + p->feats.unified_layouts.unifiedImageLayouts && + (p->feats.unified_layouts.unifiedImageLayoutsVideo || + !(hwctx->usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR | + VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR | + VK_IMAGE_USAGE_VIDEO_DECODE_SRC_BIT_KHR | + VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR | + VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)))) { + hwctx->flags |= AV_VK_FRAME_FLAG_GENERAL_LAYOUT; + av_log(hwfc, AV_LOG_VERBOSE, "Frame context using unified (GENERAL) " + "image layouts\n"); + } +#endif + if (!hwctx->lock_frame) hwctx->lock_frame = lock_frame; -- 2.52.0 >From c8119801b913943b86808fceff6c2aa300f250b4 Mon Sep 17 00:00:00 2001 From: Lynne <[email protected]> Date: Tue, 14 Jul 2026 15:41:33 +0900 Subject: [PATCH 3/3] vulkan: add read-only frame dependencies Executions which only read from a frame now wait on the frame's semaphores without signalling them or publishing new values, allowing any number of executions to read from the same frame concurrently without serializing against one another. Read-after-write ordering is preserved by waiting on the producer's published value, and write-after-read protection is provided by the AVFrame reference each execution holds until its fence completes, which also keeps the frame unwritable via av_frame_is_writable() while reads are in flight. Only affects AV_VK_FRAME_FLAG_GENERAL_LAYOUT frames; regular frames retain the existing behavior. Used for video decode reference frames, video encode source and reference pictures, filter inputs, and swscale source frames. --- libavcodec/vulkan_decode.c | 3 ++ libavcodec/vulkan_encode.c | 14 ++++---- libavfilter/vulkan_filter.c | 21 +++++++----- libavutil/vulkan.c | 65 ++++++++++++++++++++++++++++++------- libavutil/vulkan.h | 13 ++++++++ libswscale/vulkan/ops.c | 7 ++-- 6 files changed, 94 insertions(+), 29 deletions(-) diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c index 766435ea4d..a5ef9b30f8 100644 --- a/libavcodec/vulkan_decode.c +++ b/libavcodec/vulkan_decode.c @@ -577,6 +577,9 @@ int ff_vk_decode_frame(AVCodecContext *avctx, FFVulkanDecodePicture *rvp = rvkp[i]; AVFrame *ref = rvp->dpb_frame ? rvp->dpb_frame : ref_frame; + /* This cannot be a read-only dependency: the semaphore value + * mirrored below gates the destruction of rvp's image views in + * ff_vk_decode_free_frame(), so it must cover this submission. */ err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref, VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR); diff --git a/libavcodec/vulkan_encode.c b/libavcodec/vulkan_encode.c index 1e69c73242..60e7b83b77 100644 --- a/libavcodec/vulkan_encode.c +++ b/libavcodec/vulkan_encode.c @@ -363,9 +363,10 @@ static int vulkan_encode_issue(AVCodecContext *avctx, goto fail; /* Source image */ - err = ff_vk_exec_add_dep_frame(&ctx->s, exec, src, - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR); + err = ff_vk_exec_add_dep_frame_flags(&ctx->s, exec, src, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR, + FF_VK_EXEC_DEP_READ_ONLY); if (err < 0) goto fail; @@ -405,9 +406,10 @@ static int vulkan_encode_issue(AVCodecContext *avctx, for (int i = 0; i < MAX_REFERENCE_LIST_NUM; i++) { for (int j = 0; j < base_pic->nb_refs[i]; j++) { FFHWBaseEncodePicture *ref = base_pic->refs[i][j]; - err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref->recon_image, - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR); + err = ff_vk_exec_add_dep_frame_flags(&ctx->s, exec, ref->recon_image, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_VIDEO_ENCODE_BIT_KHR, + FF_VK_EXEC_DEP_READ_ONLY); if (err < 0) return err; } diff --git a/libavfilter/vulkan_filter.c b/libavfilter/vulkan_filter.c index 2fc467c6d9..1a61c94d41 100644 --- a/libavfilter/vulkan_filter.c +++ b/libavfilter/vulkan_filter.c @@ -266,9 +266,10 @@ int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e, VK_IMAGE_LAYOUT_GENERAL, VK_NULL_HANDLE); if (in_f) { - RET(ff_vk_exec_add_dep_frame(vkctx, exec, in_f, - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); + RET(ff_vk_exec_add_dep_frame_flags(vkctx, exec, in_f, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + FF_VK_EXEC_DEP_READ_ONLY)); RET(ff_vk_create_imageviews(vkctx, exec, in_views, in_f, FF_VK_REP_FLOAT)); ff_vk_shader_update_img_array(vkctx, exec, shd, in_f, in_views, 0, 0, in_layout, @@ -334,9 +335,10 @@ int ff_vk_filter_process_2pass(FFVulkanContext *vkctx, FFVkExecPool *e, FFVkExecContext *exec = ff_vk_exec_get(vkctx, e); ff_vk_exec_start(vkctx, exec); - RET(ff_vk_exec_add_dep_frame(vkctx, exec, in, - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); + RET(ff_vk_exec_add_dep_frame_flags(vkctx, exec, in, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + FF_VK_EXEC_DEP_READ_ONLY)); RET(ff_vk_exec_add_dep_frame(vkctx, exec, tmp, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); @@ -432,9 +434,10 @@ int ff_vk_filter_process_Nin(FFVulkanContext *vkctx, FFVkExecPool *e, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); RET(ff_vk_create_imageviews(vkctx, exec, out_views, out, FF_VK_REP_FLOAT)); for (int i = 0; i < nb_in; i++) { - RET(ff_vk_exec_add_dep_frame(vkctx, exec, in[i], - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT)); + RET(ff_vk_exec_add_dep_frame_flags(vkctx, exec, in[i], + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + FF_VK_EXEC_DEP_READ_ONLY)); RET(ff_vk_create_imageviews(vkctx, exec, in_views[i], in[i], FF_VK_REP_FLOAT)); } diff --git a/libavutil/vulkan.c b/libavutil/vulkan.c index ab29741f55..e2225f8030 100644 --- a/libavutil/vulkan.c +++ b/libavutil/vulkan.c @@ -321,6 +321,7 @@ void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool) av_free(e->frame_locked); av_free(e->frame_wait_base); av_free(e->frame_sig_base); + av_free(e->frame_dep_read); av_free(e->gen_mirror_dst); av_free(e->gen_mirror_dep); av_free(e->sem_sig); @@ -812,6 +813,14 @@ fail: int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage) +{ + return ff_vk_exec_add_dep_frame_flags(s, e, f, wait_stage, signal_stage, 0); +} + +int ff_vk_exec_add_dep_frame_flags(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, + VkPipelineStageFlagBits2 wait_stage, + VkPipelineStageFlagBits2 signal_stage, + int flags) { uint8_t *frame_locked; uint8_t *frame_update; @@ -822,16 +831,38 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkAccessFlagBits *access_dst; int *frame_wait_base; int *frame_sig_base; + uint8_t *frame_dep_read; AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data; AVVulkanFramesContext *vkfc = hwfc->hwctx; AVVkFrame *vkf = (AVVkFrame *)f->data[0]; int nb_images = ff_vk_count_images(vkf); - /* Don't add duplicates */ - for (int i = 0; i < e->nb_frame_deps; i++) - if (e->frame_deps[i]->data[0] == f->data[0]) + int read_only = !!(flags & FF_VK_EXEC_DEP_READ_ONLY) && ff_vk_frame_is_general(f); + + /* Don't add duplicates; upgrade read-only dependencies to full + * dependencies if needed */ + for (int i = 0; i < e->nb_frame_deps; i++) { + if (e->frame_deps[i]->data[0] == f->data[0]) { + if (e->frame_dep_read[i] && !read_only) { + AVVkFrame *dvkf = (AVVkFrame *)f->data[0]; + int dep_images = ff_vk_count_images(dvkf); + e->frame_sig_base[i] = e->sem_sig_cnt; + for (int j = 0; j < dep_images; j++) { + VkSemaphoreSubmitInfo *sem_sig; + ARR_REALLOC(e, sem_sig, &e->sem_sig_alloc, e->sem_sig_cnt); + e->sem_sig[e->sem_sig_cnt++] = (VkSemaphoreSubmitInfo) { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, + .semaphore = dvkf->sem[j], + .value = 0, + .stageMask = signal_stage, + }; + } + e->frame_dep_read[i] = 0; + } return 1; + } + } ARR_REALLOC(e, layout_dst, &e->layout_dst_alloc, e->nb_frame_deps); ARR_REALLOC(e, queue_family_dst, &e->queue_family_dst_alloc, e->nb_frame_deps); @@ -842,6 +873,7 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, ARR_REALLOC(e, frame_deps, &e->frame_deps_alloc_size, e->nb_frame_deps); ARR_REALLOC(e, frame_wait_base, &e->frame_wait_base_alloc, e->nb_frame_deps); ARR_REALLOC(e, frame_sig_base, &e->frame_sig_base_alloc, e->nb_frame_deps); + ARR_REALLOC(e, frame_dep_read, &e->frame_dep_read_alloc_size, e->nb_frame_deps); /* prepare_frame in hwcontext_vulkan.c uses the regular frame management * code but has no frame yet, and it doesn't need to actually store a ref @@ -858,7 +890,8 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, e->frame_deps[e->nb_frame_deps] = f; e->frame_wait_base[e->nb_frame_deps] = e->sem_wait_cnt; - e->frame_sig_base[e->nb_frame_deps] = e->sem_sig_cnt; + e->frame_sig_base[e->nb_frame_deps] = read_only ? -1 : e->sem_sig_cnt; + e->frame_dep_read[e->nb_frame_deps] = read_only; /* GENERAL_LAYOUT frames are not locked during recording; their * semaphore values are assigned at submission time */ @@ -877,8 +910,6 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, uint64_t **sem_sig_val_dst; ARR_REALLOC(e, sem_wait, &e->sem_wait_alloc, e->sem_wait_cnt); - ARR_REALLOC(e, sem_sig, &e->sem_sig_alloc, e->sem_sig_cnt); - ARR_REALLOC(e, sem_sig_val_dst, &e->sem_sig_val_dst_alloc, e->sem_sig_val_dst_cnt); e->sem_wait[e->sem_wait_cnt++] = (VkSemaphoreSubmitInfo) { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, @@ -887,6 +918,12 @@ int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, .stageMask = wait_stage, }; + if (read_only) + continue; + + ARR_REALLOC(e, sem_sig, &e->sem_sig_alloc, e->sem_sig_cnt); + ARR_REALLOC(e, sem_sig_val_dst, &e->sem_sig_val_dst_alloc, e->sem_sig_val_dst_cnt); + e->sem_sig[e->sem_sig_cnt++] = (VkSemaphoreSubmitInfo) { .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO, .semaphore = vkf->sem[i], @@ -1006,15 +1043,21 @@ int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e) vkfc->lock_frame(hwfc, vkf); for (int i = 0; i < nb_images; i++) { e->sem_wait[e->frame_wait_base[j] + i].value = vkf->sem_value[i]; - e->sem_sig[e->frame_sig_base[j] + i].value = vkf->sem_value[i] + 1; - vkf->sem_value[i]++; + if (e->frame_sig_base[j] >= 0) { + e->sem_sig[e->frame_sig_base[j] + i].value = vkf->sem_value[i] + 1; + vkf->sem_value[i]++; + } } vkfc->unlock_frame(hwfc, vkf); } /* Fill in deferred semaphore value mirrors */ - for (int j = 0; j < e->gen_mirror_cnt; j++) - *e->gen_mirror_dst[j] = e->sem_sig[e->frame_sig_base[e->gen_mirror_dep[j]]].value; + for (int j = 0; j < e->gen_mirror_cnt; j++) { + int dep = e->gen_mirror_dep[j]; + *e->gen_mirror_dst[j] = e->frame_sig_base[dep] >= 0 ? + e->sem_sig[e->frame_sig_base[dep]].value : + e->sem_wait[e->frame_wait_base[dep]].value; + } #if FF_API_VULKAN_SYNC_QUEUES FF_DISABLE_DEPRECATION_WARNINGS @@ -1037,7 +1080,7 @@ FF_ENABLE_DEPRECATION_WARNINGS for (int j = 0; j < e->nb_frame_deps; j++) { AVVkFrame *vkf = (AVVkFrame *)e->frame_deps[j]->data[0]; int nb_images = ff_vk_count_images(vkf); - if (e->frame_locked[j]) + if (e->frame_locked[j] || e->frame_sig_base[j] < 0) continue; for (int i = 0; i < nb_images; i++) { VkSemaphoreSignalInfo sig_info = { diff --git a/libavutil/vulkan.h b/libavutil/vulkan.h index 97407da417..84a3f09246 100644 --- a/libavutil/vulkan.h +++ b/libavutil/vulkan.h @@ -160,6 +160,9 @@ typedef struct FFVkExecContext { uint8_t *frame_locked; unsigned int frame_locked_alloc_size; + uint8_t *frame_dep_read; + unsigned int frame_dep_read_alloc_size; + /* Base indices into sem_wait/sem_sig for each frame dep, for * submission-time semaphore value assignment of GENERAL_LAYOUT frames */ int *frame_wait_base; @@ -501,9 +504,19 @@ int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, int wait); /* Ownership transferred if !wait */ int ff_vk_frame_is_general(AVFrame *f); +/* The execution only reads from the frame: its semaphores are waited upon, + * but not signalled, letting multiple executions read the frame concurrently. + * Only affects AV_VK_FRAME_FLAG_GENERAL_LAYOUT frames; adding the same frame + * again without this flag upgrades it to a regular (writing) dependency. */ +#define FF_VK_EXEC_DEP_READ_ONLY (1 << 0) + int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage); +int ff_vk_exec_add_dep_frame_flags(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, + VkPipelineStageFlagBits2 wait_stage, + VkPipelineStageFlagBits2 signal_stage, + int flags); int ff_vk_exec_add_dep_sw_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f); void ff_vk_exec_update_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, diff --git a/libswscale/vulkan/ops.c b/libswscale/vulkan/ops.c index 8fc16274a3..f235049add 100644 --- a/libswscale/vulkan/ops.c +++ b/libswscale/vulkan/ops.c @@ -103,9 +103,10 @@ static void process(const SwsFrame *dst, const SwsFrame *src, int y, int h, AVFrame *src_f = (AVFrame *) src->avframe; AVFrame *dst_f = (AVFrame *) dst->avframe; - ff_vk_exec_add_dep_frame(&p->s->vkctx, ec, src_f, - VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, - VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); + ff_vk_exec_add_dep_frame_flags(&p->s->vkctx, ec, src_f, + VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, + VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT, + FF_VK_EXEC_DEP_READ_ONLY); ff_vk_exec_add_dep_frame(&p->s->vkctx, ec, dst_f, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_2_COMPUTE_SHADER_BIT); -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
