This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit 4e141536767ef57f0aaf1f5f33bfed3001d68d7d Author: Lynne <[email protected]> AuthorDate: Sun Jul 26 16:18:12 2026 +0800 Commit: Lynne <[email protected]> CommitDate: Wed Aug 5 16:04:37 2026 +0900 vulkan: manage image views with AVRefStruct The biggest gain is no longer needing to spin on the CPU on a semaphore waiting for the image to become freeable. This was awful. --- libavcodec/vulkan_av1.c | 4 +-- libavcodec/vulkan_decode.c | 84 +++++++++++++++++----------------------------- libavcodec/vulkan_decode.h | 14 ++++---- libavcodec/vulkan_ffv1.c | 18 +++++++++- libavcodec/vulkan_h264.c | 4 +-- libavcodec/vulkan_hevc.c | 4 +-- libavcodec/vulkan_vp9.c | 4 +-- libavutil/vulkan.c | 50 +++++++++++++++++++++++---- libavutil/vulkan.h | 19 +++++++++++ 9 files changed, 126 insertions(+), 75 deletions(-) diff --git a/libavcodec/vulkan_av1.c b/libavcodec/vulkan_av1.c index 08c371746e..d2e1645a22 100644 --- a/libavcodec/vulkan_av1.c +++ b/libavcodec/vulkan_av1.c @@ -123,7 +123,7 @@ static int vk_av1_fill_pict(AVCodecContext *avctx, const AV1Frame **ref_src, .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height }, .baseArrayLayer = ((has_grain || dec->dedicated_dpb) && ctx->common.layered_dpb) ? hp->frame_id : 0, - .imageViewBinding = vkpic->view.ref[0], + .imageViewBinding = vkpic->view.ref, }; *ref_slot = (VkVideoReferenceSlotInfoKHR) { @@ -361,7 +361,7 @@ static int vk_av1_start_frame(AVCodecContext *avctx, .codedOffset = (VkOffset2D){ 0, 0 }, .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height }, .baseArrayLayer = 0, - .imageViewBinding = vp->view.out[0], + .imageViewBinding = vp->view.out, }, }; diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c index 970bf46c93..781e71b5c5 100644 --- a/libavcodec/vulkan_decode.c +++ b/libavcodec/vulkan_decode.c @@ -184,14 +184,10 @@ static void init_frame(FFVulkanDecodeContext *dec, FFVulkanDecodePicture *vkpic) FFVulkanFunctions *vk = &ctx->s.vkfn; vkpic->dpb_frame = NULL; - for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) { - vkpic->view.ref[i] = VK_NULL_HANDLE; - vkpic->view.out[i] = VK_NULL_HANDLE; - vkpic->view.dst[i] = VK_NULL_HANDLE; - } + vkpic->out_views = NULL; + vkpic->view.ref = VK_NULL_HANDLE; + vkpic->view.out = VK_NULL_HANDLE; - vkpic->destroy_image_view = vk->DestroyImageView; - vkpic->wait_semaphores = vk->WaitSemaphores; vkpic->invalidate_memory_ranges = vk->InvalidateMappedMemoryRanges; } @@ -206,14 +202,20 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, /* If the decoder made a blank frame to make up for a missing ref, or the * frame is the current frame so it's missing one, create a re-representation */ - if (vkpic->view.ref[0]) + if (vkpic->view.ref) return 0; init_frame(dec, vkpic); + /* Slot 0 holds the output view, slot 1 the DISTINCT-mode reference + * view; refcounted, so executions keep them alive past the picture */ + vkpic->out_views = ff_vk_imageviews_alloc(&ctx->s, 2); + if (!vkpic->out_views) + return AVERROR(ENOMEM); + if (ctx->common.layered_dpb && alloc_dpb) { - vkpic->view.ref[0] = ctx->common.layered_view; - vkpic->view.aspect_ref[0] = ctx->common.layered_aspect; + vkpic->view.ref = ctx->common.layered_view; + vkpic->view.aspect_ref = ctx->common.layered_aspect; } else if (alloc_dpb) { AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->common.dpb_hwfc_ref->data; AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx; @@ -223,13 +225,13 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, return AVERROR(ENOMEM); err = ff_vk_create_view(&ctx->s, &ctx->common, - &vkpic->view.ref[0], &vkpic->view.aspect_ref[0], + &vkpic->out_views->views[1], &vkpic->view.aspect_ref, (AVVkFrame *)vkpic->dpb_frame->data[0], dpb_hwfc->format[0], VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR); if (err < 0) return err; - vkpic->view.dst[0] = vkpic->view.ref[0]; + vkpic->view.ref = vkpic->out_views->views[1]; } if (!alloc_dpb || is_current) { @@ -237,7 +239,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, AVVulkanFramesContext *hwfc = frames->hwctx; err = ff_vk_create_view(&ctx->s, &ctx->common, - &vkpic->view.out[0], &vkpic->view.aspect[0], + &vkpic->out_views->views[0], &vkpic->view.aspect, (AVVkFrame *)pic->data[0], hwfc->format[0], VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR | @@ -245,10 +247,11 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, // the above fixes VUID-VkVideoBeginCodingInfoKHR-slotIndex-07245 if (err < 0) return err; + vkpic->view.out = vkpic->out_views->views[0]; if (!alloc_dpb) { - vkpic->view.ref[0] = vkpic->view.out[0]; - vkpic->view.aspect_ref[0] = vkpic->view.aspect[0]; + vkpic->view.ref = vkpic->view.out; + vkpic->view.aspect_ref = vkpic->view.aspect; } } @@ -525,10 +528,9 @@ int ff_vk_decode_frame(AVCodecContext *avctx, if (err < 0) return err; - err = ff_vk_exec_mirror_sem_value(&ctx->s, exec, &vp->sem, &vp->sem_value, - pic); - if (err < 0) - return err; + /* The output view is kept alive by the execution context; freeing the + * picture then needs no host wait. */ + ff_vk_exec_add_dep_refstruct(&ctx->s, exec, vp->out_views); /* Output image - change layout, as it comes from a pool */ img_bar[nb_img_bar] = (VkImageMemoryBarrier2) { @@ -546,7 +548,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = vkf->img[0], .subresourceRange = (VkImageSubresourceRange) { - .aspectMask = vp->view.aspect[0], + .aspectMask = vp->view.aspect, .layerCount = 1, .levelCount = 1, }, @@ -577,13 +579,10 @@ int ff_vk_decode_frame(AVCodecContext *avctx, if (err < 0) return err; - if (err == 0) { - err = ff_vk_exec_mirror_sem_value(&ctx->s, exec, - &rvp->sem, &rvp->sem_value, - ref); - if (err < 0) - return err; - } + /* The reference's image views are kept alive by the execution, + * so freeing the picture needs no host wait. */ + if (err == 0 && rvp->out_views) + ff_vk_exec_add_dep_refstruct(&ctx->s, exec, rvp->out_views); if (!rvp->dpb_frame) { AVVkFrame *rvkf = (AVVkFrame *)ref->data[0]; @@ -602,7 +601,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx, .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED, .image = rvkf->img[0], .subresourceRange = (VkImageSubresourceRange) { - .aspectMask = rvp->view.aspect_ref[0], + .aspectMask = rvp->view.aspect_ref, .layerCount = 1, .levelCount = 1, }, @@ -612,7 +611,7 @@ int ff_vk_decode_frame(AVCodecContext *avctx, } } } else if (vp->decode_info.referenceSlotCount || - vp->view.out[0] != vp->view.ref[0]) { + vp->view.out != vp->view.ref) { /* Single barrier for a single layered ref */ err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->common.layered_frame, VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR, @@ -640,33 +639,12 @@ int ff_vk_decode_frame(AVCodecContext *avctx, void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp) { - AVVulkanDeviceContext *hwctx = dev_ctx->hwctx; - - VkSemaphoreWaitInfo sem_wait = (VkSemaphoreWaitInfo) { - .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, - .pSemaphores = &vp->sem, - .pValues = &vp->sem_value, - .semaphoreCount = 1, - }; - - /* We do not have to lock the frame here because we're not interested - * in the actual current semaphore value, but only that it's later than - * the time we submitted the image for decoding. */ - if (vp->sem) - vp->wait_semaphores(hwctx->act_dev, &sem_wait, UINT64_MAX); - /* Free slices data */ av_refstruct_unref(&vp->slices_buf); - /* Destroy image view (out) */ - for (int i = 0; i < AV_NUM_DATA_POINTERS; i++) { - if (vp->view.out[i] && vp->view.out[i] != vp->view.dst[i]) - vp->destroy_image_view(hwctx->act_dev, vp->view.out[i], hwctx->alloc); - - /* Destroy image view (ref, unlayered) */ - if (vp->view.dst[i]) - vp->destroy_image_view(hwctx->act_dev, vp->view.dst[i], hwctx->alloc); - } + /* The views are kept alive by every execution referencing them, so no + * host wait is needed. */ + av_refstruct_unref(&vp->out_views); av_frame_free(&vp->dpb_frame); } diff --git a/libavcodec/vulkan_decode.h b/libavcodec/vulkan_decode.h index 5c3837cb90..8c6537194f 100644 --- a/libavcodec/vulkan_decode.h +++ b/libavcodec/vulkan_decode.h @@ -74,11 +74,10 @@ typedef struct FFVulkanDecodePicture { AVFrame *dpb_frame; /* Only used for out-of-place decoding. */ struct { - VkImageView ref[AV_NUM_DATA_POINTERS]; /* Image representation view (reference) */ - VkImageView out[AV_NUM_DATA_POINTERS]; /* Image representation view (output-only) */ - VkImageView dst[AV_NUM_DATA_POINTERS]; /* Set to img_view_out if no layered refs are used */ - VkImageAspectFlags aspect[AV_NUM_DATA_POINTERS]; /* Image plane mask bits */ - VkImageAspectFlags aspect_ref[AV_NUM_DATA_POINTERS]; /* Only used for out-of-place decoding */ + VkImageView ref; /* Image representation view (reference) */ + VkImageView out; /* Image representation view (output-only) */ + VkImageAspectFlags aspect; /* Image plane mask bits */ + VkImageAspectFlags aspect_ref; /* Only used for out-of-place decoding */ } view; VkSemaphore sem; @@ -95,13 +94,14 @@ typedef struct FFVulkanDecodePicture { /* Main decoding struct */ VkVideoDecodeInfoKHR decode_info; + /* Owner of the output image views aliased by view.out/ref */ + FFVkImageViews *out_views; + /* Slice data */ FFVkBuffer *slices_buf; size_t slices_size; /* Vulkan functions needed for destruction, as no other context is guaranteed to exist */ - PFN_vkWaitSemaphores wait_semaphores; - PFN_vkDestroyImageView destroy_image_view; PFN_vkInvalidateMappedMemoryRanges invalidate_memory_ranges; } FFVulkanDecodePicture; diff --git a/libavcodec/vulkan_ffv1.c b/libavcodec/vulkan_ffv1.c index b482df1e47..078b91f11f 100644 --- a/libavcodec/vulkan_ffv1.c +++ b/libavcodec/vulkan_ffv1.c @@ -78,6 +78,10 @@ typedef struct FFv1VulkanDecodePicture { uint32_t *slice_offset; int slice_num; int crc_checked; + + /* The context-less free callback waits for the decode before reading + * back the slice feedback */ + PFN_vkWaitSemaphores wait_semaphores; } FFv1VulkanDecodePicture; typedef struct FFv1VulkanDecodeContext { @@ -139,7 +143,7 @@ static int vk_ffv1_start_frame(AVCodecContext *avctx, /* The context-less free callback needs these device functions, which * prepare_frame_sdr() used to set. vp->sem is kept for the next * non-keyframe's wait and the free callback's CRC readback. */ - vp->wait_semaphores = ctx->s.vkfn.WaitSemaphores; + fp->wait_semaphores = ctx->s.vkfn.WaitSemaphores; vp->invalidate_memory_ranges = ctx->s.vkfn.InvalidateMappedMemoryRanges; /* Host map the input slices data if supported */ @@ -940,6 +944,18 @@ static void vk_ffv1_free_frame_priv(AVRefStructOpaque _hwctx, void *data) FFv1VulkanDecodePicture *fp = data; FFVulkanDecodePicture *vp = &fp->vp; + /* The feedback below is read back on the host: wait for the decode. + * The generic free path no longer waits for anything. */ + if (vp->sem) { + VkSemaphoreWaitInfo sem_wait = { + .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO, + .pSemaphores = &vp->sem, + .pValues = &vp->sem_value, + .semaphoreCount = 1, + }; + fp->wait_semaphores(hwctx->act_dev, &sem_wait, UINT64_MAX); + } + ff_vk_decode_free_frame(dev_ctx, vp); /* No feedback to read if setup failed or the decode was never submitted */ diff --git a/libavcodec/vulkan_h264.c b/libavcodec/vulkan_h264.c index 5fc0f4c4cf..27f9a97102 100644 --- a/libavcodec/vulkan_h264.c +++ b/libavcodec/vulkan_h264.c @@ -98,7 +98,7 @@ static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src, .codedOffset = (VkOffset2D){ 0, 0 }, .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height }, .baseArrayLayer = ctx->common.layered_dpb ? dpb_slot_index : 0, - .imageViewBinding = vkpic->view.ref[0], + .imageViewBinding = vkpic->view.ref, }; *ref_slot = (VkVideoReferenceSlotInfoKHR) { @@ -466,7 +466,7 @@ static int vk_h264_start_frame(AVCodecContext *avctx, .codedOffset = (VkOffset2D){ 0, 0 }, .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height }, .baseArrayLayer = 0, - .imageViewBinding = vp->view.out[0], + .imageViewBinding = vp->view.out, }, }; diff --git a/libavcodec/vulkan_hevc.c b/libavcodec/vulkan_hevc.c index 9b9264f1cd..3a3df7d7c1 100644 --- a/libavcodec/vulkan_hevc.c +++ b/libavcodec/vulkan_hevc.c @@ -164,7 +164,7 @@ static int vk_hevc_fill_pict(AVCodecContext *avctx, HEVCFrame **ref_src, .codedOffset = (VkOffset2D){ 0, 0 }, .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height }, .baseArrayLayer = ctx->common.layered_dpb ? pic_id : 0, - .imageViewBinding = vkpic->view.ref[0], + .imageViewBinding = vkpic->view.ref, }; *ref_slot = (VkVideoReferenceSlotInfoKHR) { @@ -818,7 +818,7 @@ static int vk_hevc_start_frame(AVCodecContext *avctx, .codedOffset = (VkOffset2D){ 0, 0 }, .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height }, .baseArrayLayer = 0, - .imageViewBinding = vp->view.out[0], + .imageViewBinding = vp->view.out, }, }; diff --git a/libavcodec/vulkan_vp9.c b/libavcodec/vulkan_vp9.c index 5aeec3fb21..26379571e1 100644 --- a/libavcodec/vulkan_vp9.c +++ b/libavcodec/vulkan_vp9.c @@ -73,7 +73,7 @@ static int vk_vp9_fill_pict(AVCodecContext *avctx, const VP9Frame **ref_src, .codedExtent = (VkExtent2D){ pic->tf.f->width, pic->tf.f->height }, .baseArrayLayer = (dec->dedicated_dpb && ctx->common.layered_dpb) ? hp->frame_id : 0, - .imageViewBinding = vkpic->view.ref[0], + .imageViewBinding = vkpic->view.ref, }; *ref_slot = (VkVideoReferenceSlotInfoKHR) { @@ -295,7 +295,7 @@ static int vk_vp9_start_frame(AVCodecContext *avctx, .codedOffset = (VkOffset2D){ 0, 0 }, .codedExtent = (VkExtent2D){ pic->tf.f->width, pic->tf.f->height }, .baseArrayLayer = 0, - .imageViewBinding = vp->view.out[0], + .imageViewBinding = vp->view.out, }, }; diff --git a/libavutil/vulkan.c b/libavutil/vulkan.c index 5dd161cae6..15777c70fc 100644 --- a/libavutil/vulkan.c +++ b/libavutil/vulkan.c @@ -145,6 +145,8 @@ static void load_enabled_qfs(FFVulkanContext *s) } } +static void reset_imageviews(AVRefStructOpaque unused, void *obj); + int ff_vk_load_props(FFVulkanContext *s) { FFVulkanFunctions *vk = &s->vkfn; @@ -185,6 +187,14 @@ int ff_vk_load_props(FFVulkanContext *s) FF_VK_STRUCT_EXT(s, &s->feats, &s->atomic_float_feats, FF_VK_EXT_ATOMIC_FLOAT, VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT); + if (!s->imageviews_pool) { + s->imageviews_pool = av_refstruct_pool_alloc_ext(sizeof(FFVkImageViews), 0, + NULL, NULL, reset_imageviews, + NULL, NULL); + if (!s->imageviews_pool) + return AVERROR(ENOMEM); + } + /* Try allocating 1024 layouts */ s->host_image_copy_layouts = av_malloc(sizeof(*s->host_image_copy_layouts)*1024); s->host_image_props.pCopySrcLayouts = s->host_image_copy_layouts; @@ -1785,6 +1795,37 @@ const char *ff_vk_shader_rep_fmt(enum AVPixelFormat pix_fmt, } } +static void reset_imageviews(AVRefStructOpaque unused, void *obj) +{ + FFVkImageViews *iv = obj; + + for (int i = 0; i < iv->nb_views; i++) { + if (iv->views[i]) + iv->destroy_image_view(iv->dev, iv->views[i], iv->alloc); + } + + memset(iv->views, 0, sizeof(iv->views)); +} + +FFVkImageViews *ff_vk_imageviews_alloc(FFVulkanContext *s, int nb_views) +{ + FFVulkanFunctions *vk = &s->vkfn; + FFVkImageViews *iv; + + av_assert1(nb_views <= AV_NUM_DATA_POINTERS); + + iv = av_refstruct_pool_get(s->imageviews_pool); + if (!iv) + return NULL; + + iv->nb_views = nb_views; + iv->dev = s->hwctx->act_dev; + iv->alloc = s->hwctx->alloc; + iv->destroy_image_view = vk->DestroyImageView; + + return iv; +} + static VkFormat map_fmt_to_rep(VkFormat fmt, enum FFVkShaderRepFormat rep_fmt) { #define REPS_FMT(fmt) \ @@ -1939,9 +1980,8 @@ int ff_vk_create_imageviews(FFVulkanContext *s, FFVkExecContext *e, VkImageView views[AV_NUM_DATA_POINTERS], AVFrame *f, enum FFVkShaderRepFormat rep_fmt) { - int err; + int err = 0; VkResult ret; - AVBufferRef *buf; FFVulkanFunctions *vk = &s->vkfn; AVHWFramesContext *hwfc = (AVHWFramesContext *)f->hw_frames_ctx->data; AVVulkanFramesContext *vkfc = hwfc->hwctx; @@ -1998,13 +2038,10 @@ int ff_vk_create_imageviews(FFVulkanContext *s, FFVkExecContext *e, return 0; - return err; - fail: - for (int i = 0; i < nb_planes; i++) { + for (int i = 0; i < nb_planes; i++) if (tmp_views[i]) vk->DestroyImageView(s->hwctx->act_dev, tmp_views[i], s->hwctx->alloc); - } return err; } @@ -2626,6 +2663,7 @@ void ff_vk_uninit(FFVulkanContext *s) av_freep(&s->video_props); av_freep(&s->coop_mat_props); av_freep(&s->host_image_copy_layouts); + av_refstruct_pool_uninit(&s->imageviews_pool); av_buffer_unref(&s->device_ref); av_buffer_unref(&s->frames_ref); diff --git a/libavutil/vulkan.h b/libavutil/vulkan.h index c6dfa5050b..49d3fbce93 100644 --- a/libavutil/vulkan.h +++ b/libavutil/vulkan.h @@ -317,6 +317,8 @@ typedef struct FFVulkanContext { uint32_t coop_mat_props_nb; VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float_feats; + AVRefStructPool *imageviews_pool; + VkPhysicalDeviceVulkan12Features feats_12; VkPhysicalDeviceFeatures2 feats; @@ -535,6 +537,23 @@ int ff_vk_create_imageview(FFVulkanContext *s, VkImageView *img_view, VkImageAspectFlags *aspect, AVFrame *f, int plane, enum FFVkShaderRepFormat rep_fmt); +/* Refcounted set of image views; stashes destruction handles to be context-free */ +typedef struct FFVkImageViews { + int nb_views; + + VkDevice dev; + const VkAllocationCallbacks *alloc; + PFN_vkDestroyImageView destroy_image_view; + + VkImageView views[AV_NUM_DATA_POINTERS]; +} FFVkImageViews; + +/** + * Allocate a reference-counted set of image views, zero-initialized for the + * caller to create. + */ +FFVkImageViews *ff_vk_imageviews_alloc(FFVulkanContext *s, int nb_views); + /** * Create an imageview and add it as a dependency to an execution. */ _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
