PR #21323 opened by bcheng URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21323 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/21323.patch
Gets rid of some validation issues, and removes the useless ycbcr sampler entirely. >From e41e21a1ec42a30f2d97a1a1c092797d4f9328b4 Mon Sep 17 00:00:00 2001 From: Benjamin Cheng <[email protected]> Date: Tue, 30 Dec 2025 14:15:43 -0500 Subject: [PATCH 1/4] avutil/hwcontext_vulkan: Limit usages for lone DPBs Lone DPBs will only be requested when the implementation requires a separate DPB. These images are never given out, and are never used for anything other than a DPB. Also most implementations requiring these won't support any other usages on DPBs, so limiting the usages would fix some validation errors. --- libavutil/hwcontext_vulkan.c | 59 +++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c index aa5f72e7f2..182eabceb0 100644 --- a/libavutil/hwcontext_vulkan.c +++ b/libavutil/hwcontext_vulkan.c @@ -2957,6 +2957,9 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc) const struct FFVkFormatEntry *fmt; int disable_multiplane = p->disable_multiplane || (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE); + int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) || + ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) && + !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))); /* Defaults */ if (!hwctx->nb_layers) @@ -3009,37 +3012,37 @@ static int vulkan_frames_init(AVHWFramesContext *hwfc) return err; } - /* Image usage flags */ - hwctx->usage |= supported_usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT | - VK_IMAGE_USAGE_TRANSFER_SRC_BIT | - VK_IMAGE_USAGE_STORAGE_BIT | - VK_IMAGE_USAGE_SAMPLED_BIT); + /* Lone DPB images do not need additional flags. */ + if (!is_lone_dpb) { + /* Image usage flags */ + hwctx->usage |= supported_usage & (VK_IMAGE_USAGE_TRANSFER_DST_BIT | + VK_IMAGE_USAGE_TRANSFER_SRC_BIT | + VK_IMAGE_USAGE_STORAGE_BIT | + VK_IMAGE_USAGE_SAMPLED_BIT); - if (p->vkctx.extensions & FF_VK_EXT_HOST_IMAGE_COPY && - !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) && - !(p->dprops.driverID == VK_DRIVER_ID_MOLTENVK)) - hwctx->usage |= supported_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT; + if (p->vkctx.extensions & FF_VK_EXT_HOST_IMAGE_COPY && + !(p->dprops.driverID == VK_DRIVER_ID_NVIDIA_PROPRIETARY) && + !(p->dprops.driverID == VK_DRIVER_ID_MOLTENVK)) + hwctx->usage |= supported_usage & VK_IMAGE_USAGE_HOST_TRANSFER_BIT_EXT; - /* Enables encoding of images, if supported by format and extensions */ - if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) && - (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE | - FF_VK_EXT_VIDEO_MAINTENANCE_1))) - hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR; + /* Enables encoding of images, if supported by format and extensions */ + if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) && + (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE | + FF_VK_EXT_VIDEO_MAINTENANCE_1))) + hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR; - /* Image creation flags. - * Only fill them in automatically if the image is not going to be used as - * a DPB-only image, and we have SAMPLED/STORAGE bits set. */ - if (!hwctx->img_flags) { - int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) || - ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) && - !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))); - int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT | - VK_IMAGE_USAGE_STORAGE_BIT); - hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; - if (sampleable && !is_lone_dpb) { - hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT; - if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf)) - hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + /* Image creation flags. + * Only fill them in automatically if the image is not going to be used as + * a DPB-only image, and we have SAMPLED/STORAGE bits set. */ + if (!hwctx->img_flags) { + int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT | + VK_IMAGE_USAGE_STORAGE_BIT); + hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT; + if (sampleable) { + hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT; + if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf)) + hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT; + } } } -- 2.49.1 >From 24db09a881c9e0eb5b36dcf693c6ac17e9d56be8 Mon Sep 17 00:00:00 2001 From: Benjamin Cheng <[email protected]> Date: Tue, 30 Dec 2025 11:47:46 -0500 Subject: [PATCH 2/4] lavc/vulkan_video: Restrict usages for image views These image views are used only internally for video coding, so they do not need all the usages of the image it's created from. --- libavcodec/vulkan_decode.c | 8 +++++--- libavcodec/vulkan_encode.c | 6 +++--- libavcodec/vulkan_video.c | 9 ++++++++- libavcodec/vulkan_video.h | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c index de786ad808..31ed14f517 100644 --- a/libavcodec/vulkan_decode.c +++ b/libavcodec/vulkan_decode.c @@ -217,7 +217,7 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, err = ff_vk_create_view(&ctx->s, &ctx->common, &vkpic->view.ref[0], &vkpic->view.aspect_ref[0], (AVVkFrame *)vkpic->dpb_frame->data[0], - dpb_hwfc->format[0], !is_current); + dpb_hwfc->format[0], VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR); if (err < 0) return err; @@ -231,7 +231,9 @@ int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, err = ff_vk_create_view(&ctx->s, &ctx->common, &vkpic->view.out[0], &vkpic->view.aspect[0], (AVVkFrame *)pic->data[0], - hwfc->format[0], !is_current); + hwfc->format[0], + VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR | + (!is_current ? VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR : 0)); if (err < 0) return err; @@ -1404,7 +1406,7 @@ int ff_vk_decode_init(AVCodecContext *avctx) &ctx->common.layered_view, &ctx->common.layered_aspect, (AVVkFrame *)ctx->common.layered_frame->data[0], - s->hwfc->format[0], 1); + s->hwfc->format[0], VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR); if (err < 0) goto fail; } diff --git a/libavcodec/vulkan_encode.c b/libavcodec/vulkan_encode.c index 7b534ffa30..c820b63666 100644 --- a/libavcodec/vulkan_encode.c +++ b/libavcodec/vulkan_encode.c @@ -71,7 +71,7 @@ static int vulkan_encode_init(AVCodecContext *avctx, FFHWBaseEncodePicture *pic) /* Input image view */ err = ff_vk_create_view(&ctx->s, &ctx->common, &vp->in.view, &vp->in.aspect, - vkf, vkfc->format[0], 0); + vkf, vkfc->format[0], VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR); if (err < 0) return err; @@ -81,7 +81,7 @@ static int vulkan_encode_init(AVCodecContext *avctx, FFHWBaseEncodePicture *pic) AVVkFrame *rvkf = (AVVkFrame *)rf->data[0]; err = ff_vk_create_view(&ctx->s, &ctx->common, &vp->dpb.view, &vp->dpb.aspect, - rvkf, ctx->pic_format, 1); + rvkf, ctx->pic_format, VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR); if (err < 0) return err; } else { @@ -635,7 +635,7 @@ static int vulkan_encode_create_dpb(AVCodecContext *avctx, FFVulkanEncodeContext &ctx->common.layered_view, &ctx->common.layered_aspect, (AVVkFrame *)ctx->common.layered_frame->data[0], - hwfc->format[0], 1); + hwfc->format[0], VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR); if (err < 0) return err; diff --git a/libavcodec/vulkan_video.c b/libavcodec/vulkan_video.c index d73479d14d..8bc3e9b907 100644 --- a/libavcodec/vulkan_video.c +++ b/libavcodec/vulkan_video.c @@ -290,14 +290,21 @@ StdVideoAV1Profile ff_vk_av1_profile_to_vk(int profile) int ff_vk_create_view(FFVulkanContext *s, FFVkVideoCommon *common, VkImageView *view, VkImageAspectFlags *aspect, - AVVkFrame *src, VkFormat vkf, int is_dpb) + AVVkFrame *src, VkFormat vkf, VkImageUsageFlags usage) { VkResult ret; FFVulkanFunctions *vk = &s->vkfn; VkImageAspectFlags aspect_mask = ff_vk_aspect_bits_from_vkfmt(vkf); + int is_dpb = usage & (VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR | + VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR); + VkImageViewUsageCreateInfo usage_create_info = { + .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO, + .usage = usage, + }; VkSamplerYcbcrConversionInfo yuv_sampler_info = { .sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO, + .pNext = &usage_create_info, .conversion = common->yuv_sampler, }; VkImageViewCreateInfo img_view_create_info = { diff --git a/libavcodec/vulkan_video.h b/libavcodec/vulkan_video.h index 1b29f7adc7..14168b7cc8 100644 --- a/libavcodec/vulkan_video.h +++ b/libavcodec/vulkan_video.h @@ -85,7 +85,7 @@ StdVideoAV1Profile ff_vk_av1_profile_to_vk(int profile); */ int ff_vk_create_view(FFVulkanContext *s, FFVkVideoCommon *common, VkImageView *view, VkImageAspectFlags *aspect, - AVVkFrame *src, VkFormat vkf, int is_dpb); + AVVkFrame *src, VkFormat vkf, VkImageUsageFlags flags); /** * Initialize video session, allocating and binding necessary memory. -- 2.49.1 >From 2a0671f28ec67ce844bbeb9769ebb7967a9bdeb6 Mon Sep 17 00:00:00 2001 From: Benjamin Cheng <[email protected]> Date: Tue, 30 Dec 2025 14:11:52 -0500 Subject: [PATCH 3/4] lavc/vulkan_video: Drop sampler With the limiting of video usages on the image views, we no longer need a yuv sampler, since no multi-plane image will be created with the SAMPLED usage bit. --- libavcodec/vulkan_decode.c | 3 +-- libavcodec/vulkan_video.c | 32 +------------------------------- libavcodec/vulkan_video.h | 2 -- 3 files changed, 2 insertions(+), 35 deletions(-) diff --git a/libavcodec/vulkan_decode.c b/libavcodec/vulkan_decode.c index 31ed14f517..eb76d5b052 100644 --- a/libavcodec/vulkan_decode.c +++ b/libavcodec/vulkan_decode.c @@ -1385,8 +1385,7 @@ int ff_vk_decode_init(AVCodecContext *avctx) VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR); dpb_hwfc->format[0] = s->hwfc->format[0]; dpb_hwfc->tiling = VK_IMAGE_TILING_OPTIMAL; - dpb_hwfc->usage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR | - VK_IMAGE_USAGE_SAMPLED_BIT; /* Shuts validator up. */ + dpb_hwfc->usage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR; if (ctx->common.layered_dpb) dpb_hwfc->nb_layers = ctx->caps.maxDpbSlots; diff --git a/libavcodec/vulkan_video.c b/libavcodec/vulkan_video.c index 8bc3e9b907..9645a2de67 100644 --- a/libavcodec/vulkan_video.c +++ b/libavcodec/vulkan_video.c @@ -302,14 +302,9 @@ int ff_vk_create_view(FFVulkanContext *s, FFVkVideoCommon *common, .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO, .usage = usage, }; - VkSamplerYcbcrConversionInfo yuv_sampler_info = { - .sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO, - .pNext = &usage_create_info, - .conversion = common->yuv_sampler, - }; VkImageViewCreateInfo img_view_create_info = { .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO, - .pNext = &yuv_sampler_info, + .pNext = &usage_create_info, .viewType = common->layered_dpb && is_dpb ? VK_IMAGE_VIEW_TYPE_2D_ARRAY : VK_IMAGE_VIEW_TYPE_2D, .format = vkf, @@ -365,12 +360,6 @@ av_cold void ff_vk_video_common_uninit(FFVulkanContext *s, av_frame_free(&common->layered_frame); av_buffer_unref(&common->dpb_hwfc_ref); - - if (common->yuv_sampler) { - vk->DestroySamplerYcbcrConversion(s->hwctx->act_dev, common->yuv_sampler, - s->hwctx->alloc); - common->yuv_sampler = VK_NULL_HANDLE; - } } av_cold int ff_vk_video_common_init(AVCodecContext *avctx, FFVulkanContext *s, @@ -383,25 +372,6 @@ av_cold int ff_vk_video_common_init(AVCodecContext *avctx, FFVulkanContext *s, VkVideoSessionMemoryRequirementsKHR *mem = NULL; VkBindVideoSessionMemoryInfoKHR *bind_mem = NULL; - int cxpos = 0, cypos = 0; - VkSamplerYcbcrConversionCreateInfo yuv_sampler_info = { - .sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO, - .components = ff_comp_identity_map, - .ycbcrModel = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY, - .ycbcrRange = avctx->color_range == AVCOL_RANGE_MPEG, /* Ignored */ - .format = session_create->pictureFormat, - }; - - /* Create identity YUV sampler - * (VkImageViews of YUV image formats require it, even if it does nothing) */ - av_chroma_location_enum_to_pos(&cxpos, &cypos, avctx->chroma_sample_location); - yuv_sampler_info.xChromaOffset = cxpos >> 7; - yuv_sampler_info.yChromaOffset = cypos >> 7; - ret = vk->CreateSamplerYcbcrConversion(s->hwctx->act_dev, &yuv_sampler_info, - s->hwctx->alloc, &common->yuv_sampler); - if (ret != VK_SUCCESS) - return AVERROR_EXTERNAL; - /* Create session */ ret = vk->CreateVideoSessionKHR(s->hwctx->act_dev, session_create, s->hwctx->alloc, &common->session); diff --git a/libavcodec/vulkan_video.h b/libavcodec/vulkan_video.h index 14168b7cc8..d63bfae3fc 100644 --- a/libavcodec/vulkan_video.h +++ b/libavcodec/vulkan_video.h @@ -34,8 +34,6 @@ typedef struct FFVkVideoSession { VkDeviceMemory *mem; uint32_t nb_mem; - VkSamplerYcbcrConversion yuv_sampler; - AVBufferRef *dpb_hwfc_ref; int layered_dpb; AVFrame *layered_frame; -- 2.49.1 >From e27ba4f25978e8931d8e5475ada7ce701c99ba6a Mon Sep 17 00:00:00 2001 From: Benjamin Cheng <[email protected]> Date: Tue, 30 Dec 2025 12:01:19 -0500 Subject: [PATCH 4/4] hwcontext_vulkan: Remove unnecessary validation filters With the latest changes, I can no longer get these to show up. --- libavutil/hwcontext_vulkan.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c index 182eabceb0..2011cdec79 100644 --- a/libavutil/hwcontext_vulkan.c +++ b/libavutil/hwcontext_vulkan.c @@ -733,8 +733,6 @@ static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEX switch (data->messageIdNumber) { case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */ case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */ - case 0x618ab1e7: /* VUID-VkImageViewCreateInfo-usage-02275 */ - case 0x30f4ac70: /* VUID-VkImageCreateInfo-pNext-06811 */ return VK_FALSE; default: break; -- 2.49.1 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
