This is an automated email from the git hooks/post-receive script. Git pushed a commit to branch master in repository ffmpeg.
commit 5d5a38a7e46be296eab75982c0986054f731e19e Author: Steven Xiao <[email protected]> AuthorDate: Thu Oct 29 07:38:00 2026 -0400 Commit: Steven Xiao <[email protected]> CommitDate: Mon Nov 2 12:35:20 2026 +0000 avfilter/vf_scale_d3d12: add RGB formats and color space conversion Extend scale_d3d12 from NV12/P010 resizing to full pixel format and color space conversion, matching what the D3D12 video processor is actually capable of. - Support converting to/from the semi-planar YUV formats (nv12, p010, p012) and the packed RGB formats (bgra, rgba, x2bgr10, rgbaf16), when the hardware supports it. - Add out_color_matrix, out_range, out_color_primaries and out_color_transfer options. - Recalculate sample_aspect_ratio and drop size-dependent side data on resize. - Harden GPU teardown/reconfiguration. Example usage: Convert a d3d12 NV12 surface to BGRA: scale_d3d12=format=bgra Convert an HDR10 (P010) surface to 10-bit RGB, without altering its HDR color properties: scale_d3d12=format=x2bgr10 Scale and convert to full-range BGRA: scale_d3d12=format=bgra:out_range=full Signed-off-by: younengxiao <[email protected]> --- doc/filters.texi | 140 +++++ libavfilter/vf_scale_d3d12.c | 1411 +++++++++++++++++++++++++++++++++++++----- 2 files changed, 1402 insertions(+), 149 deletions(-) diff --git a/doc/filters.texi b/doc/filters.texi index e2f2d38f67..b57a5e8a00 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -22392,6 +22392,146 @@ If the specified expression is not valid, it is kept at its current value. @end table +@section scale_d3d12 + +Scale (resize) and convert the pixel format of the input video using +Direct3D 12 hardware-accelerated video processing +(@code{ID3D12VideoProcessor}). It requires the input to be in @code{d3d12} +hardware pixel format. + +Setting the output width and height works in the same way as for the +@ref{scale} filter. @code{sample_aspect_ratio} is recalculated for the new +dimensions the same way @ref{scale} does it, and side data whose values +are expressed in pixel coordinates of the old size (motion vectors, +detection bounding boxes, ROI, ...) is removed when the size actually +changes, also the same way @ref{scale} does it. + +Which input/output pixel format and color space combinations are actually +supported depends on the GPU and driver. The filter first rejects color +descriptions it cannot represent using DXGI. For representable +combinations, support is queried from the GPU and driver. If that +capability query reports the conversion unsupported, @code{-v debug} +prints a probe of alternative output combinations. + +The filter accepts the following options: +@table @option +@item w +@item h +Set the output video dimension expression. Default value is the input +dimension. + +Allows for the same expressions as the @ref{scale} filter. + +@item format +Controls the output pixel format. By default, or if none is specified, the +input pixel format is used. + +Besides converting between the semi-planar YUV formats @code{nv12}, +@code{p010} and @code{p012}, this filter can also convert to the packed +RGB formats @code{bgra}, @code{rgba}, @code{x2bgr10} and @code{rgbaf16}, +when supported by the hardware. + +@item out_color_matrix +Set the output colorspace matrix. This only affects YUV output formats +and accepts the same values as the @option{space} option of the +@code{colorspace} filter, e.g. @code{smpte170m}, @code{bt709}, +@code{bt2020nc}; only those exact names are recognized, not other +common aliases (e.g. @code{bt2020nc}, not @code{bt2020ncl}). If +@option{out_color_matrix} is not specified and @option{out_color_primaries} +is explicitly set, the output matrix is derived from the requested +primaries. Otherwise, the input matrix is inherited; if it is +unspecified, the matrix is derived from the input primaries. + +@item out_range +Set the output color range. It accepts the following values: + +@table @samp +@item tv, mpeg, limited +Limited range. + +@item pc, jpeg, full +Full range. +@end table + +If not specified, it defaults to the input range, except that a +@option{out_color_transfer} of @code{linear} (explicit or defaulted, +see below) always selects @code{full} range. If the range is still +unspecified at that point, it defaults to @code{limited} for YUV +output and @code{full} for RGB output. + +@item out_color_primaries +Set the output color primaries. It accepts the same values as the +@option{primaries} option of the @code{colorspace} filter, e.g. +@code{bt709}, @code{bt2020}. If not specified, it defaults to the input +primaries. + +@item out_color_transfer +Set the output color transfer characteristics. It accepts the same +values as the @option{trc} option of the @code{colorspace} filter, e.g. +@code{bt709}, @code{smpte2084}, @code{arib-std-b67}; only those exact +names are recognized, not other common aliases (e.g. +@code{iec61966-2-1}, not @code{srgb}; @code{bt470bg}, not +@code{gamma28}). If not specified, it defaults to the input transfer +characteristics, except that @code{rgbaf16} output defaults to +@code{linear} when both this option and the input transfer +characteristics are unspecified. + +@end table + +The video processor and output color properties are configured from +the first frame and reused for subsequent frames. Changes in frame +dimensions require filter-graph reinitialization. + +If this filter changes the output color primaries and/or transfer +characteristics relative to the input, side data describing those +properties (mastering display metadata, HDR10+/Dolby Vision metadata, +ICC profile, ...) is removed from the output frame. + +Alpha is preserved for RGB formats that have it (@code{bgra}, +@code{rgba}, @code{rgbaf16}); other formats, and devices that cannot +pass alpha through, produce opaque alpha instead. A premultiplied-alpha +input is rejected if the requested conversion cannot preserve it +correctly (e.g. it also changes color space); unpremultiply the input +first in that case. + +YUV output is retagged with the chroma sample location the selected +output color space actually uses, which depends on the matrix, range +and transfer characteristic (e.g. @code{topleft} for HLG, @code{left} +for BT.601/BT.709); RGB output is always tagged @code{unspecified}. An +input chroma siting the chosen input color space cannot represent is +rejected. + +@subsection Examples + +@itemize +@item +Scale a @code{d3d12} NV12 surface to 1280x720, keeping the pixel format: +@example +scale_d3d12=w=1280:h=720 +@end example + +@item +Convert a @code{d3d12} NV12 surface to BGRA, so it can be downloaded and +encoded or displayed without a software pixel format conversion: +@example +scale_d3d12=format=bgra +@end example + +@item +Convert a 10-bit HDR10 (P010) surface to 10-bit RGB, without altering +its HDR color properties: +@example +scale_d3d12=format=x2bgr10 +@end example + +@item +Scale and convert to full-range BGRA, for example before @code{hwdownload} +feeds a renderer that expects full-range RGB: +@example +scale_d3d12=format=bgra:out_range=full +@end example +@end itemize + @section scharr Apply scharr operator to input video stream. diff --git a/libavfilter/vf_scale_d3d12.c b/libavfilter/vf_scale_d3d12.c index 7bddda6552..cadd48e5a0 100644 --- a/libavfilter/vf_scale_d3d12.c +++ b/libavfilter/vf_scale_d3d12.c @@ -19,6 +19,7 @@ #define COBJMACROS #include "libavutil/fifo.h" +#include "libavutil/frame.h" #include "libavutil/opt.h" #include "libavutil/pixdesc.h" #include "compat/w32dlfcn.h" @@ -28,20 +29,64 @@ #include "libavutil/hwcontext_d3d12va_internal.h" #include "filters.h" +#include "formats.h" #include "scale_eval.h" #include "video.h" typedef struct D3D12ScaleCommandAllocator { ID3D12CommandAllocator *command_allocator; UINT64 fence_value; + AVFrame *input_ref; } D3D12ScaleCommandAllocator; +typedef struct D3D12ScaleFormatMap { + enum AVPixelFormat pix_fmt; + DXGI_FORMAT dxgi_format; + int is_rgb; +} D3D12ScaleFormatMap; + +static const D3D12ScaleFormatMap d3d12_scale_format_map[] = { + { AV_PIX_FMT_NV12, DXGI_FORMAT_NV12, 0 }, + { AV_PIX_FMT_P010, DXGI_FORMAT_P010, 0 }, + { AV_PIX_FMT_P012, DXGI_FORMAT_P016, 0 }, + { AV_PIX_FMT_BGRA, DXGI_FORMAT_B8G8R8A8_UNORM, 1 }, + { AV_PIX_FMT_RGBA, DXGI_FORMAT_R8G8B8A8_UNORM, 1 }, + { AV_PIX_FMT_X2BGR10, DXGI_FORMAT_R10G10B10A2_UNORM, 1 }, + { AV_PIX_FMT_RGBAF16, DXGI_FORMAT_R16G16B16A16_FLOAT, 1 }, +}; + +static const D3D12ScaleFormatMap *d3d12_scale_map_from_pixfmt(enum AVPixelFormat fmt) +{ + for (int i = 0; i < FF_ARRAY_ELEMS(d3d12_scale_format_map); i++) + if (d3d12_scale_format_map[i].pix_fmt == fmt) + return &d3d12_scale_format_map[i]; + + return NULL; +} + +static const D3D12ScaleFormatMap *d3d12_scale_map_from_dxgi(DXGI_FORMAT fmt) +{ + for (int i = 0; i < FF_ARRAY_ELEMS(d3d12_scale_format_map); i++) + if (d3d12_scale_format_map[i].dxgi_format == fmt) + return &d3d12_scale_format_map[i]; + + return NULL; +} + typedef struct ScaleD3D12Context { const AVClass *classCtx; char *w_expr; char *h_expr; enum AVPixelFormat format; + char *color_matrix_string; + char *color_transfer_string; + char *color_primaries_string; + int color_range; + enum AVColorSpace color_matrix; + enum AVColorTransferCharacteristic color_transfer; + enum AVColorPrimaries color_primaries; + /* D3D12 objects */ ID3D12Device *device; ID3D12VideoDevice *video_device; @@ -51,6 +96,9 @@ typedef struct ScaleD3D12Context { AVFifo *allocator_queue; + int configured; + int gpu_work_submitted; + /* Synchronization */ ID3D12Fence *fence; UINT64 fence_value; @@ -65,19 +113,90 @@ typedef struct ScaleD3D12Context { int input_width, input_height; DXGI_FORMAT input_format; DXGI_FORMAT output_format; + int input_is_rgb; + int output_is_rgb; + int input_has_alpha; + int output_has_alpha; + int alpha_passthrough; /* Color space and frame rate */ DXGI_COLOR_SPACE_TYPE input_colorspace; + DXGI_COLOR_SPACE_TYPE output_colorspace; AVRational input_framerate; + /* AVFrame-facing output color properties, derived once from the first + * frame plus the out_* options and then reused for every frame. */ + enum AVColorSpace output_av_colorspace; + enum AVColorRange output_av_color_range; + enum AVColorPrimaries output_av_color_primaries; + enum AVColorTransferCharacteristic output_av_color_trc; + + enum AVColorPrimaries input_av_color_primaries; + enum AVColorTransferCharacteristic input_av_color_trc; + /* Video processor capabilities */ D3D12_FEATURE_DATA_VIDEO_PROCESS_SUPPORT process_support; } ScaleD3D12Context; static av_cold int scale_d3d12_init(AVFilterContext *ctx) { + ScaleD3D12Context *s = ctx->priv; + +#define STRING_OPTION(var_name, func_name, opt_name, default_value) do { \ + if (s->var_name ## _string) { \ + int var = av_ ## func_name ## _from_name(s->var_name ## _string); \ + if (var < 0) { \ + av_log(ctx, AV_LOG_ERROR, "Invalid %s.\n", opt_name); \ + return AVERROR(EINVAL); \ + } \ + s->var_name = var; \ + } else { \ + s->var_name = default_value; \ + } \ + } while (0) + + STRING_OPTION(color_matrix, color_space, "out_color_matrix", AVCOL_SPC_UNSPECIFIED); + STRING_OPTION(color_transfer, color_transfer, "out_color_transfer", AVCOL_TRC_UNSPECIFIED); + STRING_OPTION(color_primaries, color_primaries, "out_color_primaries", AVCOL_PRI_UNSPECIFIED); + +#undef STRING_OPTION + return 0; } +static int scale_d3d12_wait_fence_exact(ID3D12Fence *fence, UINT64 target, + HANDLE fence_event, DWORD timeout_ms) +{ + int has_deadline = (timeout_ms != INFINITE); + ULONGLONG deadline = 0; + + if (ID3D12Fence_GetCompletedValue(fence) >= target) + return 1; + + if (FAILED(ID3D12Fence_SetEventOnCompletion(fence, target, fence_event))) + return 0; + + if (has_deadline) + deadline = GetTickCount64() + timeout_ms; + + for (;;) { + DWORD wait_result = WaitForSingleObject(fence_event, timeout_ms); + + if (wait_result == WAIT_OBJECT_0) { + if (ID3D12Fence_GetCompletedValue(fence) >= target) + return 1; + if (has_deadline) { + ULONGLONG now = GetTickCount64(); + if (now >= deadline) + return 0; + timeout_ms = (DWORD)(deadline - now); + } + continue; + } + + return 0; + } +} + static int scale_d3d12_get_valid_command_allocator(ScaleD3D12Context *s, AVFilterContext *ctx, ID3D12CommandAllocator **ppAllocator) { @@ -89,6 +208,7 @@ static int scale_d3d12_get_valid_command_allocator(ScaleD3D12Context *s, AVFilte if (completed >= allocator.fence_value) { *ppAllocator = allocator.command_allocator; av_fifo_read(s->allocator_queue, &allocator, 1); + av_frame_free(&allocator.input_ref); return 0; } } @@ -103,39 +223,73 @@ static int scale_d3d12_get_valid_command_allocator(ScaleD3D12Context *s, AVFilte return 0; } -static int scale_d3d12_discard_command_allocator(ScaleD3D12Context *s, ID3D12CommandAllocator *pAllocator, - UINT64 fence_value) +static int scale_d3d12_discard_command_allocator(ScaleD3D12Context *s, AVFilterContext *ctx, + ID3D12CommandAllocator *pAllocator, + UINT64 fence_value, AVFrame *input_ref) { D3D12ScaleCommandAllocator allocator = { .command_allocator = pAllocator, .fence_value = fence_value, + .input_ref = input_ref, }; + int ret = av_fifo_write(s->allocator_queue, &allocator, 1); - av_fifo_write(s->allocator_queue, &allocator, 1); + if (ret < 0) { + int confirmed_done = s->fence && s->fence_event && + scale_d3d12_wait_fence_exact(s->fence, fence_value, s->fence_event, INFINITE); - return 0; + if (confirmed_done) { + ID3D12CommandAllocator_Release(pAllocator); + av_frame_free(&input_ref); + } else { + av_log(ctx, AV_LOG_ERROR, + "Failed to queue a command allocator for later reuse\n"); + } + } + + return ret; } -static void release_d3d12_resources(ScaleD3D12Context *s) { - UINT64 fence_value; - HRESULT hr; - D3D12ScaleCommandAllocator allocator; +static int release_d3d12_resources(ScaleD3D12Context *s, AVFilterContext *ctx) { + int gpu_confirmed_done = 1; - /* Wait for all GPU operations to complete before releasing resources */ - if (s->command_queue && s->fence && s->fence_event) { - fence_value = s->fence_value - 1; - hr = ID3D12CommandQueue_Signal(s->command_queue, s->fence, fence_value); - if (SUCCEEDED(hr)) { - UINT64 completed = ID3D12Fence_GetCompletedValue(s->fence); - if (completed < fence_value) { - hr = ID3D12Fence_SetEventOnCompletion(s->fence, fence_value, s->fence_event); - if (SUCCEEDED(hr)) { - WaitForSingleObject(s->fence_event, INFINITE); - } - } + s->configured = 0; + + if (s->gpu_work_submitted && s->command_queue && s->fence && s->fence_event) { + UINT64 drain_value = s->fence_value + 1; + + gpu_confirmed_done = 0; + + if (SUCCEEDED(ID3D12CommandQueue_Signal(s->command_queue, s->fence, drain_value))) { + UINT64 completed; + /* this gives one more value as defense in depth, while cost is almost nothing */ + s->fence_value = drain_value + 1; + completed = ID3D12Fence_GetCompletedValue(s->fence); + if (completed >= drain_value) + gpu_confirmed_done = 1; + else + gpu_confirmed_done = scale_d3d12_wait_fence_exact(s->fence, drain_value, + s->fence_event, INFINITE); + } + + if (!gpu_confirmed_done) { + HRESULT removed_reason = ID3D12Device_GetDeviceRemovedReason(s->device); + gpu_confirmed_done = FAILED(removed_reason); + if (gpu_confirmed_done) + av_log(ctx, AV_LOG_ERROR, + "D3D12 device removed (HRESULT 0x%lX) while tearing " + "down scale_d3d12\n", removed_reason); } } + if (!gpu_confirmed_done) { + av_log(ctx, AV_LOG_ERROR, + "Could not confirm the GPU finished all scale_d3d12 work " + "during teardown\n"); + + return 0; + } + if (s->fence_event) { CloseHandle(s->fence_event); s->fence_event = NULL; @@ -152,8 +306,11 @@ static void release_d3d12_resources(ScaleD3D12Context *s) { } if (s->allocator_queue) { - while (av_fifo_read(s->allocator_queue, &allocator, 1) >= 0) + D3D12ScaleCommandAllocator allocator; + while (av_fifo_read(s->allocator_queue, &allocator, 1) >= 0) { ID3D12CommandAllocator_Release(allocator.command_allocator); + av_frame_free(&allocator.input_ref); + } av_fifo_freep2(&s->allocator_queue); } @@ -171,54 +328,552 @@ static void release_d3d12_resources(ScaleD3D12Context *s) { ID3D12CommandQueue_Release(s->command_queue); s->command_queue = NULL; } + + s->gpu_work_submitted = 0; + + return 1; +} + +static int d3d12_scale_known_matrix(enum AVColorSpace spc) +{ + switch (spc) { + case AVCOL_SPC_UNSPECIFIED: + case AVCOL_SPC_BT709: + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + case AVCOL_SPC_BT2020_NCL: + case AVCOL_SPC_BT2020_CL: + return 1; + + default: + return 0; + } +} + +static int d3d12_scale_known_sdr_trc(enum AVColorTransferCharacteristic trc) +{ + switch (trc) { + case AVCOL_TRC_UNSPECIFIED: + case AVCOL_TRC_BT709: + case AVCOL_TRC_GAMMA22: + case AVCOL_TRC_GAMMA28: + case AVCOL_TRC_SMPTE170M: + case AVCOL_TRC_SMPTE240M: + case AVCOL_TRC_IEC61966_2_1: + case AVCOL_TRC_BT2020_10: + case AVCOL_TRC_BT2020_12: + return 1; + + default: + return 0; + } +} + +static int d3d12_scale_trc_group(enum AVColorTransferCharacteristic trc) +{ + switch (trc) { + case AVCOL_TRC_BT709: + case AVCOL_TRC_SMPTE170M: + case AVCOL_TRC_BT2020_10: + case AVCOL_TRC_BT2020_12: + return 0; /* trc_bt709 */ + + case AVCOL_TRC_GAMMA22: + return 1; /* trc_gamma22 */ + + case AVCOL_TRC_GAMMA28: + return 2; /* trc_gamma28 */ + + case AVCOL_TRC_SMPTE240M: + return 3; /* trc_smpte240M */ + + case AVCOL_TRC_IEC61966_2_1: + return 4; /* trc_iec61966_2_1 (sRGB) */ + + default: + return -1; + } +} + +static int d3d12_scale_finalize_output_trc(AVFilterContext *ctx, + DXGI_COLOR_SPACE_TYPE input_space, + DXGI_COLOR_SPACE_TYPE output_space, + int explicit_request, + enum AVColorTransferCharacteristic input_trc, + enum AVColorTransferCharacteristic *trc) +{ + enum AVColorTransferCharacteristic canonical = + (output_space == DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709) ? + AVCOL_TRC_IEC61966_2_1 : AVCOL_TRC_BT709; + enum AVColorTransferCharacteristic canonical_input = + (input_trc == AVCOL_TRC_SMPTE2084 || + input_trc == AVCOL_TRC_ARIB_STD_B67 || + input_trc == AVCOL_TRC_LINEAR) ? input_trc : + (input_space == DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709) ? + AVCOL_TRC_IEC61966_2_1 : AVCOL_TRC_BT709; + + if (canonical_input != canonical) { + if (*trc != canonical) { + const char *old_name = av_color_transfer_name(*trc); + const char *new_name = av_color_transfer_name(canonical); + av_log(ctx, AV_LOG_WARNING, + "No exact DXGI color space; retagging transfer " + "characteristic from %s to %s\n", + old_name ? old_name : "unknown", new_name ? new_name : "unknown"); + *trc = canonical; + } + + return 0; + } + + if (!explicit_request) + return 0; + + if (d3d12_scale_trc_group(*trc) == d3d12_scale_trc_group(input_trc)) + return 0; + + const char *req_name = av_color_transfer_name(*trc); + + if (input_trc == AVCOL_TRC_UNSPECIFIED) { + av_log(ctx, AV_LOG_ERROR, + "Cannot verify requested transfer characteristic (%s); " + "input's curve is unspecified/unknown\n", + req_name ? req_name : "requested value"); + } else { + const char *old_name = av_color_transfer_name(input_trc); + av_log(ctx, AV_LOG_ERROR, + "Requested transfer characteristic (%s) differs from " + "input's (%s); same DXGI color space means no curve " + "change applies\n", + req_name ? req_name : "requested value", + old_name ? old_name : "transfer characteristic"); + } + + return AVERROR(EINVAL); +} + +static int d3d12_scale_reconcile_primaries(AVFilterContext *ctx, int is_output, + enum AVColorPrimaries *primaries, + enum AVColorPrimaries expected_primaries, + const char *space_description) +{ + if (*primaries != AVCOL_PRI_UNSPECIFIED && *primaries != expected_primaries) { + const char *old_name = av_color_primaries_name(*primaries); + const char *new_name = av_color_primaries_name(expected_primaries); + if (is_output) { + av_log(ctx, AV_LOG_WARNING, + "%s forces fixed primaries; retagging output from " + "%s to %s\n", + space_description, + old_name ? old_name : "unknown", new_name ? new_name : "unknown"); + } else { + av_log(ctx, AV_LOG_ERROR, + "%s forces fixed primaries, which disagrees with " + "this input's %s primaries tag\n", + space_description, old_name ? old_name : "unknown"); + + return AVERROR(EINVAL); + } + } + + *primaries = expected_primaries; + return 0; } -static DXGI_COLOR_SPACE_TYPE get_dxgi_colorspace(enum AVColorSpace colorspace, enum AVColorTransferCharacteristic trc, int is_10bit) +static int d3d12_scale_reject_unrepresentable_nonleft(AVFilterContext *ctx, int is_output, + enum AVChromaLocation chroma_loc) { - /* Map FFmpeg color space to DXGI color space */ - if (is_10bit) { - /* 10-bit formats (P010) */ - if (colorspace == AVCOL_SPC_BT2020_NCL || colorspace == AVCOL_SPC_BT2020_CL) { - if (trc == AVCOL_TRC_SMPTE2084) { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020; ///< HDR10 - } else if (trc == AVCOL_TRC_ARIB_STD_B67) { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020; ///< HLG + if (chroma_loc == AVCHROMA_LOC_UNSPECIFIED || chroma_loc == AVCHROMA_LOC_LEFT) + return 0; + + if (is_output) { + av_log(ctx, AV_LOG_WARNING, + "This DXGI color space has no chroma entry other than " + "left-sited; the output will actually be left-sited\n"); + + return 0; + } + + av_log(ctx, AV_LOG_ERROR, + "This DXGI color space has no chroma entry other than " + "left-sited, but this input is tagged with a different siting\n"); + + return AVERROR(EINVAL); +} + +static int d3d12_scale_reject_unrepresentable_left(AVFilterContext *ctx, int is_output, + enum AVChromaLocation chroma_loc) +{ + if (chroma_loc == AVCHROMA_LOC_UNSPECIFIED || chroma_loc == AVCHROMA_LOC_TOPLEFT) + return 0; + + if (is_output) { + av_log(ctx, AV_LOG_WARNING, + "HLG has no DXGI entry with any chroma siting other than " + "top-left; the output will actually be top-left-sited\n"); + + return 0; + } + + av_log(ctx, AV_LOG_ERROR, + "HLG has no DXGI entry with any chroma siting other than " + "top-left, but this input is tagged with a different siting\n"); + + return AVERROR(EINVAL); +} + +static int d3d12_scale_reject_unrepresentable_siting_2020(AVFilterContext *ctx, int is_output, + enum AVChromaLocation chroma_loc) +{ + if (chroma_loc == AVCHROMA_LOC_UNSPECIFIED || + chroma_loc == AVCHROMA_LOC_TOPLEFT || + chroma_loc == AVCHROMA_LOC_LEFT) + return 0; + + if (is_output) { + av_log(ctx, AV_LOG_WARNING, + "DXGI BT.2020 only supports top-left/left chroma " + "siting; output will use one of those instead\n"); + + return 0; + } + + av_log(ctx, AV_LOG_ERROR, + "DXGI BT.2020 only supports top-left/left chroma siting, " + "but this input is tagged with a different siting\n"); + + return AVERROR(EINVAL); +} + +static int d3d12_scale_get_colorspace(AVFilterContext *ctx, int is_rgb, int is_output, + enum AVColorSpace *spc, + enum AVColorRange *range, + enum AVColorPrimaries *primaries, + enum AVColorTransferCharacteristic *trc, + enum AVChromaLocation chroma_loc, + enum AVColorPrimaries input_601_hint, + DXGI_COLOR_SPACE_TYPE *out_space) +{ + int full_range = (*range == AVCOL_RANGE_JPEG); + int ret; + + if (!is_rgb && *spc == AVCOL_SPC_BT2020_CL) { + if (is_output) { + av_log(ctx, AV_LOG_WARNING, + "BT.2020 constant-luminance has no DXGI color space; " + "retagging output as bt2020nc\n"); + *spc = AVCOL_SPC_BT2020_NCL; + } else { + av_log(ctx, AV_LOG_ERROR, + "BT.2020 constant-luminance matrix has no DXGI color " + "space; cannot process this input without misinterpreting\n"); + + return AVERROR(EINVAL); + } + } + + if (!is_rgb && !d3d12_scale_known_matrix(*spc)) { + const char *name = av_color_space_name(*spc); + av_log(ctx, AV_LOG_ERROR, + "%s matrix has no DXGI YCbCr color space; only BT.601, " + "BT.709 or BT.2020 are supported\n", name ? name : "this"); + + return AVERROR(EINVAL); + } + + if (!is_rgb && *trc != AVCOL_TRC_SMPTE2084 && *trc != AVCOL_TRC_ARIB_STD_B67 && + !d3d12_scale_known_sdr_trc(*trc)) { + const char *name = av_color_transfer_name(*trc); + av_log(ctx, AV_LOG_ERROR, + "%s transfer has no DXGI YCbCr color space; only SDR " + "gamma, smpte2084 or HLG (arib-std-b67) are " + "supported\n", + name ? name : "this"); + + return AVERROR(EINVAL); + } + + int primaries_is_2020 = (*primaries == AVCOL_PRI_BT2020); + int is_2020 = (*spc == AVCOL_SPC_BT2020_NCL || *spc == AVCOL_SPC_BT2020_CL || + (*spc == AVCOL_SPC_UNSPECIFIED && primaries_is_2020)); + int is_601 = (*spc == AVCOL_SPC_BT470BG || *spc == AVCOL_SPC_SMPTE170M || + (*spc == AVCOL_SPC_UNSPECIFIED && + (*primaries == AVCOL_PRI_BT470BG || *primaries == AVCOL_PRI_SMPTE170M))); + + if (is_rgb) { + enum AVColorPrimaries expected_primaries; + + switch (*trc) { + case AVCOL_TRC_SMPTE2084: + *out_space = full_range ? DXGI_COLOR_SPACE_RGB_FULL_G2084_NONE_P2020 + : DXGI_COLOR_SPACE_RGB_STUDIO_G2084_NONE_P2020; + expected_primaries = AVCOL_PRI_BT2020; + break; + + case AVCOL_TRC_ARIB_STD_B67: + if (is_output) + av_log(ctx, AV_LOG_ERROR, + "HLG has no RGB DXGI color space; use a YUV " + "output format or a supported out_color_transfer\n"); + else + av_log(ctx, AV_LOG_ERROR, + "HLG transfer characteristic has no RGB DXGI color " + "space; this RGB input cannot be processed\n"); + return AVERROR(EINVAL); + + case AVCOL_TRC_LINEAR: + if (!full_range) { + if (is_output) + av_log(ctx, AV_LOG_ERROR, + "Studio range linear RGB has no DXGI color " + "space; use out_range=full, or a non-linear " + "out_color_transfer (e.g. bt709)\n"); + else + av_log(ctx, AV_LOG_ERROR, + "Studio range linear RGB has no DXGI color " + "space; this input cannot be processed\n"); + + return AVERROR(EINVAL); + } + *out_space = DXGI_COLOR_SPACE_RGB_FULL_G10_NONE_P709; + expected_primaries = AVCOL_PRI_BT709; + break; + + default: + if (!d3d12_scale_known_sdr_trc(*trc)) { + const char *name = av_color_transfer_name(*trc); + av_log(ctx, AV_LOG_ERROR, + "%s transfer has no RGB DXGI color space; only " + "SDR gamma, linear or smpte2084 are " + "supported\n", name ? name : "this"); + + return AVERROR(EINVAL); + } + if (primaries_is_2020) { + *out_space = full_range ? DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P2020 + : DXGI_COLOR_SPACE_RGB_STUDIO_G22_NONE_P2020; + expected_primaries = AVCOL_PRI_BT2020; } else { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020; + *out_space = full_range ? DXGI_COLOR_SPACE_RGB_FULL_G22_NONE_P709 + : DXGI_COLOR_SPACE_RGB_STUDIO_G22_NONE_P709; + expected_primaries = AVCOL_PRI_BT709; + } + break; + } + + return d3d12_scale_reconcile_primaries(ctx, is_output, primaries, expected_primaries, + "this RGB transfer characteristic"); + } + + /* YCbCr family. */ + if (is_2020) { + int want_topleft = (chroma_loc == AVCHROMA_LOC_TOPLEFT); + ret = d3d12_scale_reject_unrepresentable_siting_2020(ctx, is_output, chroma_loc); + if (ret < 0) + return ret; + + if (*trc == AVCOL_TRC_SMPTE2084) { + if (full_range) { + av_log(ctx, AV_LOG_ERROR, + "No full-range smpte2084 YCbCr DXGI color space " + "exists; cannot process this %s\n", + is_output ? "output request" : "input"); + + return AVERROR(EINVAL); } + *out_space = want_topleft ? DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020 + : DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_LEFT_P2020; + } else if (*trc == AVCOL_TRC_ARIB_STD_B67) { + ret = d3d12_scale_reject_unrepresentable_left(ctx, is_output, chroma_loc); + if (ret < 0) + return ret; + *out_space = full_range ? DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020 + : DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020; } else { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709; ///< Rec.709 10-bit + if (want_topleft && full_range) { + if (!is_output) { + av_log(ctx, AV_LOG_ERROR, + "Full-range BT.2020 SDR has no top-left-sited " + "DXGI YCbCr color space\n"); + + return AVERROR(EINVAL); + } + av_log(ctx, AV_LOG_WARNING, + "Full-range BT.2020 SDR has no top-left-sited DXGI " + "YCbCr color space; falling back to left-sited\n"); + want_topleft = 0; + } + if (want_topleft) + *out_space = DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_TOPLEFT_P2020; + else + *out_space = full_range ? DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P2020 + : DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020; } - } else { - /* 8-bit formats (NV12) */ - if (colorspace == AVCOL_SPC_BT2020_NCL || colorspace == AVCOL_SPC_BT2020_CL) { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P2020; - } else if (colorspace == AVCOL_SPC_BT470BG || colorspace == AVCOL_SPC_SMPTE170M) { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601; + + ret = d3d12_scale_reconcile_primaries(ctx, is_output, primaries, AVCOL_PRI_BT2020, + "a BT.2020 matrix"); + if (ret < 0) + return ret; + } else if (is_601) { + if (*trc == AVCOL_TRC_SMPTE2084 || *trc == AVCOL_TRC_ARIB_STD_B67) { + if (is_output) + av_log(ctx, AV_LOG_ERROR, + "%s transfer has no BT.601 DXGI color space; use " + "out_color_matrix=bt2020nc or an SDR transfer\n", + *trc == AVCOL_TRC_SMPTE2084 ? "smpte2084" : "HLG"); + else + av_log(ctx, AV_LOG_ERROR, + "%s transfer has no BT.601 DXGI color space; this " + "input cannot be processed.\n", + *trc == AVCOL_TRC_SMPTE2084 ? "smpte2084" : "HLG"); + + return AVERROR(EINVAL); + } + ret = d3d12_scale_reject_unrepresentable_nonleft(ctx, is_output, chroma_loc); + if (ret < 0) + return ret; + *out_space = full_range ? DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601 + : DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601; + + enum AVColorPrimaries this_601_primaries = + *spc == AVCOL_SPC_BT470BG ? AVCOL_PRI_BT470BG : + *spc == AVCOL_SPC_SMPTE170M ? AVCOL_PRI_SMPTE170M : + *primaries; + + if (is_output && input_601_hint != AVCOL_PRI_UNSPECIFIED) { + if (*primaries != AVCOL_PRI_UNSPECIFIED && *primaries != input_601_hint) { + const char *in_name = av_color_primaries_name(input_601_hint); + const char *out_name = av_color_primaries_name(*primaries); + av_log(ctx, AV_LOG_ERROR, + "DXGI's BT.601 color spaces cannot distinguish " + "PAL/NTSC primaries (%s input, %s output)\n", + in_name ? in_name : "unknown", out_name ? out_name : "unknown"); + + return AVERROR(EINVAL); + } + if (*primaries != input_601_hint) { + const char *new_name = av_color_primaries_name(input_601_hint); + av_log(ctx, AV_LOG_WARNING, + "BT.601 cannot change primaries; tagging output " + "as %s\n", + new_name ? new_name : "unknown"); + } + *primaries = input_601_hint; + } else if (!is_output) { + if (*primaries != AVCOL_PRI_UNSPECIFIED && + *primaries != AVCOL_PRI_BT470BG && *primaries != AVCOL_PRI_SMPTE170M) { + const char *old_name = av_color_primaries_name(*primaries); + av_log(ctx, AV_LOG_ERROR, + "BT.601 only represents PAL/NTSC primaries, " + "which disagrees with this input's %s tag\n", + old_name ? old_name : "unknown"); + + return AVERROR(EINVAL); + } + if (*primaries == AVCOL_PRI_UNSPECIFIED) + *primaries = this_601_primaries; } else { - return DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709; ///< Default to Rec.709 + ret = d3d12_scale_reconcile_primaries(ctx, is_output, primaries, this_601_primaries, + "this BT.601 matrix"); + if (ret < 0) + return ret; + } + } else { + /* Same gap as the BT.601 case above: no smpte2084/HLG DXGI color + * space exists paired with a BT.709 matrix either. */ + if (*trc == AVCOL_TRC_SMPTE2084 || *trc == AVCOL_TRC_ARIB_STD_B67) { + if (is_output) + av_log(ctx, AV_LOG_ERROR, + "%s has no BT.709 DXGI space; use " + "out_color_matrix=bt2020nc or SDR\n", + *trc == AVCOL_TRC_SMPTE2084 ? "smpte2084" : "HLG"); + else + av_log(ctx, AV_LOG_ERROR, + "%s has no BT.709 DXGI space; retag/convert to " + "BT.2020 or SDR before scale_d3d12\n", + *trc == AVCOL_TRC_SMPTE2084 ? "smpte2084" : "HLG"); + + return AVERROR(EINVAL); } + ret = d3d12_scale_reject_unrepresentable_nonleft(ctx, is_output, chroma_loc); + if (ret < 0) + return ret; + *out_space = full_range ? DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P709 + : DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P709; ///< Default to Rec.709 + + ret = d3d12_scale_reconcile_primaries(ctx, is_output, primaries, AVCOL_PRI_BT709, + "the default BT.709 matrix"); + if (ret < 0) + return ret; + } + + return 0; +} + +static enum AVChromaLocation d3d12_scale_dxgi_chroma_location(DXGI_COLOR_SPACE_TYPE space) +{ + switch (space) { + case DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_TOPLEFT_P2020: + case DXGI_COLOR_SPACE_YCBCR_STUDIO_G2084_TOPLEFT_P2020: + case DXGI_COLOR_SPACE_YCBCR_STUDIO_GHLG_TOPLEFT_P2020: + case DXGI_COLOR_SPACE_YCBCR_FULL_GHLG_TOPLEFT_P2020: + case DXGI_COLOR_SPACE_YCBCR_STUDIO_G24_TOPLEFT_P2020: + return AVCHROMA_LOC_TOPLEFT; + + default: + return AVCHROMA_LOC_LEFT; + } +} + +static void scale_d3d12_report_output_support(ScaleD3D12Context *s, AVFilterContext *ctx) +{ + D3D12_FEATURE_DATA_VIDEO_PROCESS_SUPPORT probe; + + if (av_log_get_level() < AV_LOG_DEBUG) + return; + + for (int i = 0; i < FF_ARRAY_ELEMS(d3d12_scale_format_map); i++) { + const D3D12ScaleFormatMap *map = &d3d12_scale_format_map[i]; + DXGI_COLOR_SPACE_TYPE candidate_space; + enum AVColorRange candidate_range = map->is_rgb ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + enum AVColorSpace candidate_spc = AVCOL_SPC_UNSPECIFIED; + enum AVColorPrimaries candidate_primaries = AVCOL_PRI_UNSPECIFIED; + enum AVColorTransferCharacteristic candidate_trc = AVCOL_TRC_UNSPECIFIED; + HRESULT hr; + + if (d3d12_scale_get_colorspace(ctx, map->is_rgb, 1, &candidate_spc, + &candidate_range, &candidate_primaries, + &candidate_trc, AVCHROMA_LOC_UNSPECIFIED, + AVCOL_PRI_UNSPECIFIED, + &candidate_space) < 0) + continue; + + probe = s->process_support; + probe.OutputFormat.Format = map->dxgi_format; + probe.OutputFormat.ColorSpace = candidate_space; + + hr = ID3D12VideoDevice_CheckFeatureSupport(s->video_device, D3D12_FEATURE_VIDEO_PROCESS_SUPPORT, + &probe, sizeof(probe)); + + av_log(ctx, AV_LOG_DEBUG, "Probe output %s (DXGI %d, colorspace %d): %s\n", + av_get_pix_fmt_name(map->pix_fmt), map->dxgi_format, candidate_space, + (SUCCEEDED(hr) && (probe.SupportFlags & D3D12_VIDEO_PROCESS_SUPPORT_FLAG_SUPPORTED)) ? + "supported" : "unsupported"); } } static AVRational get_input_framerate(AVFilterContext *ctx, AVFilterLink *inlink, AVFrame *in) { AVRational framerate = {0, 0}; + FilterLink *inl = ff_filter_link(inlink); - if (in->duration > 0 && inlink->time_base.num > 0 && inlink->time_base.den > 0) { - /* - * Calculate framerate from frame duration and timebase - * framerate = 1 / (duration * timebase) - */ + if (inl->frame_rate.num > 0 && inl->frame_rate.den > 0) { + framerate = inl->frame_rate; + } else if (in->duration > 0 && inlink->time_base.num > 0 && inlink->time_base.den > 0) { av_reduce(&framerate.num, &framerate.den, inlink->time_base.den, in->duration * inlink->time_base.num, INT_MAX); - } else if (inlink->time_base.num > 0 && inlink->time_base.den > 0) { - /* Estimate from timebase (inverse of timebase is often the framerate) */ - framerate.num = inlink->time_base.den; - framerate.den = inlink->time_base.num; } else { - /* Default to 30fps if framerate cannot be determined */ framerate.num = 30; framerate.den = 1; av_log(ctx, AV_LOG_WARNING, "Input framerate not determinable, defaulting to 30fps\n"); @@ -232,6 +887,15 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext int ret; ID3D12CommandAllocator *command_allocator = NULL; + if (s->video_device || s->command_queue || s->fence || s->fence_event || + s->command_list || s->video_processor || s->allocator_queue) { + if (!release_d3d12_resources(s, ctx)) { + av_log(ctx, AV_LOG_ERROR, + "Cannot configure a new D3D12 video processor\n"); + return AVERROR_EXTERNAL; + } + } + if (s->output_format == DXGI_FORMAT_UNKNOWN) { av_log(ctx, AV_LOG_ERROR, "Output format not initialized\n"); return AVERROR(EINVAL); @@ -247,6 +911,7 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext hr = ID3D12Device_QueryInterface(s->device, &IID_ID3D12VideoDevice, (void **)&s->video_device); if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to get D3D12 video device interface: HRESULT 0x%lX\n", hr); + return AVERROR_EXTERNAL; } @@ -260,6 +925,8 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext hr = ID3D12Device_CreateCommandQueue(s->device, &queue_desc, &IID_ID3D12CommandQueue, (void **)&s->command_queue); if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to create command queue: HRESULT 0x%lX\n", hr); + release_d3d12_resources(s, ctx); + return AVERROR_EXTERNAL; } @@ -275,7 +942,7 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext s->process_support.InputStereoFormat = D3D12_VIDEO_FRAME_STEREO_FORMAT_NONE; s->process_support.OutputFormat.Format = s->output_format; - s->process_support.OutputFormat.ColorSpace = s->input_colorspace; + s->process_support.OutputFormat.ColorSpace = s->output_colorspace; s->process_support.OutputFrameRate.Numerator = s->input_framerate.num; s->process_support.OutputFrameRate.Denominator = s->input_framerate.den; s->process_support.OutputStereoFormat = D3D12_VIDEO_FRAME_STEREO_FORMAT_NONE; @@ -289,18 +956,42 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Video process feature not supported: HRESULT 0x%lX\n", hr); + release_d3d12_resources(s, ctx); + return AVERROR_EXTERNAL; } if (!(s->process_support.SupportFlags & D3D12_VIDEO_PROCESS_SUPPORT_FLAG_SUPPORTED)) { - av_log(ctx, AV_LOG_ERROR, "Video process configuration not supported by hardware\n"); - return AVERROR_EXTERNAL; + av_log(ctx, AV_LOG_ERROR, + "D3D12 video processor does not support converting %s (colorspace %d) to %s " + "(colorspace %d) on this device\n", + av_get_pix_fmt_name(d3d12_scale_map_from_dxgi(s->input_format) ? + d3d12_scale_map_from_dxgi(s->input_format)->pix_fmt : AV_PIX_FMT_NONE), + s->input_colorspace, + av_get_pix_fmt_name(d3d12_scale_map_from_dxgi(s->output_format) ? + d3d12_scale_map_from_dxgi(s->output_format)->pix_fmt : AV_PIX_FMT_NONE), + s->output_colorspace); + scale_d3d12_report_output_support(s, ctx); + release_d3d12_resources(s, ctx); + + return AVERROR(ENOSYS); + } + + int want_alpha_passthrough = s->input_has_alpha && s->output_has_alpha; + if (want_alpha_passthrough && + !(s->process_support.FeatureSupport & D3D12_VIDEO_PROCESS_FEATURE_FLAG_ALPHA_FILL)) { + av_log(ctx, AV_LOG_WARNING, + "Device lacks ALPHA_FILL support; output alpha will be " + "forced opaque instead of preserved\n"); + want_alpha_passthrough = 0; } + s->alpha_passthrough = want_alpha_passthrough; D3D12_VIDEO_PROCESS_OUTPUT_STREAM_DESC processor_output_desc = { .Format = s->output_format, - .ColorSpace = s->input_colorspace, - .AlphaFillMode = D3D12_VIDEO_PROCESS_ALPHA_FILL_MODE_OPAQUE, + .ColorSpace = s->output_colorspace, + .AlphaFillMode = want_alpha_passthrough ? D3D12_VIDEO_PROCESS_ALPHA_FILL_MODE_SOURCE_STREAM + : D3D12_VIDEO_PROCESS_ALPHA_FILL_MODE_OPAQUE, .AlphaFillModeSourceStreamIndex = 0, .BackgroundColor = { 0.0f, 0.0f, 0.0f, 1.0f }, .FrameRate = { s->input_framerate.num, s->input_framerate.den }, @@ -310,8 +1001,8 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext D3D12_VIDEO_PROCESS_INPUT_STREAM_DESC processor_input_desc = { .Format = s->input_format, .ColorSpace = s->input_colorspace, - .SourceAspectRatio = { s->input_width, s->input_height }, - .DestinationAspectRatio = { s->width, s->height }, + .SourceAspectRatio = { 1, 1 }, + .DestinationAspectRatio = { 1, 1 }, .FrameRate = { s->input_framerate.num, s->input_framerate.den }, .StereoFormat = D3D12_VIDEO_FRAME_STEREO_FORMAT_NONE, .FieldType = D3D12_VIDEO_FIELD_TYPE_NONE, @@ -337,15 +1028,6 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext .EnableAutoProcessing = FALSE, }; - /* If pixel aspect ratio adjustment is not supported, set to 1:1 and warn */ - if (!(s->process_support.FeatureSupport & D3D12_VIDEO_PROCESS_FEATURE_FLAG_PIXEL_ASPECT_RATIO)) { - processor_input_desc.SourceAspectRatio.Numerator = 1; - processor_input_desc.SourceAspectRatio.Denominator = 1; - processor_input_desc.DestinationAspectRatio.Numerator = 1; - processor_input_desc.DestinationAspectRatio.Denominator = 1; - av_log(ctx, AV_LOG_WARNING, "Pixel aspect ratio adjustment not supported by hardware\n"); - } - hr = ID3D12VideoDevice_CreateVideoProcessor( s->video_device, 0, @@ -358,12 +1040,14 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to create video processor: HRESULT 0x%lX\n", hr); + release_d3d12_resources(s, ctx); return AVERROR_EXTERNAL; } hr = ID3D12Device_CreateFence(s->device, 0, D3D12_FENCE_FLAG_NONE, &IID_ID3D12Fence, (void **)&s->fence); if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to create fence: HRESULT 0x%lX\n", hr); + release_d3d12_resources(s, ctx); return AVERROR_EXTERNAL; } @@ -371,16 +1055,22 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext s->fence_event = CreateEvent(NULL, FALSE, FALSE, NULL); if (!s->fence_event) { av_log(ctx, AV_LOG_ERROR, "Failed to create fence event\n"); + release_d3d12_resources(s, ctx); + return AVERROR_EXTERNAL; } s->allocator_queue = av_fifo_alloc2(2, sizeof(D3D12ScaleCommandAllocator), AV_FIFO_FLAG_AUTO_GROW); - if (!s->allocator_queue) + if (!s->allocator_queue) { + release_d3d12_resources(s, ctx); return AVERROR(ENOMEM); + } ret = scale_d3d12_get_valid_command_allocator(s, ctx, &command_allocator); - if (ret < 0) + if (ret < 0) { + release_d3d12_resources(s, ctx); return ret; + } hr = ID3D12Device_CreateCommandList( s->device, @@ -395,14 +1085,27 @@ static int scale_d3d12_configure_processor(ScaleD3D12Context *s, AVFilterContext if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to create command list: HRESULT 0x%lX\n", hr); ID3D12CommandAllocator_Release(command_allocator); + release_d3d12_resources(s, ctx); + return AVERROR_EXTERNAL; } - ID3D12VideoProcessCommandList_Close(s->command_list); + hr = ID3D12VideoProcessCommandList_Close(s->command_list); + if (FAILED(hr)) { + av_log(ctx, AV_LOG_ERROR, "Failed to close command list: HRESULT 0x%lX\n", hr); + if (release_d3d12_resources(s, ctx)) + ID3D12CommandAllocator_Release(command_allocator); - ret = scale_d3d12_discard_command_allocator(s, command_allocator, 0); - if (ret < 0) + return AVERROR_EXTERNAL; + } + + ret = scale_d3d12_discard_command_allocator(s, ctx, command_allocator, 0, NULL); + if (ret < 0) { + release_d3d12_resources(s, ctx); return ret; + } + + s->configured = 1; av_log(ctx, AV_LOG_VERBOSE, "D3D12 video processor successfully configured\n"); return 0; @@ -417,6 +1120,8 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) int ret = 0; HRESULT hr; ID3D12CommandAllocator *command_allocator = NULL; + HRESULT removed_reason; + int gpu_confirmed_done; if (!in) { av_log(ctx, AV_LOG_ERROR, "Null input frame\n"); @@ -426,6 +1131,7 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) if (!in->hw_frames_ctx) { av_log(ctx, AV_LOG_ERROR, "No hardware frames context in input frame\n"); av_frame_free(&in); + return AVERROR(EINVAL); } @@ -434,6 +1140,7 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) if (!s->hw_device_ctx) { av_log(ctx, AV_LOG_ERROR, "Filter hardware device context is uninitialized\n"); av_frame_free(&in); + return AVERROR(EINVAL); } @@ -443,6 +1150,18 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) if (input_device_ctx->type != filter_device_ctx->type) { av_log(ctx, AV_LOG_ERROR, "Mismatch between input and filter hardware device types\n"); av_frame_free(&in); + + return AVERROR(EINVAL); + } + + AVD3D12VADeviceContext *input_d3d12_hwctx = (AVD3D12VADeviceContext *)input_device_ctx->hwctx; + AVD3D12VADeviceContext *filter_d3d12_hwctx = (AVD3D12VADeviceContext *)filter_device_ctx->hwctx; + if (input_d3d12_hwctx->device != filter_d3d12_hwctx->device) { + av_log(ctx, AV_LOG_ERROR, + "Input frame's hw_frames_ctx belongs to a different " + "ID3D12Device than this filter's own device\n"); + av_frame_free(&in); + return AVERROR(EINVAL); } @@ -450,6 +1169,7 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) if (!out) { av_log(ctx, AV_LOG_ERROR, "Failed to allocate output frame\n"); av_frame_free(&in); + return AVERROR(ENOMEM); } @@ -459,37 +1179,161 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) goto fail; } - if (!s->video_processor) { + if (!s->configured) { AVHWFramesContext *input_frames_ctx = (AVHWFramesContext *)in->hw_frames_ctx->data; + const D3D12ScaleFormatMap *in_map, *out_map; + enum AVColorSpace out_spc; + enum AVColorRange out_range; + enum AVColorPrimaries out_primaries; + enum AVColorTransferCharacteristic out_trc; - s->input_width = input_frames_ctx->width; - s->input_height = input_frames_ctx->height; + s->input_width = in->width; + s->input_height = in->height; AVD3D12VAFramesContext *input_hwctx = (AVD3D12VAFramesContext *)input_frames_ctx->hwctx; s->input_format = input_hwctx->format; if (s->input_format == DXGI_FORMAT_UNKNOWN) { - switch (input_frames_ctx->sw_format) { - case AV_PIX_FMT_NV12: - s->input_format = DXGI_FORMAT_NV12; - break; - case AV_PIX_FMT_P010: - s->input_format = DXGI_FORMAT_P010; - break; - default: - av_log(ctx, AV_LOG_ERROR, "Unsupported input format\n"); - ret = AVERROR(EINVAL); - goto fail; + in_map = d3d12_scale_map_from_pixfmt(input_frames_ctx->sw_format); + if (!in_map) { + av_log(ctx, AV_LOG_ERROR, "Unsupported input format: %s\n", + av_get_pix_fmt_name(input_frames_ctx->sw_format)); + ret = AVERROR(EINVAL); + goto fail; } + s->input_format = in_map->dxgi_format; } - int is_10bit = (s->input_format == DXGI_FORMAT_P010); - s->input_colorspace = get_dxgi_colorspace(in->colorspace, in->color_trc, is_10bit); + in_map = d3d12_scale_map_from_dxgi(s->input_format); + s->input_is_rgb = in_map ? in_map->is_rgb : 0; + const AVPixFmtDescriptor *in_desc = in_map ? av_pix_fmt_desc_get(in_map->pix_fmt) : NULL; + s->input_has_alpha = in_desc && (in_desc->flags & AV_PIX_FMT_FLAG_ALPHA); + + enum AVColorRange in_range = in->color_range != AVCOL_RANGE_UNSPECIFIED ? in->color_range + : (s->input_is_rgb ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG); + enum AVColorSpace in_spc = in->colorspace; + enum AVColorPrimaries in_primaries = in->color_primaries; + enum AVColorTransferCharacteristic in_trc = in->color_trc; + + ret = d3d12_scale_get_colorspace(ctx, s->input_is_rgb, 0, &in_spc, &in_range, + &in_primaries, &in_trc, in->chroma_location, + AVCOL_PRI_UNSPECIFIED, &s->input_colorspace); + if (ret < 0) + goto fail; + + s->input_av_color_primaries = in_primaries; + s->input_av_color_trc = in_trc; + + enum AVColorPrimaries input_601_hint = AVCOL_PRI_UNSPECIFIED; + if (s->input_colorspace == DXGI_COLOR_SPACE_YCBCR_STUDIO_G22_LEFT_P601 || + s->input_colorspace == DXGI_COLOR_SPACE_YCBCR_FULL_G22_LEFT_P601) + input_601_hint = in_primaries; + + /* Output color properties: options override, otherwise inherit + * from the input. RGB output defaults to full range. */ + out_map = d3d12_scale_map_from_dxgi(s->output_format); + s->output_is_rgb = out_map ? out_map->is_rgb : 0; + const AVPixFmtDescriptor *out_desc = out_map ? av_pix_fmt_desc_get(out_map->pix_fmt) : NULL; + s->output_has_alpha = out_desc && (out_desc->flags & AV_PIX_FMT_FLAG_ALPHA); + + out_spc = s->color_matrix != AVCOL_SPC_UNSPECIFIED ? s->color_matrix : + s->color_primaries != AVCOL_PRI_UNSPECIFIED ? AVCOL_SPC_UNSPECIFIED : + in->colorspace; + out_primaries = s->color_primaries != AVCOL_PRI_UNSPECIFIED ? s->color_primaries : in_primaries; + out_trc = s->color_transfer != AVCOL_TRC_UNSPECIFIED ? s->color_transfer : in->color_trc; + if (s->color_transfer == AVCOL_TRC_UNSPECIFIED && in->color_trc == AVCOL_TRC_UNSPECIFIED && + out_map && out_map->pix_fmt == AV_PIX_FMT_RGBAF16) + out_trc = AVCOL_TRC_LINEAR; + /* AVCOL_SPC_RGB has no YCbCr matrix of its own, so when converting + * an RGB input frame back to a YUV output format Treat it as unspecified + * instead */ + if (!s->output_is_rgb && out_spc == AVCOL_SPC_RGB) + out_spc = AVCOL_SPC_UNSPECIFIED; + + if (s->color_range != AVCOL_RANGE_UNSPECIFIED) + out_range = s->color_range; + else if (out_trc == AVCOL_TRC_LINEAR) + /* Linear RGB has no studio-range DXGI color space either default + * to full range and let the processor do the limited->full expansion + * instead. */ + out_range = AVCOL_RANGE_JPEG; + else if (in->color_range != AVCOL_RANGE_UNSPECIFIED) + out_range = in->color_range; + else + out_range = s->output_is_rgb ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + + ret = d3d12_scale_get_colorspace(ctx, s->output_is_rgb, 1, &out_spc, &out_range, + &out_primaries, &out_trc, in->chroma_location, + input_601_hint, &s->output_colorspace); + if (ret < 0) + goto fail; + + if (out_trc != AVCOL_TRC_SMPTE2084 && out_trc != AVCOL_TRC_ARIB_STD_B67 && + out_trc != AVCOL_TRC_LINEAR) { + ret = d3d12_scale_finalize_output_trc(ctx, s->input_colorspace, s->output_colorspace, + s->color_transfer != AVCOL_TRC_UNSPECIFIED, + in->color_trc, &out_trc); + if (ret < 0) + goto fail; + } + + if (s->output_is_rgb) + s->output_av_colorspace = AVCOL_SPC_RGB; + else if (out_spc != AVCOL_SPC_UNSPECIFIED) + s->output_av_colorspace = out_spc; + else if (out_primaries == AVCOL_PRI_BT2020) + s->output_av_colorspace = AVCOL_SPC_BT2020_NCL; + else if (out_primaries == AVCOL_PRI_BT470BG || out_primaries == AVCOL_PRI_SMPTE170M) + s->output_av_colorspace = AVCOL_SPC_SMPTE170M; + else + s->output_av_colorspace = AVCOL_SPC_BT709; + s->output_av_color_range = out_range; + s->output_av_color_primaries = out_primaries; + s->output_av_color_trc = out_trc; + + /* outlink's colorspace/color_range/chroma_location describe a YUV, + * subsampled surface; they do not constrain an RGB output, so these + * checks only apply when the output is YUV. */ + if (!s->output_is_rgb && outlink->colorspace != AVCOL_SPC_UNSPECIFIED && + outlink->colorspace != s->output_av_colorspace) { + av_log(ctx, AV_LOG_ERROR, + "Negotiated output color space (%s) does not match the " + "%s produced from this input; set an explicit " + "out_color_matrix\n", + av_color_space_name(outlink->colorspace), + av_color_space_name(s->output_av_colorspace)); + ret = AVERROR(EINVAL); + goto fail; + } + if (!s->output_is_rgb && outlink->color_range != AVCOL_RANGE_UNSPECIFIED && + outlink->color_range != s->output_av_color_range) { + av_log(ctx, AV_LOG_ERROR, + "Negotiated output color range (%s) does not match the " + "%s produced from this input; set an explicit out_range\n", + av_color_range_name(outlink->color_range), + av_color_range_name(s->output_av_color_range)); + ret = AVERROR(EINVAL); + goto fail; + } + if (!s->output_is_rgb) { + enum AVChromaLocation out_chroma_loc = d3d12_scale_dxgi_chroma_location(s->output_colorspace); + if (outlink->chroma_location != AVCHROMA_LOC_UNSPECIFIED && + outlink->chroma_location != out_chroma_loc) { + av_log(ctx, AV_LOG_ERROR, + "Negotiated output chroma siting does not match the " + "siting produced from this input\n"); + ret = AVERROR(EINVAL); + goto fail; + } + } s->input_framerate = get_input_framerate(ctx, inlink, in); - av_log(ctx, AV_LOG_VERBOSE, "Input format: %dx%d, DXGI format: %d, colorspace: %d, framerate: %d/%d\n", + av_log(ctx, AV_LOG_VERBOSE, + "Input format: %dx%d, DXGI format: %d, colorspace: %d -> " + "Output DXGI format: %d, colorspace: %d, framerate: %d/%d\n", s->input_width, s->input_height, s->input_format, s->input_colorspace, + s->output_format, s->output_colorspace, s->input_framerate.num, s->input_framerate.den); ret = scale_d3d12_configure_processor(s, ctx); @@ -499,6 +1343,19 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) } } + /* alpha_mode is a per-frame AVFrame property, not a fixed property of + * the negotiated pixel formats, so this must be checked for every + * frame, not just the first one. */ + if (s->input_has_alpha && in->alpha_mode == AVALPHA_MODE_PREMULTIPLIED && + (!s->output_has_alpha || !s->alpha_passthrough || + s->input_colorspace != s->output_colorspace)) { + av_log(ctx, AV_LOG_ERROR, + "Input has premultiplied alpha, which this conversion " + "cannot preserve; unpremultiply before scale_d3d12\n"); + ret = AVERROR(EINVAL); + goto fail; + } + AVD3D12VAFrame *input_frame = (AVD3D12VAFrame *)in->data[0]; AVD3D12VAFrame *output_frame = (AVD3D12VAFrame *)out->data[0]; @@ -517,19 +1374,6 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) goto fail; } - /* Wait for input frame's fence before accessing it */ - if (input_frame->sync_ctx.fence && input_frame->sync_ctx.fence_value > 0) { - UINT64 completed = ID3D12Fence_GetCompletedValue(input_frame->sync_ctx.fence); - if (completed < input_frame->sync_ctx.fence_value) { - hr = ID3D12CommandQueue_Wait(s->command_queue, input_frame->sync_ctx.fence, input_frame->sync_ctx.fence_value); - if (FAILED(hr)) { - av_log(ctx, AV_LOG_ERROR, "Failed to wait for input fence: HRESULT 0x%lX\n", hr); - ret = AVERROR_EXTERNAL; - goto fail; - } - } - } - ret = scale_d3d12_get_valid_command_allocator(s, ctx, &command_allocator); if (ret < 0) goto fail; @@ -545,37 +1389,55 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to reset command list: HRESULT 0x%lX\n", hr); ret = AVERROR_EXTERNAL; - goto fail; + goto force_reconfigure; } - D3D12_RESOURCE_BARRIER barriers[2] = { - { + D3D12_RESOURCE_DESC input_res_desc, output_res_desc; + input_resource->lpVtbl->GetDesc(input_resource, &input_res_desc); + output_resource->lpVtbl->GetDesc(output_resource, &output_res_desc); + + int input_nb_planes = s->input_is_rgb ? 1 : 2; + int output_nb_planes = s->output_is_rgb ? 1 : 2; + + D3D12_RESOURCE_BARRIER barriers[4]; + int nb_barriers = 0; + + for (int p = 0; p < input_nb_planes; p++) { + D3D12_RESOURCE_BARRIER b = { .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION, .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE, .Transition = { .pResource = input_resource, - .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, + .Subresource = input_frame->subresource_index + p * input_res_desc.DepthOrArraySize, .StateBefore = D3D12_RESOURCE_STATE_COMMON, .StateAfter = D3D12_RESOURCE_STATE_VIDEO_PROCESS_READ } - }, - { + }; + barriers[nb_barriers++] = b; + } + for (int p = 0; p < output_nb_planes; p++) { + D3D12_RESOURCE_BARRIER b = { .Type = D3D12_RESOURCE_BARRIER_TYPE_TRANSITION, .Flags = D3D12_RESOURCE_BARRIER_FLAG_NONE, .Transition = { .pResource = output_resource, - .Subresource = D3D12_RESOURCE_BARRIER_ALL_SUBRESOURCES, + .Subresource = output_frame->subresource_index + p * output_res_desc.DepthOrArraySize, .StateBefore = D3D12_RESOURCE_STATE_COMMON, .StateAfter = D3D12_RESOURCE_STATE_VIDEO_PROCESS_WRITE } - } - }; + }; + barriers[nb_barriers++] = b; + } - ID3D12VideoProcessCommandList_ResourceBarrier(s->command_list, 2, barriers); + ID3D12VideoProcessCommandList_ResourceBarrier(s->command_list, nb_barriers, barriers); D3D12_VIDEO_PROCESS_INPUT_STREAM_ARGUMENTS input_args = {0}; - input_args.InputStream[0].pTexture2D = input_resource; + /* subresource_index is 0 for a plain texture, but for a texture-array + * input (AV_D3D12VA_FRAME_FLAG_TEXTURE_ARRAY) it is the real array + * slice and must be passed through as-is. */ + input_args.InputStream[0].pTexture2D = input_resource; + input_args.InputStream[0].Subresource = input_frame->subresource_index; input_args.Transform.SourceRectangle.right = s->input_width; input_args.Transform.SourceRectangle.bottom = s->input_height; input_args.Transform.DestinationRectangle.right = s->width; @@ -594,7 +1456,9 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) D3D12_VIDEO_PROCESS_OUTPUT_STREAM_ARGUMENTS output_args = {0}; - output_args.OutputStream[0].pTexture2D = output_resource; + /* Same reasoning as input_args.InputStream[0].Subresource above. */ + output_args.OutputStream[0].pTexture2D = output_resource; + output_args.OutputStream[0].Subresource = output_frame->subresource_index; output_args.TargetRectangle.right = s->width; output_args.TargetRectangle.bottom = s->height; @@ -606,56 +1470,191 @@ static int scale_d3d12_filter_frame(AVFilterLink *inlink, AVFrame *in) &input_args ); - for (int i = 0; i < 2; i++) { + for (int i = 0; i < nb_barriers; i++) { FFSWAP(D3D12_RESOURCE_STATES, barriers[i].Transition.StateBefore, barriers[i].Transition.StateAfter); } - ID3D12VideoProcessCommandList_ResourceBarrier(s->command_list, 2, barriers); + ID3D12VideoProcessCommandList_ResourceBarrier(s->command_list, nb_barriers, barriers); hr = ID3D12VideoProcessCommandList_Close(s->command_list); if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to close command list: HRESULT 0x%lX\n", hr); ret = AVERROR_EXTERNAL; - goto fail; + goto force_reconfigure; + } + + int wait_enqueued = 0; + + /* Wait for the input frame's fence so prior GPU work on this + * surface completes before the video processor reads it. */ + if (input_frame->sync_ctx.fence && input_frame->sync_ctx.fence_value > 0) { + UINT64 completed = ID3D12Fence_GetCompletedValue(input_frame->sync_ctx.fence); + if (completed < input_frame->sync_ctx.fence_value) { + hr = ID3D12CommandQueue_Wait(s->command_queue, input_frame->sync_ctx.fence, input_frame->sync_ctx.fence_value); + if (FAILED(hr)) { + av_log(ctx, AV_LOG_ERROR, "Failed to wait for input fence: HRESULT 0x%lX\n", hr); + ret = AVERROR_EXTERNAL; + goto fail; + } + wait_enqueued = 1; + s->gpu_work_submitted = 1; + } + } + + if (output_frame->sync_ctx.fence && output_frame->sync_ctx.fence_value > 0) { + UINT64 completed = ID3D12Fence_GetCompletedValue(output_frame->sync_ctx.fence); + if (completed < output_frame->sync_ctx.fence_value) { + hr = ID3D12CommandQueue_Wait(s->command_queue, output_frame->sync_ctx.fence, output_frame->sync_ctx.fence_value); + if (FAILED(hr)) { + av_log(ctx, AV_LOG_ERROR, "Failed to wait for output frame fence: HRESULT 0x%lX\n", hr); + ret = AVERROR_EXTERNAL; + if (wait_enqueued) + goto submit_fail; + goto fail; + } + s->gpu_work_submitted = 1; + } } ID3D12CommandList *cmd_lists[] = { (ID3D12CommandList *)s->command_list }; ID3D12CommandQueue_ExecuteCommandLists(s->command_queue, 1, cmd_lists); + s->gpu_work_submitted = 1; - hr = ID3D12CommandQueue_Signal(s->command_queue, s->fence, s->fence_value); + UINT64 next_output_fence_value = output_frame->sync_ctx.fence_value + 1; + UINT64 next_private_fence_value = s->fence_value; + + hr = ID3D12CommandQueue_Signal(s->command_queue, output_frame->sync_ctx.fence, + next_output_fence_value); + if (FAILED(hr)) { + av_log(ctx, AV_LOG_ERROR, "Failed to signal output frame fence: HRESULT 0x%lX\n", hr); + ret = AVERROR_EXTERNAL; + goto submit_fail; + } + output_frame->sync_ctx.fence_value = next_output_fence_value; + + hr = ID3D12CommandQueue_Signal(s->command_queue, s->fence, next_private_fence_value); if (FAILED(hr)) { av_log(ctx, AV_LOG_ERROR, "Failed to signal fence: HRESULT 0x%lX\n", hr); ret = AVERROR_EXTERNAL; - goto fail; + goto submit_fail; } - output_frame->sync_ctx.fence = s->fence; - output_frame->sync_ctx.fence_value = s->fence_value; - ID3D12Fence_AddRef(s->fence); ///< Increment reference count + s->fence_value++; + + AVFrame *input_ref = av_frame_clone(in); + if (!input_ref) { + ret = AVERROR(ENOMEM); + goto submit_fail; + } - ret = scale_d3d12_discard_command_allocator(s, command_allocator, s->fence_value); + ret = scale_d3d12_discard_command_allocator(s, ctx, command_allocator, next_private_fence_value, + input_ref); command_allocator = NULL; if (ret < 0) goto fail; - s->fence_value++; - ret = av_frame_copy_props(out, in); if (ret < 0) { av_log(ctx, AV_LOG_ERROR, "Failed to copy frame properties\n"); goto fail; } + int in_width = in->width; + int in_height = in->height; + out->width = s->width; out->height = s->height; out->format = AV_PIX_FMT_D3D12; + if (out->width != in_width || out->height != in_height) + av_reduce(&out->sample_aspect_ratio.num, &out->sample_aspect_ratio.den, + (int64_t)in->sample_aspect_ratio.num * out->height * in_width, + (int64_t)in->sample_aspect_ratio.den * out->width * in_height, + INT_MAX); + + if (out->width != in_width || out->height != in_height) { + av_frame_side_data_remove_by_props(&out->side_data, &out->nb_side_data, + AV_SIDE_DATA_PROP_SIZE_DEPENDENT); + out->crop_left = out->crop_right = out->crop_top = out->crop_bottom = 0; + } + + out->colorspace = s->output_av_colorspace; + out->color_range = s->output_av_color_range; + out->color_primaries = s->output_av_color_primaries; + out->color_trc = s->output_av_color_trc; + + if (out->color_primaries != s->input_av_color_primaries || out->color_trc != s->input_av_color_trc) + av_frame_side_data_remove_by_props(&out->side_data, &out->nb_side_data, + AV_SIDE_DATA_PROP_COLOR_DEPENDENT); + + if (!s->output_has_alpha) + out->alpha_mode = AVALPHA_MODE_UNSPECIFIED; + + if (s->output_is_rgb) { + out->chroma_location = AVCHROMA_LOC_UNSPECIFIED; + } else { + out->chroma_location = d3d12_scale_dxgi_chroma_location(s->output_colorspace); + } + av_frame_free(&in); return ff_filter_frame(outlink, out); +submit_fail: + removed_reason = ID3D12Device_GetDeviceRemovedReason(s->device); + gpu_confirmed_done = FAILED(removed_reason); + + if (gpu_confirmed_done) { + av_log(ctx, AV_LOG_ERROR, + "D3D12 device removed (HRESULT 0x%lX); frame " + "resources are safe to release\n", removed_reason); + } else if (s->command_queue && s->fence && s->fence_event) { + UINT64 last_resort = s->fence_value + 1; + if (SUCCEEDED(ID3D12CommandQueue_Signal(s->command_queue, s->fence, last_resort))) { + UINT64 completed; + + if (last_resort >= s->fence_value) + s->fence_value = last_resort + 1; + + completed = ID3D12Fence_GetCompletedValue(s->fence); + if (completed >= last_resort) { + gpu_confirmed_done = 1; + } else { + gpu_confirmed_done = scale_d3d12_wait_fence_exact(s->fence, last_resort, + s->fence_event, 5000); + if (!gpu_confirmed_done) + av_log(ctx, AV_LOG_ERROR, + "Post-failure GPU wait timed out; frame " + "resources cannot be safely recycled\n"); + } + } else { + av_log(ctx, AV_LOG_ERROR, + "Best-effort post-failure GPU sync Signal also " + "failed; frame resources cannot be safely " + "recycled\n"); + } + } else { + av_log(ctx, AV_LOG_ERROR, + "No fence/queue for post-failure GPU sync; frame " + "resources cannot be safely recycled\n"); + } + + if (!gpu_confirmed_done) { + av_log(ctx, AV_LOG_ERROR, + "Abandoning this frame's input/output surfaces and " + "command allocator without releasing them\n"); + return ret; + } + + goto fail; + +force_reconfigure: + s->configured = 0; + if (!release_d3d12_resources(s, ctx)) + command_allocator = NULL; + fail: if (command_allocator) - scale_d3d12_discard_command_allocator(s, command_allocator, s->fence_value); + ID3D12CommandAllocator_Release(command_allocator); av_frame_free(&in); av_frame_free(&out); return ret; @@ -670,8 +1669,13 @@ static int scale_d3d12_config_props(AVFilterLink *outlink) FilterLink *outl = ff_filter_link(outlink); int ret; - release_d3d12_resources(s); + if (!release_d3d12_resources(s, ctx)) { + av_log(ctx, AV_LOG_ERROR, + "Cannot (re)configure scale_d3d12: a prior configuration's " + "D3D12 resources could not be confirmed released\n"); + return AVERROR_EXTERNAL; + } av_buffer_unref(&s->hw_frames_ctx_out); av_buffer_unref(&s->hw_device_ctx); @@ -689,6 +1693,14 @@ static int scale_d3d12_config_props(AVFilterLink *outlink) outlink->w = s->width; outlink->h = s->height; + if (inlink->sample_aspect_ratio.num) { + AVRational q = av_div_q((AVRational){ inlink->w, inlink->h }, + (AVRational){ outlink->w, outlink->h }); + outlink->sample_aspect_ratio = av_mul_q(q, inlink->sample_aspect_ratio); + } else { + outlink->sample_aspect_ratio = inlink->sample_aspect_ratio; + } + if (!inl->hw_frames_ctx) { av_log(ctx, AV_LOG_ERROR, "No hw_frames_ctx available on input link\n"); return AVERROR(EINVAL); @@ -720,35 +1732,43 @@ static int scale_d3d12_config_props(AVFilterLink *outlink) AVHWFramesContext *frames_ctx = (AVHWFramesContext *)s->hw_frames_ctx_out->data; AVHWFramesContext *in_frames_ctx = (AVHWFramesContext *)inl->hw_frames_ctx->data; - if (s->format == AV_PIX_FMT_NONE) { + enum AVPixelFormat resolved_format = s->format; + if (resolved_format == AV_PIX_FMT_NONE) { /* If format is not specified, use the same format as input */ - frames_ctx->sw_format = in_frames_ctx->sw_format; - s->format = in_frames_ctx->sw_format; + resolved_format = in_frames_ctx->sw_format; av_log(ctx, AV_LOG_VERBOSE, "D3D12 scale output format not specified, using input format: %s\n", - av_get_pix_fmt_name(s->format)); - } else { - frames_ctx->sw_format = s->format; + av_get_pix_fmt_name(resolved_format)); } + frames_ctx->sw_format = resolved_format; /* Set output format based on sw_format */ - switch (frames_ctx->sw_format) { - case AV_PIX_FMT_NV12: - s->output_format = DXGI_FORMAT_NV12; - break; - case AV_PIX_FMT_P010: - s->output_format = DXGI_FORMAT_P010; - break; - default: - av_log(ctx, AV_LOG_ERROR, "Unsupported output format: %s\n", - av_get_pix_fmt_name(frames_ctx->sw_format)); - av_buffer_unref(&s->hw_frames_ctx_out); - return AVERROR(EINVAL); + const D3D12ScaleFormatMap *out_map = d3d12_scale_map_from_pixfmt(frames_ctx->sw_format); + if (!out_map) { + av_log(ctx, AV_LOG_ERROR, "Unsupported output format: %s\n", + av_get_pix_fmt_name(frames_ctx->sw_format)); + av_buffer_unref(&s->hw_frames_ctx_out); + + return AVERROR(EINVAL); } + s->output_format = out_map->dxgi_format; + s->output_is_rgb = out_map->is_rgb; + + const D3D12ScaleFormatMap *in_map = d3d12_scale_map_from_pixfmt(in_frames_ctx->sw_format); + int input_is_rgb = in_map ? in_map->is_rgb : 0; + + if (s->color_primaries != AVCOL_PRI_UNSPECIFIED || + s->color_transfer != AVCOL_TRC_UNSPECIFIED || + s->color_matrix != AVCOL_SPC_UNSPECIFIED || + input_is_rgb != s->output_is_rgb || + frames_ctx->sw_format == AV_PIX_FMT_RGBAF16 || + (input_is_rgb && s->color_range != AVCOL_RANGE_UNSPECIFIED)) + av_frame_side_data_remove_by_props(&outlink->side_data, &outlink->nb_side_data, + AV_SIDE_DATA_PROP_COLOR_DEPENDENT); frames_ctx->width = s->width; frames_ctx->height = s->height; frames_ctx->format = AV_PIX_FMT_D3D12; - frames_ctx->initial_pool_size = 10; + frames_ctx->initial_pool_size = 4; if (ctx->extra_hw_frames > 0) frames_ctx->initial_pool_size += ctx->extra_hw_frames; @@ -775,19 +1795,94 @@ static int scale_d3d12_config_props(AVFilterLink *outlink) av_log(ctx, AV_LOG_VERBOSE, "D3D12 scale config: %dx%d -> %dx%d\n", inlink->w, inlink->h, outlink->w, outlink->h); + return 0; } static av_cold void scale_d3d12_uninit(AVFilterContext *ctx) { ScaleD3D12Context *s = ctx->priv; - release_d3d12_resources(s); - - av_buffer_unref(&s->hw_frames_ctx_out); - av_buffer_unref(&s->hw_device_ctx); + if (release_d3d12_resources(s, ctx)) { + av_buffer_unref(&s->hw_frames_ctx_out); + av_buffer_unref(&s->hw_device_ctx); + } av_freep(&s->w_expr); av_freep(&s->h_expr); + + av_freep(&s->color_matrix_string); + av_freep(&s->color_transfer_string); + av_freep(&s->color_primaries_string); +} + +static enum AVColorSpace d3d12_scale_matrix_from_primaries(enum AVColorPrimaries primaries) +{ + if (primaries == AVCOL_PRI_BT2020) + return AVCOL_SPC_BT2020_NCL; + if (primaries == AVCOL_PRI_BT470BG || primaries == AVCOL_PRI_SMPTE170M) + return AVCOL_SPC_SMPTE170M; + return AVCOL_SPC_BT709; +} + +static int scale_d3d12_query_formats(const AVFilterContext *avctx, + AVFilterFormatsConfig **cfg_in, + AVFilterFormatsConfig **cfg_out) +{ + static const enum AVPixelFormat pix_fmts[] = { + AV_PIX_FMT_D3D12, AV_PIX_FMT_NONE, + }; + static const int out_chroma_locs_any[] = { + AVCHROMA_LOC_UNSPECIFIED, AVCHROMA_LOC_LEFT, AVCHROMA_LOC_TOPLEFT, -1, + }; + static const int out_chroma_locs_left[] = { + AVCHROMA_LOC_LEFT, -1, + }; + const ScaleD3D12Context *s = avctx->priv; + AVFilterFormats *color_spaces, *color_ranges; + enum AVColorSpace out_matrix; + int out_matrix_known; + int ret; + + ret = ff_set_pixel_formats_from_list2(avctx, cfg_in, cfg_out, pix_fmts); + if (ret < 0) + return ret; + + out_matrix_known = s->color_matrix != AVCOL_SPC_UNSPECIFIED || + s->color_primaries != AVCOL_PRI_UNSPECIFIED; + if (s->color_matrix != AVCOL_SPC_UNSPECIFIED) + out_matrix = s->color_matrix == AVCOL_SPC_BT2020_CL ? AVCOL_SPC_BT2020_NCL : s->color_matrix; + else if (s->color_primaries != AVCOL_PRI_UNSPECIFIED) + out_matrix = d3d12_scale_matrix_from_primaries(s->color_primaries); + else + out_matrix = AVCOL_SPC_UNSPECIFIED; + + color_spaces = ff_all_color_spaces(); + if ((ret = ff_formats_ref(color_spaces, &cfg_in[0]->color_spaces)) < 0) + return ret; + if (out_matrix_known) + color_spaces = ff_make_formats_list_singleton(out_matrix); + if ((ret = ff_formats_ref(color_spaces, &cfg_out[0]->color_spaces)) < 0) + return ret; + + color_ranges = ff_all_color_ranges(); + if ((ret = ff_formats_ref(color_ranges, &cfg_in[0]->color_ranges)) < 0) + return ret; + if (s->color_range != AVCOL_RANGE_UNSPECIFIED) + color_ranges = ff_make_formats_list_singleton(s->color_range); + if ((ret = ff_formats_ref(color_ranges, &cfg_out[0]->color_ranges)) < 0) + return ret; + + if ((ret = ff_formats_ref(ff_all_chroma_locations(), &cfg_in[0]->chroma_locations)) < 0) + return ret; + if (out_matrix_known && out_matrix != AVCOL_SPC_BT2020_NCL) { + if ((ret = ff_formats_ref(ff_make_format_list(out_chroma_locs_left), &cfg_out[0]->chroma_locations)) < 0) + return ret; + } else { + if ((ret = ff_formats_ref(ff_make_format_list(out_chroma_locs_any), &cfg_out[0]->chroma_locations)) < 0) + return ret; + } + + return 0; } static const AVFilterPad scale_d3d12_inputs[] = { @@ -813,6 +1908,24 @@ static const AVOption scale_d3d12_options[] = { { "w", "Output video width", OFFSET(w_expr), AV_OPT_TYPE_STRING, {.str = "iw"}, .flags = FLAGS }, { "h", "Output video height", OFFSET(h_expr), AV_OPT_TYPE_STRING, {.str = "ih"}, .flags = FLAGS }, { "format", "Output video pixel format", OFFSET(format), AV_OPT_TYPE_PIXEL_FMT, { .i64 = AV_PIX_FMT_NONE }, INT_MIN, INT_MAX, .flags=FLAGS }, + + /* These color properties match the identically named ones in vf_scale_vaapi. */ + { "out_color_matrix", "Output color matrix coefficient set", + OFFSET(color_matrix_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS }, + { "out_range", "Output color range", + OFFSET(color_range), AV_OPT_TYPE_INT, { .i64 = AVCOL_RANGE_UNSPECIFIED }, + AVCOL_RANGE_UNSPECIFIED, AVCOL_RANGE_JPEG, FLAGS, .unit = "range" }, + { "full", "Full range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_JPEG }, 0, 0, FLAGS, .unit = "range" }, + { "limited", "Limited range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_MPEG }, 0, 0, FLAGS, .unit = "range" }, + { "jpeg", "Full range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_JPEG }, 0, 0, FLAGS, .unit = "range" }, + { "mpeg", "Limited range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_MPEG }, 0, 0, FLAGS, .unit = "range" }, + { "tv", "Limited range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_MPEG }, 0, 0, FLAGS, .unit = "range" }, + { "pc", "Full range", 0, AV_OPT_TYPE_CONST, { .i64 = AVCOL_RANGE_JPEG }, 0, 0, FLAGS, .unit = "range" }, + { "out_color_primaries", "Output color primaries", + OFFSET(color_primaries_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS }, + { "out_color_transfer", "Output color transfer characteristics", + OFFSET(color_transfer_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS }, + { NULL } }; @@ -827,7 +1940,7 @@ const FFFilter ff_vf_scale_d3d12 = { .uninit = scale_d3d12_uninit, FILTER_INPUTS(scale_d3d12_inputs), FILTER_OUTPUTS(scale_d3d12_outputs), - FILTER_SINGLE_PIXFMT(AV_PIX_FMT_D3D12), + FILTER_QUERY_FUNC2(scale_d3d12_query_formats), .p.flags = AVFILTER_FLAG_HWDEVICE, .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, }; -- To stop receiving notification emails like this one, please contact [email protected]. _______________________________________________ ffmpeg-cvslog mailing list -- [email protected] To unsubscribe send an email to [email protected]
