PR #24826 opened by Forgejo_Fairy URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24826 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24826.patch
Scaling only one dimension can execute an uninitialized intermediate filter pass, producing corrupted output or CUDA errors as reported in #24126. Current release/8.0 at a11352c9a0cc produces all-zero frames for both 320×240 → 300×240 and 320×240 → 320×220. Backport the five-commit series from #24154, 50a9e34e1bc4 through a621ef429114, preserving original authorship. It skips disabled passes and unnecessary intermediate allocation, initializes pass selection consistently, and accounts for chroma subsampling. Validated on an RTX 5090 with driver 595.84, nv-codec-headers 12.1.14.0, and Clang 18.1.3 CUDA kernels. One-axis and two-axis downscaling outputs match upstream 50a9e34e1bc4 byte-for-byte. Both ten-frame H.264 encodes pass checks of every decoded frame. Upscaling, nearest interpolation, disabled generic filtering, passthrough, forced filtering, and NV12-to-YUV420P/YUV444P/P010LE conversion checks also pass. Passthrough and format-only YUV420P conversion match direct downloads byte-for-byte. `git diff --check` passes. Reproduction commands and branch comparisons are recorded in #24126. This preserves the upstream series, including the pre-existing odd-size chroma auto-selection limitation documented in #24154; that separate limitation is outside this backport. From 17ddfe3ef322fb1b5b329a94ff60dfd249524f1f Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:47:47 +0200 Subject: [PATCH 1/5] avfilter/vf_scale_cuda: don't try to execute disabled filter passes (cherry picked from commit 50a9e34e1bc4bcf54331af39d4a53d8cffde3c22) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 7217c318c6..ca0619d8b3 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -157,6 +157,7 @@ typedef struct CUDAScaleContext { CUDAScaleFilter filters_uv[FILTER_NB]; CUDATex inter_tex; int use_filters; /* -1 for auto */ + int pass_x, pass_y; float param; } CUDAScaleContext; @@ -407,6 +408,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int s->frames_ctx = av_buffer_ref(inl->hw_frames_ctx); if (!s->frames_ctx) return AVERROR(ENOMEM); + + s->use_filters = 0; } else { s->passthrough = 0; @@ -451,9 +454,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx) if (s->use_filters) { /* Final pass is always vertical unless not vertically scaling */ - AVFilterLink *inlink = ctx->inputs[0]; - AVFilterLink *outlink = ctx->outputs[0]; - function_infix = inlink->h == outlink->h ? "Generic_h" : "Generic_v"; + function_infix = s->pass_y == FILTER_OUT ? "Generic_v" : "Generic_h"; s->interp_use_linear = 0; s->interp_as_integer = 0; } else { @@ -623,19 +624,19 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - int pass_x = -1, pass_y = -1; + s->pass_x = s->pass_y = -1; if (inlink->w != outlink->w && inlink->h != outlink->h) { /* Always perform the horizontal scaling pass first */ - pass_x = FILTER_TMP; - pass_y = FILTER_OUT; + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { - pass_x = FILTER_OUT; + s->pass_x = FILTER_OUT; } else if (inlink->h != outlink->h) { - pass_y = FILTER_OUT; + s->pass_y = FILTER_OUT; } - if (pass_x >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_x], + if (s->pass_x >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); if (ret < 0) goto fail; @@ -643,15 +644,15 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->w, in_sub_x); const int dst_size = AV_CEIL_RSHIFT(outlink->w, out_sub_x); const double virtual_size = (double) outlink->w / (1 << out_sub_x); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_x], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_x], src_size, dst_size, virtual_size); if (ret < 0) goto fail; } } - if (pass_y >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_y], + if (s->pass_y >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_y], inlink->h, outlink->h, 0.0); if (ret < 0) goto fail; @@ -659,7 +660,7 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->h, in_sub_y); const int dst_size = AV_CEIL_RSHIFT(outlink->h, out_sub_y); const double virtual_size = (double) outlink->h / (1 << out_sub_y); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_y], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_y], src_size, dst_size, virtual_size); if (ret < 0) goto fail; @@ -912,7 +913,7 @@ static int cudascale_scale(AVFilterContext *ctx, AVFrame *out, AVFrame *in) goto fail; const CUDATex *src = &in_tex; - if (s->use_filters) { + if (s->pass_x == FILTER_TMP) { /* Handle first pass separately */ s->inter_tex.color_range = in->color_range; ret = scalecuda_resize(ctx, FILTER_TMP, &s->inter_tex, src); -- 2.52.0 From f54f34e2b8c0921eca335842f719c85a8096e42d Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:56:20 +0200 Subject: [PATCH 2/5] avfilter/vf_scale_cuda: skip allocating intermediate texture if not needed (cherry picked from commit 3d7ea1dac4007b4897ad00991268bbdb51816743) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index ca0619d8b3..552d01e112 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -667,9 +667,11 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) } } - ret = inter_buf_init(ctx, outlink->w, inlink->h); - if (ret < 0) - goto fail; + if (s->pass_x == FILTER_TMP) { + ret = inter_buf_init(ctx, outlink->w, inlink->h); + if (ret < 0) + goto fail; + } ret = 0; -- 2.52.0 From f5ee98d5cb040395c4433a69b0d8381592954449 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:13:16 +0200 Subject: [PATCH 3/5] avfilter/vf_scale_cuda: extract pass determination into its own function (cherry picked from commit dda8b8253cb020d0a09942e6507505ab608128b3) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 552d01e112..e62f8b1a5e 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -369,6 +369,27 @@ static av_cold void set_format_info(AVFilterContext *ctx, enum AVPixelFormat in_ } } +static av_cold void cudascale_setup_passes(AVFilterContext *ctx) +{ + CUDAScaleContext *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *outlink = ctx->outputs[0]; + + s->pass_x = s->pass_y = -1; + if (!s->use_filters) + return; + + if (inlink->w != outlink->w && inlink->h != outlink->h) { + /* Always perform the horizontal scaling pass first */ + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; + } else if (inlink->w != outlink->w) { + s->pass_x = FILTER_OUT; + } else if (inlink->h != outlink->h) { + s->pass_y = FILTER_OUT; + } +} + static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int in_height, int out_width, int out_height) { @@ -433,6 +454,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (!outl->hw_frames_ctx) return AVERROR(ENOMEM); + cudascale_setup_passes(ctx); + return 0; } @@ -624,17 +647,6 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - s->pass_x = s->pass_y = -1; - if (inlink->w != outlink->w && inlink->h != outlink->h) { - /* Always perform the horizontal scaling pass first */ - s->pass_x = FILTER_TMP; - s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { - s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { - s->pass_y = FILTER_OUT; - } - if (s->pass_x >= 0) { ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); -- 2.52.0 From 8f19abedf1a4a18d9f32a7d30daf1b1be6fab5fc Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:26:14 +0200 Subject: [PATCH 4/5] avfilter/vf_scale_cuda: always have valid passes setup (cherry picked from commit 4566eee58816d9a927b460f73c773d18e6060274) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index e62f8b1a5e..dbc55e4b74 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -385,7 +385,7 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { + } else { s->pass_y = FILTER_OUT; } } -- 2.52.0 From 44c539c742ce16f5df6672341e9f9aacce59e780 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:52:00 +0200 Subject: [PATCH 5/5] avfilter/vf_scale_cuda: consider chroma subsampling for scaling decisions (cherry picked from commit a621ef4291148739e6a301ac5e7aee839c5d0933) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index dbc55e4b74..abb3998cfd 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -379,11 +379,16 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) if (!s->use_filters) return; - if (inlink->w != outlink->w && inlink->h != outlink->h) { + const int scale_x = inlink->w != outlink->w || + s->in_desc->log2_chroma_w != s->out_desc->log2_chroma_w; + const int scale_y = inlink->h != outlink->h || + s->in_desc->log2_chroma_h != s->out_desc->log2_chroma_h; + + if (scale_x && scale_y) { /* Always perform the horizontal scaling pass first */ s->pass_x = FILTER_TMP; s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { + } else if (scale_x) { s->pass_x = FILTER_OUT; } else { s->pass_y = FILTER_OUT; @@ -444,7 +449,11 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (s->interp_algo == INTERP_ALGO_NEAREST) { s->use_filters = 0; - } else if (s->use_filters < 0 && (out_width < in_width || out_height < in_height)) + } else if (s->use_filters < 0 && ( + out_width < in_width || out_height < in_height || + AV_CEIL_RSHIFT(out_width, s->out_desc->log2_chroma_w) < AV_CEIL_RSHIFT(in_width, s->in_desc->log2_chroma_w) || + AV_CEIL_RSHIFT(out_height, s->out_desc->log2_chroma_h) < AV_CEIL_RSHIFT(in_height, s->in_desc->log2_chroma_h) + )) s->use_filters = 1; /* downscaling; needed for anti-aliasing */ else if (s->use_filters < 0) s->use_filters = 0; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
