PR #24827 opened by Forgejo_Fairy URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24827 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24827.patch
Scaling only one dimension can execute an uninitialized intermediate filter pass, producing corrupted output or CUDA errors as reported in #24126. Current release/8.1 at 29e619e767cd produces all-zero frames for both 320×240 → 300×240 and 320×240 → 320×220. Backport the five-commit series from #24154, 50a9e34e1bc4 through a621ef429114, preserving original authorship. It skips disabled passes and unnecessary intermediate allocation, initializes pass selection consistently, and accounts for chroma subsampling. Validated on an RTX 5090 with driver 595.84, nv-codec-headers 12.1.14.0, and Clang 18.1.3 CUDA kernels. One-axis and two-axis downscaling outputs match upstream 50a9e34e1bc4 byte-for-byte. Both ten-frame H.264 encodes pass checks of every decoded frame. Upscaling, nearest interpolation, disabled generic filtering, passthrough, forced filtering, and NV12-to-YUV420P/YUV444P/P010LE conversion checks also pass. Passthrough and format-only YUV420P conversion match direct downloads byte-for-byte. `git diff --check` passes. Reproduction commands and branch comparisons are recorded in #24126. This preserves the upstream series, including the pre-existing odd-size chroma auto-selection limitation documented in #24154; that separate limitation is outside this backport. From e6579884991b36760b0ef319683c4fd53ec9d5b3 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:47:47 +0200 Subject: [PATCH 1/5] avfilter/vf_scale_cuda: don't try to execute disabled filter passes (cherry picked from commit 50a9e34e1bc4bcf54331af39d4a53d8cffde3c22) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 4acb4c91a8..c5d2b2b36f 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -157,6 +157,7 @@ typedef struct CUDAScaleContext { CUDAScaleFilter filters_uv[FILTER_NB]; CUDATex inter_tex; int use_filters; /* -1 for auto */ + int pass_x, pass_y; float param; } CUDAScaleContext; @@ -407,6 +408,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int s->frames_ctx = av_buffer_ref(inl->hw_frames_ctx); if (!s->frames_ctx) return AVERROR(ENOMEM); + + s->use_filters = 0; } else { s->passthrough = 0; @@ -451,9 +454,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx) if (s->use_filters) { /* Final pass is always vertical unless not vertically scaling */ - AVFilterLink *inlink = ctx->inputs[0]; - AVFilterLink *outlink = ctx->outputs[0]; - function_infix = inlink->h == outlink->h ? "Generic_h" : "Generic_v"; + function_infix = s->pass_y == FILTER_OUT ? "Generic_v" : "Generic_h"; s->interp_use_linear = 0; s->interp_as_integer = 0; } else { @@ -623,19 +624,19 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - int pass_x = -1, pass_y = -1; + s->pass_x = s->pass_y = -1; if (inlink->w != outlink->w && inlink->h != outlink->h) { /* Always perform the horizontal scaling pass first */ - pass_x = FILTER_TMP; - pass_y = FILTER_OUT; + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { - pass_x = FILTER_OUT; + s->pass_x = FILTER_OUT; } else if (inlink->h != outlink->h) { - pass_y = FILTER_OUT; + s->pass_y = FILTER_OUT; } - if (pass_x >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_x], + if (s->pass_x >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); if (ret < 0) goto fail; @@ -643,15 +644,15 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->w, in_sub_x); const int dst_size = AV_CEIL_RSHIFT(outlink->w, out_sub_x); const double virtual_size = (double) outlink->w / (1 << out_sub_x); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_x], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_x], src_size, dst_size, virtual_size); if (ret < 0) goto fail; } } - if (pass_y >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_y], + if (s->pass_y >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_y], inlink->h, outlink->h, 0.0); if (ret < 0) goto fail; @@ -659,7 +660,7 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->h, in_sub_y); const int dst_size = AV_CEIL_RSHIFT(outlink->h, out_sub_y); const double virtual_size = (double) outlink->h / (1 << out_sub_y); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_y], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_y], src_size, dst_size, virtual_size); if (ret < 0) goto fail; @@ -912,7 +913,7 @@ static int cudascale_scale(AVFilterContext *ctx, AVFrame *out, AVFrame *in) goto fail; const CUDATex *src = &in_tex; - if (s->use_filters) { + if (s->pass_x == FILTER_TMP) { /* Handle first pass separately */ s->inter_tex.color_range = in->color_range; ret = scalecuda_resize(ctx, FILTER_TMP, &s->inter_tex, src); -- 2.52.0 From c11c663ca52dc77833b15645d7e8ce36e063f666 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:56:20 +0200 Subject: [PATCH 2/5] avfilter/vf_scale_cuda: skip allocating intermediate texture if not needed (cherry picked from commit 3d7ea1dac4007b4897ad00991268bbdb51816743) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index c5d2b2b36f..20a4c80c28 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -667,9 +667,11 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) } } - ret = inter_buf_init(ctx, outlink->w, inlink->h); - if (ret < 0) - goto fail; + if (s->pass_x == FILTER_TMP) { + ret = inter_buf_init(ctx, outlink->w, inlink->h); + if (ret < 0) + goto fail; + } ret = 0; -- 2.52.0 From 39ee21dc9a8c40f6d1aa69e880f414692f226e80 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:13:16 +0200 Subject: [PATCH 3/5] avfilter/vf_scale_cuda: extract pass determination into its own function (cherry picked from commit dda8b8253cb020d0a09942e6507505ab608128b3) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 20a4c80c28..a1cc3c1968 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -369,6 +369,27 @@ static av_cold void set_format_info(AVFilterContext *ctx, enum AVPixelFormat in_ } } +static av_cold void cudascale_setup_passes(AVFilterContext *ctx) +{ + CUDAScaleContext *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *outlink = ctx->outputs[0]; + + s->pass_x = s->pass_y = -1; + if (!s->use_filters) + return; + + if (inlink->w != outlink->w && inlink->h != outlink->h) { + /* Always perform the horizontal scaling pass first */ + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; + } else if (inlink->w != outlink->w) { + s->pass_x = FILTER_OUT; + } else if (inlink->h != outlink->h) { + s->pass_y = FILTER_OUT; + } +} + static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int in_height, int out_width, int out_height) { @@ -433,6 +454,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (!outl->hw_frames_ctx) return AVERROR(ENOMEM); + cudascale_setup_passes(ctx); + return 0; } @@ -624,17 +647,6 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - s->pass_x = s->pass_y = -1; - if (inlink->w != outlink->w && inlink->h != outlink->h) { - /* Always perform the horizontal scaling pass first */ - s->pass_x = FILTER_TMP; - s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { - s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { - s->pass_y = FILTER_OUT; - } - if (s->pass_x >= 0) { ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); -- 2.52.0 From de8f9b5f273df41d6478d6a822ba0b20be81fd91 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:26:14 +0200 Subject: [PATCH 4/5] avfilter/vf_scale_cuda: always have valid passes setup (cherry picked from commit 4566eee58816d9a927b460f73c773d18e6060274) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index a1cc3c1968..2dae1e60a6 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -385,7 +385,7 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { + } else { s->pass_y = FILTER_OUT; } } -- 2.52.0 From 330caae0c1acccd2222edc52a05940c574561ce5 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:52:00 +0200 Subject: [PATCH 5/5] avfilter/vf_scale_cuda: consider chroma subsampling for scaling decisions (cherry picked from commit a621ef4291148739e6a301ac5e7aee839c5d0933) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 2dae1e60a6..1bc7ca67da 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -379,11 +379,16 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) if (!s->use_filters) return; - if (inlink->w != outlink->w && inlink->h != outlink->h) { + const int scale_x = inlink->w != outlink->w || + s->in_desc->log2_chroma_w != s->out_desc->log2_chroma_w; + const int scale_y = inlink->h != outlink->h || + s->in_desc->log2_chroma_h != s->out_desc->log2_chroma_h; + + if (scale_x && scale_y) { /* Always perform the horizontal scaling pass first */ s->pass_x = FILTER_TMP; s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { + } else if (scale_x) { s->pass_x = FILTER_OUT; } else { s->pass_y = FILTER_OUT; @@ -444,7 +449,11 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (s->interp_algo == INTERP_ALGO_NEAREST) { s->use_filters = 0; - } else if (s->use_filters < 0 && (out_width < in_width || out_height < in_height)) + } else if (s->use_filters < 0 && ( + out_width < in_width || out_height < in_height || + AV_CEIL_RSHIFT(out_width, s->out_desc->log2_chroma_w) < AV_CEIL_RSHIFT(in_width, s->in_desc->log2_chroma_w) || + AV_CEIL_RSHIFT(out_height, s->out_desc->log2_chroma_h) < AV_CEIL_RSHIFT(in_height, s->in_desc->log2_chroma_h) + )) s->use_filters = 1; /* downscaling; needed for anti-aliasing */ else if (s->use_filters < 0) s->use_filters = 0; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
