PR #24828 opened by Forgejo_Fairy URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24828 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/24828.patch
Scaling only one dimension can execute an uninitialized intermediate filter pass, producing corrupted output or CUDA errors as reported in #24126. Current release/9.0 at 2a571b606854 produces all-zero frames for both 320×240 → 300×240 and 320×240 → 320×220. Backport the five-commit series from #24154, 50a9e34e1bc4 through a621ef429114, preserving original authorship. It skips disabled passes and unnecessary intermediate allocation, initializes pass selection consistently, and accounts for chroma subsampling. Validated on an RTX 5090 with driver 595.84, nv-codec-headers 12.1.14.0, and Clang 18.1.3 CUDA kernels. One-axis and two-axis downscaling outputs match upstream 50a9e34e1bc4 byte-for-byte. Both ten-frame H.264 encodes pass checks of every decoded frame. Upscaling, nearest interpolation, disabled generic filtering, passthrough, forced filtering, and NV12-to-YUV420P/YUV444P/P010LE conversion checks also pass. Passthrough and format-only YUV420P conversion match direct downloads byte-for-byte. `git diff --check` passes. Reproduction commands and branch comparisons are recorded in #24126. This preserves the upstream series, including the pre-existing odd-size chroma auto-selection limitation documented in #24154; that separate limitation is outside this backport. From 7832132d32054b9f9b04d479e0a52262ad07461b Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:47:47 +0200 Subject: [PATCH 1/5] avfilter/vf_scale_cuda: don't try to execute disabled filter passes (cherry picked from commit 50a9e34e1bc4bcf54331af39d4a53d8cffde3c22) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 240e40d43b..95ecde2774 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -157,6 +157,7 @@ typedef struct CUDAScaleContext { CUDAScaleFilter filters_uv[FILTER_NB]; CUDATex inter_tex; int use_filters; /* -1 for auto */ + int pass_x, pass_y; float param; } CUDAScaleContext; @@ -407,6 +408,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int s->frames_ctx = av_buffer_ref(inl->hw_frames_ctx); if (!s->frames_ctx) return AVERROR(ENOMEM); + + s->use_filters = 0; } else { s->passthrough = 0; @@ -451,9 +454,7 @@ static av_cold int cudascale_load_functions(AVFilterContext *ctx) if (s->use_filters) { /* Final pass is always vertical unless not vertically scaling */ - AVFilterLink *inlink = ctx->inputs[0]; - AVFilterLink *outlink = ctx->outputs[0]; - function_infix = inlink->h == outlink->h ? "Generic_h" : "Generic_v"; + function_infix = s->pass_y == FILTER_OUT ? "Generic_v" : "Generic_h"; s->interp_use_linear = 0; s->interp_as_integer = 0; } else { @@ -623,19 +624,19 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - int pass_x = -1, pass_y = -1; + s->pass_x = s->pass_y = -1; if (inlink->w != outlink->w && inlink->h != outlink->h) { /* Always perform the horizontal scaling pass first */ - pass_x = FILTER_TMP; - pass_y = FILTER_OUT; + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { - pass_x = FILTER_OUT; + s->pass_x = FILTER_OUT; } else if (inlink->h != outlink->h) { - pass_y = FILTER_OUT; + s->pass_y = FILTER_OUT; } - if (pass_x >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_x], + if (s->pass_x >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); if (ret < 0) goto fail; @@ -643,15 +644,15 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->w, in_sub_x); const int dst_size = AV_CEIL_RSHIFT(outlink->w, out_sub_x); const double virtual_size = (double) outlink->w / (1 << out_sub_x); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_x], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_x], src_size, dst_size, virtual_size); if (ret < 0) goto fail; } } - if (pass_y >= 0) { - ret = cudascale_filter_init(ctx, &s->filters[pass_y], + if (s->pass_y >= 0) { + ret = cudascale_filter_init(ctx, &s->filters[s->pass_y], inlink->h, outlink->h, 0.0); if (ret < 0) goto fail; @@ -659,7 +660,7 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) const int src_size = AV_CEIL_RSHIFT(inlink->h, in_sub_y); const int dst_size = AV_CEIL_RSHIFT(outlink->h, out_sub_y); const double virtual_size = (double) outlink->h / (1 << out_sub_y); - ret = cudascale_filter_init(ctx, &s->filters_uv[pass_y], + ret = cudascale_filter_init(ctx, &s->filters_uv[s->pass_y], src_size, dst_size, virtual_size); if (ret < 0) goto fail; @@ -915,7 +916,7 @@ static int cudascale_scale(AVFilterContext *ctx, AVFrame *out, AVFrame *in) goto fail; const CUDATex *src = &in_tex; - if (s->use_filters) { + if (s->pass_x == FILTER_TMP) { /* Handle first pass separately */ s->inter_tex.color_range = in->color_range; ret = scalecuda_resize(ctx, FILTER_TMP, &s->inter_tex, src); -- 2.52.0 From 979b335e33376a10e315ada4f1dd3f267c39e8b0 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 22:56:20 +0200 Subject: [PATCH 2/5] avfilter/vf_scale_cuda: skip allocating intermediate texture if not needed (cherry picked from commit 3d7ea1dac4007b4897ad00991268bbdb51816743) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 95ecde2774..2c22ec1aa4 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -667,9 +667,11 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) } } - ret = inter_buf_init(ctx, outlink->w, inlink->h); - if (ret < 0) - goto fail; + if (s->pass_x == FILTER_TMP) { + ret = inter_buf_init(ctx, outlink->w, inlink->h); + if (ret < 0) + goto fail; + } ret = 0; -- 2.52.0 From 1da95554740faa52799f3b7ff045da8b0d6e5270 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:13:16 +0200 Subject: [PATCH 3/5] avfilter/vf_scale_cuda: extract pass determination into its own function (cherry picked from commit dda8b8253cb020d0a09942e6507505ab608128b3) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 2c22ec1aa4..fc3a91d299 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -369,6 +369,27 @@ static av_cold void set_format_info(AVFilterContext *ctx, enum AVPixelFormat in_ } } +static av_cold void cudascale_setup_passes(AVFilterContext *ctx) +{ + CUDAScaleContext *s = ctx->priv; + AVFilterLink *inlink = ctx->inputs[0]; + AVFilterLink *outlink = ctx->outputs[0]; + + s->pass_x = s->pass_y = -1; + if (!s->use_filters) + return; + + if (inlink->w != outlink->w && inlink->h != outlink->h) { + /* Always perform the horizontal scaling pass first */ + s->pass_x = FILTER_TMP; + s->pass_y = FILTER_OUT; + } else if (inlink->w != outlink->w) { + s->pass_x = FILTER_OUT; + } else if (inlink->h != outlink->h) { + s->pass_y = FILTER_OUT; + } +} + static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int in_height, int out_width, int out_height) { @@ -433,6 +454,8 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (!outl->hw_frames_ctx) return AVERROR(ENOMEM); + cudascale_setup_passes(ctx); + return 0; } @@ -624,17 +647,6 @@ static av_cold int cudascale_setup_filters(AVFilterContext *ctx) if (ret < 0) return ret; - s->pass_x = s->pass_y = -1; - if (inlink->w != outlink->w && inlink->h != outlink->h) { - /* Always perform the horizontal scaling pass first */ - s->pass_x = FILTER_TMP; - s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { - s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { - s->pass_y = FILTER_OUT; - } - if (s->pass_x >= 0) { ret = cudascale_filter_init(ctx, &s->filters[s->pass_x], inlink->w, outlink->w, 0.0); -- 2.52.0 From 38a8f88ca846ca8c98cf9481d31875331b84bbf4 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:26:14 +0200 Subject: [PATCH 4/5] avfilter/vf_scale_cuda: always have valid passes setup (cherry picked from commit 4566eee58816d9a927b460f73c773d18e6060274) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index fc3a91d299..7b001c9f5b 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -385,7 +385,7 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) s->pass_y = FILTER_OUT; } else if (inlink->w != outlink->w) { s->pass_x = FILTER_OUT; - } else if (inlink->h != outlink->h) { + } else { s->pass_y = FILTER_OUT; } } -- 2.52.0 From 46d8f462eeb87ee1f704d8c44a0ee24fca471ad1 Mon Sep 17 00:00:00 2001 From: Timo Rothenpieler <[email protected]> Date: Fri, 14 Aug 2026 23:52:00 +0200 Subject: [PATCH 5/5] avfilter/vf_scale_cuda: consider chroma subsampling for scaling decisions (cherry picked from commit a621ef4291148739e6a301ac5e7aee839c5d0933) Assisted-by: Fairy --- libavfilter/vf_scale_cuda.c | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_scale_cuda.c b/libavfilter/vf_scale_cuda.c index 7b001c9f5b..36c53288be 100644 --- a/libavfilter/vf_scale_cuda.c +++ b/libavfilter/vf_scale_cuda.c @@ -379,11 +379,16 @@ static av_cold void cudascale_setup_passes(AVFilterContext *ctx) if (!s->use_filters) return; - if (inlink->w != outlink->w && inlink->h != outlink->h) { + const int scale_x = inlink->w != outlink->w || + s->in_desc->log2_chroma_w != s->out_desc->log2_chroma_w; + const int scale_y = inlink->h != outlink->h || + s->in_desc->log2_chroma_h != s->out_desc->log2_chroma_h; + + if (scale_x && scale_y) { /* Always perform the horizontal scaling pass first */ s->pass_x = FILTER_TMP; s->pass_y = FILTER_OUT; - } else if (inlink->w != outlink->w) { + } else if (scale_x) { s->pass_x = FILTER_OUT; } else { s->pass_y = FILTER_OUT; @@ -444,7 +449,11 @@ static av_cold int init_processing_chain(AVFilterContext *ctx, int in_width, int if (s->interp_algo == INTERP_ALGO_NEAREST) { s->use_filters = 0; - } else if (s->use_filters < 0 && (out_width < in_width || out_height < in_height)) + } else if (s->use_filters < 0 && ( + out_width < in_width || out_height < in_height || + AV_CEIL_RSHIFT(out_width, s->out_desc->log2_chroma_w) < AV_CEIL_RSHIFT(in_width, s->in_desc->log2_chroma_w) || + AV_CEIL_RSHIFT(out_height, s->out_desc->log2_chroma_h) < AV_CEIL_RSHIFT(in_height, s->in_desc->log2_chroma_h) + )) s->use_filters = 1; /* downscaling; needed for anti-aliasing */ else if (s->use_filters < 0) s->use_filters = 0; -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
