PR #25740 opened by Niklas Haas (haasn) URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25740 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25740.patch
The only notable change here is that, apart from data getting a bit denser in general, the linear ops are now able to store their data inline as well. The 16-byte limitation had always been annoying, and I sometimes had to work around it in the past. This solves that completely. >From a0333005f87b92ad609a2b87d7e9ec1be0157012 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 17:17:17 +0100 Subject: [PATCH 01/24] swscale/filters: allocate padding past end of weights/offsets Useful to avoid a memcpy in the SIMD path. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/filters.c | 6 ++++-- libswscale/filters.h | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libswscale/filters.c b/libswscale/filters.c index fb54d2bbd8b..eac13afa961 100644 --- a/libswscale/filters.c +++ b/libswscale/filters.c @@ -271,13 +271,15 @@ int ff_sws_filter_generate(void *log, const SwsFilterParams *params, filter->name, filter->filter_size, radius); filter->num_weights = (size_t) params->dst_size * filter->filter_size; - filter->weights = av_refstruct_allocz(filter->num_weights * sizeof(*filter->weights)); + const size_t weights_size = filter->num_weights * sizeof(*filter->weights); + filter->weights = av_refstruct_allocz(weights_size + SWS_FILTER_PADDING); if (!filter->weights) { av_refstruct_unref(&filter); return AVERROR(ENOMEM); } - filter->offsets = av_refstruct_allocz(params->dst_size * sizeof(*filter->offsets)); + const size_t offsets_size = (size_t) params->dst_size * sizeof(*filter->offsets); + filter->offsets = av_refstruct_allocz(offsets_size + SWS_FILTER_PADDING); if (!filter->offsets) { av_refstruct_unref(&filter); return AVERROR(ENOMEM); diff --git a/libswscale/filters.h b/libswscale/filters.h index a10e6f39640..8f66a378022 100644 --- a/libswscale/filters.h +++ b/libswscale/filters.h @@ -39,6 +39,7 @@ enum { */ SWS_FILTER_SCALE = (1 << 14), SWS_FILTER_SIZE_MAX = 256, + SWS_FILTER_PADDING = 256, /* extra zero bytes past `weights`/`offsets` */ }; /* Parameters for filter generation. */ -- 2.54.0 >From 2f5093e7dd9b6365ab273ddd3f2a7a35dd7584bc Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 17:18:06 +0100 Subject: [PATCH 02/24] Revert "swscale/uops: pad the horizontal filter weights to whole blocks" This reverts commit 90ba80f81113d4f8e91c68c0f5207a9f6962e906. No longer needed after the previous commit, which already allocates extra padding. The condition has been lowered to a static_assert() instead. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/uops_tmpl.c | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index 56e28749b12..baa5ff8ec26 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -314,25 +314,12 @@ DECL_SETUP(setup_filter_h, params, out) return AVERROR(ENOTSUP); SwsFilterWeights *filter = params->uop->data.kernel; + static_assert(SWS_FILTER_PADDING >= SWS_BLOCK_SIZE * sizeof(*filter->weights), + "filter weights countain sufficient padding for block over-read"); + + out->priv.ptr = av_refstruct_ref(filter->weights); out->priv.i32[2] = filter->filter_size; - - /* The horizontal filter uop reads weights for a full SWS_BLOCK_SIZE - * outputs per block (see read_planar_fh), so the weights array must be - * padded up to the block-aligned output count. Pad the tail with zero - * weights, which contribute nothing to the accumulated sums. */ - const size_t padded_w = FFALIGN(filter->dst_size, SWS_BLOCK_SIZE); - if (padded_w == filter->dst_size) { - out->priv.ptr = av_refstruct_ref(filter->weights); - out->free = ff_op_priv_unref; - return 0; - } - - int *weights = av_calloc(padded_w * filter->filter_size, sizeof(*weights)); - if (!weights) - return AVERROR(ENOMEM); - memcpy(weights, filter->weights, filter->num_weights * sizeof(*weights)); - out->priv.ptr = weights; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; return 0; } -- 2.54.0 >From b15f8bbb1d2c625fe0e30d15fcbf9a0873943743 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 17:32:49 +0100 Subject: [PATCH 03/24] swscale/ops_chain: add ff_refstruct_calloc() helper Useful for building op private data, which tends to be needed for large structures like filter kernels. Avoids the theoretical risk of overflow from such allocations. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/ops_chain.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index e3a25e3af54..2d4e01bdb65 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -143,6 +143,13 @@ static inline void ff_op_priv_unref(SwsOpPriv *priv) av_refstruct_unref(&priv->ptr); } +static inline void *ff_refstruct_calloc(size_t n, size_t size) +{ + if (n && size > SIZE_MAX / n) + return NULL; + return av_refstruct_allocz(n * size); +} + struct SwsUOpTable { unsigned cpu_flags; /* required CPU flags for this table */ int block_size; /* fixed block size of this table */ -- 2.54.0 >From 5b6dc68a5c389cf301e111970aa5f8a0d3be2c2f Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 17:21:39 +0100 Subject: [PATCH 04/24] swscale/ops: allocate external kernel data as refstructs Having a common allocation scheme for external kernel data will allow us to get rid of SwsOpPriv. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 4 ++-- libswscale/uops_tmpl.c | 17 ++++++++++------- libswscale/x86/ops.c | 25 ++++++++++++++----------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 0c4fb7d7682..36a65b7fa39 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -74,7 +74,7 @@ static int aarch64_setup_linear(const SwsAArch64OpImplParams *p, */ const int num_vregs = linear_num_vregs(p); av_assert0(num_vregs <= 4); - void *coeffs = av_malloc(num_vregs * 16); + void *coeffs = av_refstruct_allocz(num_vregs * 16); if (!coeffs) return AVERROR(ENOMEM); @@ -99,7 +99,7 @@ static int aarch64_setup_linear(const SwsAArch64OpImplParams *p, } res->priv.ptr = coeffs; - res->free = ff_op_priv_free; + res->free = ff_op_priv_unref; return 0; } diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index baa5ff8ec26..c681ce88a9d 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -255,7 +255,7 @@ DECL_SETUP(setup_filter_v, params, out) ">8 byte pointers not supported"); /* Pre-convert weights to float */ - float *weights = av_calloc(filter->num_weights, sizeof(float)); + float *weights = ff_refstruct_calloc(filter->num_weights, sizeof(float)); if (!weights) return AVERROR(ENOMEM); @@ -264,7 +264,7 @@ DECL_SETUP(setup_filter_v, params, out) out->priv.ptr = weights; out->priv.i32[2] = filter->filter_size; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; return 0; } @@ -733,11 +733,11 @@ DECL_SETUP(setup_dither, params, out) const int stride = FFMAX(size, SWS_BLOCK_SIZE); const int height = ff_sws_dither_height(dither); - pixel_t *matrix = av_malloc(sizeof(pixel_t) * height * stride); + pixel_t *matrix = ff_refstruct_calloc(sizeof(pixel_t) * stride, height); if (!matrix) return AVERROR(ENOMEM); out->priv.ptr = matrix; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; /* Pad to multiple of block size. We don't need extra padding for the * height because ff_sws_dither_height() already includes any padding @@ -805,9 +805,12 @@ DECL_SETUP(setup_linear, params, out) c.k[i] = uop->data.mat4x5[i][4].px; } - out->priv.ptr = av_memdup(&c, sizeof(c)); - out->free = ff_op_priv_free; - return out->priv.ptr ? 0 : AVERROR(ENOMEM); + out->priv.ptr = av_refstruct_allocz(sizeof(c)); + if (!out->priv.ptr) + return AVERROR(ENOMEM); + memcpy(out->priv.ptr, &c, sizeof(c)); + out->free = ff_op_priv_unref; + return 0; } /** diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 9cc5ba7a0aa..5f605c28221 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -50,7 +50,7 @@ static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) ">8 byte pointers not supported"); /* Pre-convert weights to float */ - float *weights = av_calloc(filter->num_weights, sizeof(float)); + float *weights = ff_refstruct_calloc(filter->num_weights, sizeof(float)); if (!weights) return AVERROR(ENOMEM); @@ -59,7 +59,7 @@ static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) out->priv.ptr = weights; out->priv.uptr[1] = filter->filter_size; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; return 0; } @@ -100,7 +100,7 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) } weights; const int sizeof_weight = hscale_sizeof_weight(uop); - weights.ptr = av_calloc(line_size, sizeof_weight * aligned_size); + weights.ptr = ff_refstruct_calloc(line_size, sizeof_weight * aligned_size); if (!weights.ptr) return AVERROR(ENOMEM); @@ -152,7 +152,7 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) out->priv.ptr = weights.ptr; out->priv.uptr[1] = aligned_size; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; for (int i = 0; i < 4; i++) { if (uop->mask & SWS_COMP(i)) @@ -205,7 +205,7 @@ static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out) float *f32; } weights; - weights.ptr = av_calloc(line_size, aligned_size * sizeof_weights); + weights.ptr = ff_refstruct_calloc(line_size, aligned_size * sizeof_weights); if (!weights.ptr) return AVERROR(ENOMEM); @@ -239,7 +239,7 @@ static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out) out->priv.ptr = weights.ptr; out->priv.uptr[1] = aligned_size * sizeof_weights; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; for (int i = 0; i < 4; i++) { if (uop->mask & SWS_COMP(i)) @@ -298,16 +298,19 @@ static int setup_linear(const SwsImplParams *params, SwsImplResult *out) { const SwsUOp *uop = params->uop; if (uop->type == SWS_PIXEL_F32) { - out->priv.ptr = av_memdup(uop->data.mat4x5, sizeof(uop->data.mat4x5)); - out->free = ff_op_priv_free; - return out->priv.ptr ? 0 : AVERROR(ENOMEM); + out->priv.ptr = av_refstruct_allocz(sizeof(uop->data.mat4x5)); + if (!out->priv.ptr) + return AVERROR(ENOMEM); + memcpy(out->priv.ptr, uop->data.mat4x5, sizeof(uop->data.mat4x5)); + out->free = ff_op_priv_unref; + return 0; } - uint8_t *mat = av_malloc(4 * 5 * 16); /* one lane per component */ + uint8_t *mat = av_refstruct_allocz(4 * 5 * 16); /* one lane per component */ if (!mat) return AVERROR(ENOMEM); out->priv.ptr = mat; - out->free = ff_op_priv_free; + out->free = ff_op_priv_unref; for (int i = 0; i < 4; i++) { for (int j = 0; j < 5; j++) { -- 2.54.0 >From 884d2ea0a1012fd47b3f5f2806208da4877dab6c Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 16:59:46 +0100 Subject: [PATCH 05/24] swscale/ops_chain: track references instead of arbitrary free callbacks We don't really need this level of generality. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 10 ++++------ libswscale/ops_chain.c | 19 ++++++++----------- libswscale/ops_chain.h | 8 +++++--- libswscale/uops_tmpl.c | 18 ++++++------------ libswscale/x86/ops.c | 18 ++++++------------ 5 files changed, 29 insertions(+), 44 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 36a65b7fa39..1ed102ba0f6 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -98,17 +98,14 @@ static int aarch64_setup_linear(const SwsAArch64OpImplParams *p, } } - res->priv.ptr = coeffs; - res->free = ff_op_priv_unref; - + res->priv.ptr = res->ref = coeffs; return 0; } /*********************************************************************/ static int aarch64_setup_dither(const SwsUOp *uop, SwsImplResult *res) { - res->priv.ptr = av_refstruct_ref(uop->data.ptr); - res->free = ff_op_priv_unref; + res->priv.ptr = res->ref = av_refstruct_ref(uop->data.ptr); return 0; } @@ -188,7 +185,8 @@ static int compile_uops_aarch64(SwsContext *ctx, const SwsUOpList *uops, ret = aarch64_setup(&uops->ops[i], ¶ms, &res); if (ret < 0) goto error; - ret = ff_sws_op_chain_append(chain, func, res.free, &res.priv); + ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv); + av_refstruct_unref(&res.ref); if (ret < 0) goto error; } diff --git a/libswscale/ops_chain.c b/libswscale/ops_chain.c index 42f26035a7b..07b6e60ef83 100644 --- a/libswscale/ops_chain.c +++ b/libswscale/ops_chain.c @@ -20,6 +20,7 @@ #include "libavutil/avassert.h" #include "libavutil/mem.h" +#include "libavutil/refstruct.h" #include "ops_chain.h" @@ -34,16 +35,14 @@ void ff_sws_op_chain_free_cb(void *ptr) return; SwsOpChain *chain = ptr; - for (int i = 0; i < chain->num_impl + 1; i++) { - if (chain->free[i]) - chain->free[i](&chain->impl[i].priv); - } + for (int i = 0; i < chain->num_impl; i++) + av_refstruct_unref(&chain->ref[i]); av_free(chain); } int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, - void (*free)(SwsOpPriv *), const SwsOpPriv *priv) + const void *ref, const SwsOpPriv *priv) { const int idx = chain->num_impl; if (idx == SWS_MAX_OPS) @@ -52,7 +51,7 @@ int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, av_assert1(func); chain->impl[idx].cont = func; chain->impl[idx + 1].priv = *priv; - chain->free[idx + 1] = free; + chain->ref[idx] = ref ? av_refstruct_ref_c(ref) : NULL; chain->num_impl++; return 0; } @@ -110,12 +109,10 @@ int ff_sws_uop_lookup(SwsContext *ctx, const SwsUOpTable *const tables[], } ret = ff_sws_op_chain_append(chain, res.func ? res.func : match->func, - res.free, &res.priv); - if (ret < 0) { - if (res.free) - res.free(&res.priv); + res.ref, &res.priv); + av_refstruct_unref(&res.ref); + if (ret < 0) return ret; - } for (int i = 0; i < 4; i++) { chain->over_read[i] = FFMAX(chain->over_read[i], res.over_read[i]); diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index 2d4e01bdb65..5f82607aa05 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -84,11 +84,13 @@ static_assert(offsetof(SwsOpImpl, priv) == 16, "SwsOpImpl layout mismatch"); typedef struct SwsOpChain { #define SWS_MAX_OPS 16 SwsOpImpl impl[SWS_MAX_OPS + 1]; /* reserve extra space for the entrypoint */ - void (*free[SWS_MAX_OPS + 1])(SwsOpPriv *); int num_impl; int cpu_flags; /* set of all used CPU flags */ int over_read[4]; /* chain over-reads input by this many bytes */ int over_write[4]; /* chain over-writes output by this many bytes */ + + /* Extra references owned by each kernel */ + const void *ref[SWS_MAX_OPS]; } SwsOpChain; SwsOpChain *ff_sws_op_chain_alloc(void); @@ -100,7 +102,7 @@ static inline void ff_sws_op_chain_free(SwsOpChain *chain) /* Returns 0 on success, or a negative error code. */ int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, - void (*free)(SwsOpPriv *), const SwsOpPriv *priv); + const void *ref, const SwsOpPriv *priv); typedef struct SwsImplParams { const SwsUOpTable *table; @@ -111,9 +113,9 @@ typedef struct SwsImplParams { typedef struct SwsImplResult { SwsFuncPtr func; /* overrides `SwsUOpEntry.func` if non-NULL */ SwsOpPriv priv; /* private data for this implementation instance */ - void (*free)(SwsOpPriv *priv); /* free function for `priv` */ int over_read[4]; /* implementation over-reads input by this many bytes */ int over_write[4]; /* implementation over-writes output by this many bytes */ + void *ref; /* external data backing `priv`, ownership passes to SwsOpChain */ } SwsImplResult; typedef struct SwsUOpEntry { diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index c681ce88a9d..d191032e14d 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -262,9 +262,8 @@ DECL_SETUP(setup_filter_v, params, out) for (int i = 0; i < filter->num_weights; i++) weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE; - out->priv.ptr = weights; + out->priv.ptr = out->ref = weights; out->priv.i32[2] = filter->filter_size; - out->free = ff_op_priv_unref; return 0; } @@ -317,9 +316,8 @@ DECL_SETUP(setup_filter_h, params, out) static_assert(SWS_FILTER_PADDING >= SWS_BLOCK_SIZE * sizeof(*filter->weights), "filter weights countain sufficient padding for block over-read"); - out->priv.ptr = av_refstruct_ref(filter->weights); + out->priv.ptr = out->ref = av_refstruct_ref(filter->weights); out->priv.i32[2] = filter->filter_size; - out->free = ff_op_priv_unref; return 0; } @@ -726,8 +724,7 @@ DECL_SETUP(setup_dither, params, out) const int size = 1 << dither->size_log2; if (size >= SWS_BLOCK_SIZE) { /* No extra padding needed */ - out->priv.ptr = av_refstruct_ref(uop->data.ptr); - out->free = ff_op_priv_unref; + out->priv.ptr = out->ref = av_refstruct_ref(uop->data.ptr); return 0; } @@ -736,8 +733,7 @@ DECL_SETUP(setup_dither, params, out) pixel_t *matrix = ff_refstruct_calloc(sizeof(pixel_t) * stride, height); if (!matrix) return AVERROR(ENOMEM); - out->priv.ptr = matrix; - out->free = ff_op_priv_unref; + out->priv.ptr = out->ref = matrix; /* Pad to multiple of block size. We don't need extra padding for the * height because ff_sws_dither_height() already includes any padding @@ -805,11 +801,10 @@ DECL_SETUP(setup_linear, params, out) c.k[i] = uop->data.mat4x5[i][4].px; } - out->priv.ptr = av_refstruct_allocz(sizeof(c)); + out->priv.ptr = out->ref = av_refstruct_allocz(sizeof(c)); if (!out->priv.ptr) return AVERROR(ENOMEM); memcpy(out->priv.ptr, &c, sizeof(c)); - out->free = ff_op_priv_unref; return 0; } @@ -861,8 +856,7 @@ SWS_FOR_STRUCT(PX, LINEAR, DECL_ENTRY, .setup = fn(setup_linear) ) DECL_SETUP(setup_lut3d, params, out) { const SwsLut3D *lut = params->uop->data.lut3d; - out->priv.ptr = (void *) av_refstruct_ref_c(lut); - out->free = ff_op_priv_unref; + out->priv.ptr = out->ref = (void *) av_refstruct_ref_c(lut); return 0; } diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 5f605c28221..0ff39f58efb 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -57,9 +57,8 @@ static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) for (int i = 0; i < filter->num_weights; i++) weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE; - out->priv.ptr = weights; + out->priv.ptr = out->ref = weights; out->priv.uptr[1] = filter->filter_size; - out->free = ff_op_priv_unref; return 0; } @@ -150,9 +149,8 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) } } - out->priv.ptr = weights.ptr; + out->priv.ptr = out->ref = weights.ptr; out->priv.uptr[1] = aligned_size; - out->free = ff_op_priv_unref; for (int i = 0; i < 4; i++) { if (uop->mask & SWS_COMP(i)) @@ -237,9 +235,8 @@ static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out) } } - out->priv.ptr = weights.ptr; + out->priv.ptr = out->ref = weights.ptr; out->priv.uptr[1] = aligned_size * sizeof_weights; - out->free = ff_op_priv_unref; for (int i = 0; i < 4; i++) { if (uop->mask & SWS_COMP(i)) @@ -272,8 +269,7 @@ static int setup_clear(const SwsImplParams *params, SwsImplResult *out) static int setup_dither(const SwsImplParams *params, SwsImplResult *out) { - out->priv.ptr = av_refstruct_ref(params->uop->data.ptr); - out->free = ff_op_priv_unref; + out->priv.ptr = out->ref = av_refstruct_ref(params->uop->data.ptr); return 0; } @@ -298,19 +294,17 @@ static int setup_linear(const SwsImplParams *params, SwsImplResult *out) { const SwsUOp *uop = params->uop; if (uop->type == SWS_PIXEL_F32) { - out->priv.ptr = av_refstruct_allocz(sizeof(uop->data.mat4x5)); + out->priv.ptr = out->ref = av_refstruct_allocz(sizeof(uop->data.mat4x5)); if (!out->priv.ptr) return AVERROR(ENOMEM); memcpy(out->priv.ptr, uop->data.mat4x5, sizeof(uop->data.mat4x5)); - out->free = ff_op_priv_unref; return 0; } uint8_t *mat = av_refstruct_allocz(4 * 5 * 16); /* one lane per component */ if (!mat) return AVERROR(ENOMEM); - out->priv.ptr = mat; - out->free = ff_op_priv_unref; + out->priv.ptr = out->ref = mat; for (int i = 0; i < 4; i++) { for (int j = 0; j < 5; j++) { -- 2.54.0 >From 287a076bc0a83f57b72413227ea5d8d5008e9e51 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 19:39:58 +0100 Subject: [PATCH 06/24] swscale/ops_chain: drop unused helpers Signed-off-by: Niklas Haas <[email protected]> --- libswscale/ops_chain.h | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index 5f82607aa05..9f8fcfd0cd4 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -135,16 +135,6 @@ typedef struct SwsUOpEntry { int ff_sws_setup_scalar(const SwsImplParams *params, SwsImplResult *out); int ff_sws_setup_vec4(const SwsImplParams *params, SwsImplResult *out); -static inline void ff_op_priv_free(SwsOpPriv *priv) -{ - av_freep(&priv->ptr); -} - -static inline void ff_op_priv_unref(SwsOpPriv *priv) -{ - av_refstruct_unref(&priv->ptr); -} - static inline void *ff_refstruct_calloc(size_t n, size_t size) { if (n && size > SIZE_MAX / n) -- 2.54.0 >From 1348a243c3af989ea4a5c470038b11e38ddfe2d5 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Sun, 1 Nov 2026 18:31:17 +0100 Subject: [PATCH 07/24] swscale/ops_chain: drop explicit alignment from SwsOpPriv No kernel needs its private data to be 16-byte aligned. This drops the alignment down to the implicit pointer alignment (8 bytes on most platforms). SwsOpImpl goes from 32 bytes to 24 bytes. This does require dropping the alignment assertions inside a guard for the specific architectures that we have SIMD for, since on 32-bit platforms, the C code now packs this struct more tightly (at offset=4). Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 1 + libswscale/aarch64/ops_impl.h | 4 ++-- libswscale/ops_chain.h | 13 ++++++++----- libswscale/x86/ops_include.asm | 2 +- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 1ed102ba0f6..08f220fbbe2 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -39,6 +39,7 @@ static_assert(offsetof_exec_in_bump == offsetof(SwsOpExec, in_bump), "SwsOpExe static_assert(offsetof_exec_out_bump == offsetof(SwsOpExec, out_bump), "SwsOpExec layout mismatch"); static_assert(offsetof_impl_cont == offsetof(SwsOpImpl, cont), "SwsOpImpl layout mismatch"); static_assert(offsetof_impl_priv == offsetof(SwsOpImpl, priv), "SwsOpImpl layout mismatch"); +static_assert(sizeof_impl == sizeof(SwsOpImpl), "SwsOpImpl layout mismatch"); /*********************************************************************/ /* Forward-declare exported functions. */ diff --git a/libswscale/aarch64/ops_impl.h b/libswscale/aarch64/ops_impl.h index ff3181f3961..2553805967c 100644 --- a/libswscale/aarch64/ops_impl.h +++ b/libswscale/aarch64/ops_impl.h @@ -92,7 +92,7 @@ static inline int linear_num_vregs(const SwsAArch64OpImplParams *params) #define offsetof_exec_in_bump 128 #define offsetof_exec_out_bump 160 #define offsetof_impl_cont 0 -#define offsetof_impl_priv 16 -#define sizeof_impl 32 +#define offsetof_impl_priv 8 +#define sizeof_impl 24 #endif /* SWSCALE_AARCH64_OPS_IMPL_H */ diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index 9f8fcfd0cd4..544421687d8 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -21,6 +21,7 @@ #ifndef SWSCALE_OPS_CHAIN_H #define SWSCALE_OPS_CHAIN_H +#include "libavutil/avconfig.h" #include "libavutil/cpu.h" #include "libavutil/mem.h" @@ -43,7 +44,7 @@ typedef struct SwsUOpTable SwsUOpTable; * Private data for each kernel. */ typedef union SwsOpPriv { - DECLARE_ALIGNED_16(char, data)[16]; + char data[16]; /* Common types */ void *ptr; @@ -69,12 +70,14 @@ static_assert(sizeof(SwsOpPriv) == 16, "SwsOpPriv size mismatch"); */ typedef void (*SwsFuncPtr)(void); typedef struct SwsOpImpl { - SwsFuncPtr cont; /* [offset = 0] Continuation for this operation. */ - SwsOpPriv priv; /* [offset = 16] Private data for this operation. */ + SwsFuncPtr cont; /* Continuation for this operation. */ + SwsOpPriv priv; /* Private data for this operation. */ } SwsOpImpl; -static_assert(sizeof(SwsOpImpl) == 32, "SwsOpImpl layout mismatch"); -static_assert(offsetof(SwsOpImpl, priv) == 16, "SwsOpImpl layout mismatch"); +#if ARCH_X86_64 || ARCH_AARCH64 +static_assert(sizeof(SwsOpImpl) == 24, "SwsOpImpl layout mismatch"); +static_assert(offsetof(SwsOpImpl, priv) == 8, "SwsOpImpl layout mismatch"); +#endif /** * Compiled "chain" of operations, which can be dispatched efficiently. diff --git a/libswscale/x86/ops_include.asm b/libswscale/x86/ops_include.asm index a71255ea872..b8f41ca07b2 100644 --- a/libswscale/x86/ops_include.asm +++ b/libswscale/x86/ops_include.asm @@ -134,7 +134,7 @@ struc SwsOpExec endstruc struc SwsOpImpl - .cont resb 16 + .cont resq 1 .priv resb 16 .next resb 0 endstruc -- 2.54.0 >From 253842379b07a86209e44916e671bc7dde3fe4aa Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 20:45:50 +0100 Subject: [PATCH 08/24] swscale/ops_chain: use abstract packed data buffer Instead of forcing a single layout (SwsOpPriv), this will allow each op to define its own data layout; freeing us from the 16-byte constraint for op private data. Currently, we still hard-code the assumption that the data layout exactly matches SwsOpPriv, but this will change in the following commits. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 2 +- libswscale/ops_chain.c | 41 +++++++++++++++++++++++++++------------ libswscale/ops_chain.h | 21 ++++++++++++++------ libswscale/uops_backend.c | 2 +- 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 08f220fbbe2..7fb46e76b91 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -186,7 +186,7 @@ static int compile_uops_aarch64(SwsContext *ctx, const SwsUOpList *uops, ret = aarch64_setup(&uops->ops[i], ¶ms, &res); if (ret < 0) goto error; - ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv); + ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv, sizeof(res.priv)); av_refstruct_unref(&res.ref); if (ret < 0) goto error; diff --git a/libswscale/ops_chain.c b/libswscale/ops_chain.c index 07b6e60ef83..8be53aafbe0 100644 --- a/libswscale/ops_chain.c +++ b/libswscale/ops_chain.c @@ -26,7 +26,13 @@ SwsOpChain *ff_sws_op_chain_alloc(void) { - return av_mallocz(sizeof(SwsOpChain)); + SwsOpChain *chain = av_mallocz(sizeof(SwsOpChain)); + if (!chain) + return NULL; + + /* Reserve extra space for the entrypoint */ + chain->tail = sizeof(SwsOpImpl); + return chain; } void ff_sws_op_chain_free_cb(void *ptr) @@ -35,24 +41,35 @@ void ff_sws_op_chain_free_cb(void *ptr) return; SwsOpChain *chain = ptr; - for (int i = 0; i < chain->num_impl; i++) + for (int i = 0; i < chain->num_ref; i++) av_refstruct_unref(&chain->ref[i]); av_free(chain); } -int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, - const void *ref, const SwsOpPriv *priv) +int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, const void *ref, + const void *data, size_t data_size) { - const int idx = chain->num_impl; - if (idx == SWS_MAX_OPS) + /* Ensures each SwsOpImpl is stored aligned, so that kernels can deref it */ + const size_t op_size = FFALIGN(offsetof(SwsOpImpl, priv) + data_size, + alignof(SwsOpImpl)); + + if (chain->tail + op_size > sizeof(chain->impl)) + return AVERROR(EINVAL); + if (ref && chain->num_ref >= FF_ARRAY_ELEMS(chain->ref)) return AVERROR(EINVAL); - av_assert1(func); - chain->impl[idx].cont = func; - chain->impl[idx + 1].priv = *priv; - chain->ref[idx] = ref ? av_refstruct_ref_c(ref) : NULL; - chain->num_impl++; + /* Link the new function pointer into the previous op's struct */ + void *last = &chain->impl[chain->last]; + void *priv = &chain->impl[chain->tail + offsetof(SwsOpImpl, priv)]; + memcpy(last, &func, sizeof(func)); + memcpy(priv, data, data_size); + + chain->last = chain->tail; + chain->tail += op_size; + if (ref) + chain->ref[chain->num_ref++] = av_refstruct_ref_c(ref); + return 0; } @@ -109,7 +126,7 @@ int ff_sws_uop_lookup(SwsContext *ctx, const SwsUOpTable *const tables[], } ret = ff_sws_op_chain_append(chain, res.func ? res.func : match->func, - res.ref, &res.priv); + res.ref, &res.priv, sizeof(res.priv)); av_refstruct_unref(&res.ref); if (ret < 0) return ret; diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index 544421687d8..c2089cb1de1 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -77,6 +77,7 @@ typedef struct SwsOpImpl { #if ARCH_X86_64 || ARCH_AARCH64 static_assert(sizeof(SwsOpImpl) == 24, "SwsOpImpl layout mismatch"); static_assert(offsetof(SwsOpImpl, priv) == 8, "SwsOpImpl layout mismatch"); +static_assert(alignof(SwsOpImpl) == sizeof(void *), "SwsOpImpl alignment mismatch"); #endif /** @@ -85,15 +86,23 @@ static_assert(offsetof(SwsOpImpl, priv) == 8, "SwsOpImpl layout mismatch"); * private data for each operation. */ typedef struct SwsOpChain { -#define SWS_MAX_OPS 16 - SwsOpImpl impl[SWS_MAX_OPS + 1]; /* reserve extra space for the entrypoint */ - int num_impl; + /* Generic data buffer. This contains a packed represetation of + * continuation addresses and kernel private data. This is placed at + * the front of the struct so SIMD kernels can access it directly. */ + DECLARE_ALIGNED_16(uint8_t, impl)[1024]; + + /* Metadata for the chain */ int cpu_flags; /* set of all used CPU flags */ int over_read[4]; /* chain over-reads input by this many bytes */ int over_write[4]; /* chain over-writes output by this many bytes */ /* Extra references owned by each kernel */ - const void *ref[SWS_MAX_OPS]; + const void *ref[16]; + int num_ref; + + /* Offsets into data[], used internally during chain construction */ + int last; /* offset of last op registered op */ + int tail; /* offset of next free byte */ } SwsOpChain; SwsOpChain *ff_sws_op_chain_alloc(void); @@ -104,8 +113,8 @@ static inline void ff_sws_op_chain_free(SwsOpChain *chain) } /* Returns 0 on success, or a negative error code. */ -int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, - const void *ref, const SwsOpPriv *priv); +int ff_sws_op_chain_append(SwsOpChain *chain, SwsFuncPtr func, const void *ref, + const void *data, size_t data_size); typedef struct SwsImplParams { const SwsUOpTable *table; diff --git a/libswscale/uops_backend.c b/libswscale/uops_backend.c index 1fd07fdeeec..d92c989058b 100644 --- a/libswscale/uops_backend.c +++ b/libswscale/uops_backend.c @@ -111,7 +111,7 @@ static void process_c(const SwsOpExec *exec, const void *priv, int bx_end, int y_end) { const SwsOpChain *chain = priv; - const SwsOpImpl *impl = chain->impl; + const SwsOpImpl *impl = (const SwsOpImpl *) chain->impl; block_t x, y, z, w; /* allocate enough space for any intermediate */ SwsOpIter iterdata; -- 2.54.0 >From b03bc308461b902b22e666c795a4997ae6644b5f Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 20:46:07 +0100 Subject: [PATCH 09/24] swscale/ops_chain: directly pack data into SwsOpPriv The next series of commits will gradually switch over the backends to packing their kernel private data directly into SwsOpPriv.data[], rather than using the typed arrays. This will allow us to remove the typed arrays entirely and then unbound the per-op size. It's worth pointing out that, as of this commit, the packed size is still ignored in favor of storing the entire SwsOpPriv struct in the chain data. This is to preserve the alignment for the current code, which directly dereference the SwsOpImpl pointers. This can be relaxed after the op kernels stop hard-coding this dereference. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/ops_chain.c | 16 ++++++++-------- libswscale/ops_chain.h | 14 +++++++++++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/libswscale/ops_chain.c b/libswscale/ops_chain.c index 8be53aafbe0..96645259c4e 100644 --- a/libswscale/ops_chain.c +++ b/libswscale/ops_chain.c @@ -145,10 +145,10 @@ int ff_sws_setup_scalar(const SwsImplParams *params, SwsImplResult *out) const SwsUOp *uop = params->uop; const SwsPixel scalar = uop->data.scalar; switch (uop->type) { - case SWS_PIXEL_U8: out->priv.u8[0] = scalar.u8; break; - case SWS_PIXEL_U16: out->priv.u16[0] = scalar.u16; break; - case SWS_PIXEL_U32: out->priv.u32[0] = scalar.u32; break; - case SWS_PIXEL_F32: out->priv.f32[0] = scalar.f32; break; + case SWS_PIXEL_U8: SWS_OP_PRIV_PACK(out, scalar.u8); break; + case SWS_PIXEL_U16: SWS_OP_PRIV_PACK(out, scalar.u16); break; + case SWS_PIXEL_U32: SWS_OP_PRIV_PACK(out, scalar.u32); break; + case SWS_PIXEL_F32: SWS_OP_PRIV_PACK(out, scalar.f32); break; default: return AVERROR(EINVAL); } @@ -161,10 +161,10 @@ int ff_sws_setup_vec4(const SwsImplParams *params, SwsImplResult *out) for (int i = 0; i < 4; i++) { const SwsPixel vi = uop->data.vec4[i]; switch (uop->type) { - case SWS_PIXEL_U8: out->priv.u8[i] = vi.u8; break; - case SWS_PIXEL_U16: out->priv.u16[i] = vi.u16; break; - case SWS_PIXEL_U32: out->priv.u32[i] = vi.u32; break; - case SWS_PIXEL_F32: out->priv.f32[i] = vi.f32; break; + case SWS_PIXEL_U8: SWS_OP_PRIV_PACK(out, vi.u8); break; + case SWS_PIXEL_U16: SWS_OP_PRIV_PACK(out, vi.u16); break; + case SWS_PIXEL_U32: SWS_OP_PRIV_PACK(out, vi.u32); break; + case SWS_PIXEL_F32: SWS_OP_PRIV_PACK(out, vi.f32); break; default: return AVERROR(EINVAL); } } diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index c2089cb1de1..6ea369a31fc 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -21,6 +21,7 @@ #ifndef SWSCALE_OPS_CHAIN_H #define SWSCALE_OPS_CHAIN_H +#include "libavutil/avassert.h" #include "libavutil/avconfig.h" #include "libavutil/cpu.h" #include "libavutil/mem.h" @@ -124,10 +125,13 @@ typedef struct SwsImplParams { typedef struct SwsImplResult { SwsFuncPtr func; /* overrides `SwsUOpEntry.func` if non-NULL */ - SwsOpPriv priv; /* private data for this implementation instance */ int over_read[4]; /* implementation over-reads input by this many bytes */ int over_write[4]; /* implementation over-writes output by this many bytes */ void *ref; /* external data backing `priv`, ownership passes to SwsOpChain */ + + /* Private data for this kernel */ + SwsOpPriv priv; + size_t priv_size; } SwsImplResult; typedef struct SwsUOpEntry { @@ -154,6 +158,14 @@ static inline void *ff_refstruct_calloc(size_t n, size_t size) return av_refstruct_allocz(n * size); } +#define SWS_OP_PRIV_PACK(out, data) ff_op_priv_pack(out, &data, sizeof(data)) +static inline void ff_op_priv_pack(SwsImplResult *out, const void *data, size_t size) +{ + av_assert0(out->priv_size + size <= sizeof(out->priv)); + memcpy(&out->priv.data[out->priv_size], data, size); + out->priv_size += size; +} + struct SwsUOpTable { unsigned cpu_flags; /* required CPU flags for this table */ int block_size; /* fixed block size of this table */ -- 2.54.0 >From efb378afa4f518c3864c1f80ffc035483f2bfc97 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 20:47:51 +0100 Subject: [PATCH 10/24] swscale/uops_tmpl: pack pointer private data Co-Authored-By: Claude Opus 5.5 <[email protected]> --- libswscale/uops_tmpl.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index d191032e14d..552fe01f123 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -724,7 +724,8 @@ DECL_SETUP(setup_dither, params, out) const int size = 1 << dither->size_log2; if (size >= SWS_BLOCK_SIZE) { /* No extra padding needed */ - out->priv.ptr = out->ref = av_refstruct_ref(uop->data.ptr); + out->ref = av_refstruct_ref(uop->data.ptr); + SWS_OP_PRIV_PACK(out, out->ref); return 0; } @@ -733,7 +734,8 @@ DECL_SETUP(setup_dither, params, out) pixel_t *matrix = ff_refstruct_calloc(sizeof(pixel_t) * stride, height); if (!matrix) return AVERROR(ENOMEM); - out->priv.ptr = out->ref = matrix; + out->ref = matrix; + SWS_OP_PRIV_PACK(out, out->ref); /* Pad to multiple of block size. We don't need extra padding for the * height because ff_sws_dither_height() already includes any padding @@ -801,10 +803,11 @@ DECL_SETUP(setup_linear, params, out) c.k[i] = uop->data.mat4x5[i][4].px; } - out->priv.ptr = out->ref = av_refstruct_allocz(sizeof(c)); - if (!out->priv.ptr) + out->ref = av_refstruct_allocz(sizeof(c)); + if (!out->ref) return AVERROR(ENOMEM); - memcpy(out->priv.ptr, &c, sizeof(c)); + memcpy(out->ref, &c, sizeof(c)); + SWS_OP_PRIV_PACK(out, out->ref); return 0; } @@ -856,7 +859,8 @@ SWS_FOR_STRUCT(PX, LINEAR, DECL_ENTRY, .setup = fn(setup_linear) ) DECL_SETUP(setup_lut3d, params, out) { const SwsLut3D *lut = params->uop->data.lut3d; - out->priv.ptr = out->ref = (void *) av_refstruct_ref_c(lut); + out->ref = (void *) av_refstruct_ref_c(lut); + SWS_OP_PRIV_PACK(out, out->ref); return 0; } -- 2.54.0 >From 597d827ebc784e46dd97d0b15b0da60bce7737a4 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 20:48:44 +0100 Subject: [PATCH 11/24] swscale/uops_tmpl: pack the filter weights and size Instead of punning the size into SwsOpPriv.i32[2], declare a dedicated struct for the filter private data. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/uops_tmpl.c | 30 ++++++++++++++++++++---------- libswscale/uops_tmpl.h | 9 +++++++++ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index 552fe01f123..7d36b44d22e 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -251,8 +251,6 @@ DECL_SETUP(setup_filter_v, params, out) return AVERROR(ENOTSUP); const SwsFilterWeights *filter = params->uop->data.kernel; - static_assert(sizeof(out->priv.ptr) <= sizeof(int32_t[2]), - ">8 byte pointers not supported"); /* Pre-convert weights to float */ float *weights = ff_refstruct_calloc(filter->num_weights, sizeof(float)); @@ -262,8 +260,13 @@ DECL_SETUP(setup_filter_v, params, out) for (int i = 0; i < filter->num_weights; i++) weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE; - out->priv.ptr = out->ref = weights; - out->priv.i32[2] = filter->filter_size; + const SwsFilterPriv priv = { + .weights = weights, + .filter_size = filter->filter_size, + }; + + SWS_OP_PRIV_PACK(out, priv); + out->ref = priv.weights; return 0; } @@ -272,8 +275,9 @@ DECL_READ(read_planar_fv, const SwsCompMask mask, const SwsPixelType type) { av_assert2(type == SWS_PIXEL_F32); const SwsOpExec *exec = iter->exec; - const float *restrict weights = impl->priv.ptr; - const int filter_size = impl->priv.i32[2]; + const SwsFilterPriv *priv = (const SwsFilterPriv *) impl->priv.data; + const float *restrict weights = priv->weights; + const int filter_size = priv->filter_size; weights += filter_size * iter->y; block_t xs, ys, zs, ws; @@ -316,8 +320,13 @@ DECL_SETUP(setup_filter_h, params, out) static_assert(SWS_FILTER_PADDING >= SWS_BLOCK_SIZE * sizeof(*filter->weights), "filter weights countain sufficient padding for block over-read"); - out->priv.ptr = out->ref = av_refstruct_ref(filter->weights); - out->priv.i32[2] = filter->filter_size; + const SwsFilterPriv priv = { + .weights = av_refstruct_ref(filter->weights), + .filter_size = filter->filter_size, + }; + + SWS_OP_PRIV_PACK(out, priv); + out->ref = priv.weights; return 0; } @@ -326,8 +335,9 @@ DECL_READ(read_planar_fh, const SwsCompMask mask, const SwsPixelType type) { av_assert2(type == SWS_PIXEL_F32); const SwsOpExec *exec = iter->exec; - const int *restrict weights = impl->priv.ptr; - const int filter_size = impl->priv.i32[2]; + const SwsFilterPriv *priv = (const SwsFilterPriv *) impl->priv.data; + const int *restrict weights = priv->weights; + const int filter_size = priv->filter_size; const float scale = 1.0f / SWS_FILTER_SCALE; const int xpos = iter->x; weights += filter_size * iter->x; diff --git a/libswscale/uops_tmpl.h b/libswscale/uops_tmpl.h index c684ad36452..73815fd4130 100644 --- a/libswscale/uops_tmpl.h +++ b/libswscale/uops_tmpl.h @@ -63,6 +63,15 @@ typedef struct SwsOpIter { const SwsOpExec *exec; } SwsOpIter; +/* Private data for filtered reads */ +typedef struct SwsFilterPriv { + void *weights; + int filter_size; +} SwsFilterPriv; + +static_assert(alignof(SwsFilterPriv) <= alignof(SwsOpPriv), + "SwsFilterPriv alignment exceeds SwsOpPriv"); + /* UBSan checks inside the loop body defeat the vectorizer, so do not ask * for it in those builds. Clang warns for every loop otherwise. */ #if HAVE_UBSAN -- 2.54.0 >From 0d1dbb090e1e4557cb3ab05d2fec87f90ca94d4b Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 20:49:12 +0100 Subject: [PATCH 12/24] swscale/x86/ops: pack the filter private data Signed-off-by: Niklas Haas <[email protected]> --- libswscale/x86/ops.c | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 0ff39f58efb..42ab0f67e21 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -43,11 +43,17 @@ static int setup_rw_packed(const SwsImplParams *params, SwsImplResult *out) return 0; } +/* Private data for the filter kernels; layout hard-coded in ops_float.asm */ +typedef struct FilterPriv { + void *weights; + size_t filter_size; +} FilterPriv; + +static_assert(offsetof(FilterPriv, filter_size) == 8, "FilterPriv layout mismatch"); + static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) { const SwsFilterWeights *filter = params->uop->data.kernel; - static_assert(sizeof(out->priv.ptr) <= sizeof(int32_t[2]), - ">8 byte pointers not supported"); /* Pre-convert weights to float */ float *weights = ff_refstruct_calloc(filter->num_weights, sizeof(float)); @@ -57,8 +63,13 @@ static int setup_filter_v(const SwsImplParams *params, SwsImplResult *out) for (int i = 0; i < filter->num_weights; i++) weights[i] = (float) filter->weights[i] / SWS_FILTER_SCALE; - out->priv.ptr = out->ref = weights; - out->priv.uptr[1] = filter->filter_size; + const FilterPriv priv = { + .weights = weights, + .filter_size = filter->filter_size, + }; + + SWS_OP_PRIV_PACK(out, priv); + out->ref = priv.weights; return 0; } @@ -149,8 +160,13 @@ static int setup_filter_h(const SwsImplParams *params, SwsImplResult *out) } } - out->priv.ptr = out->ref = weights.ptr; - out->priv.uptr[1] = aligned_size; + const FilterPriv priv = { + .weights = weights.ptr, + .filter_size = aligned_size, + }; + + SWS_OP_PRIV_PACK(out, priv); + out->ref = priv.weights; for (int i = 0; i < 4; i++) { if (uop->mask & SWS_COMP(i)) @@ -235,8 +251,13 @@ static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out) } } - out->priv.ptr = out->ref = weights.ptr; - out->priv.uptr[1] = aligned_size * sizeof_weights; + const FilterPriv priv = { + .weights = weights.ptr, + .filter_size = aligned_size * sizeof_weights, + }; + + SWS_OP_PRIV_PACK(out, priv); + out->ref = priv.weights; for (int i = 0; i < 4; i++) { if (uop->mask & SWS_COMP(i)) -- 2.54.0 >From bbe3d39a2f5998dcfa77e88d4c3bf2df48afd436 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 21:07:44 +0100 Subject: [PATCH 13/24] swscale/x86/ops: pack SCALE constant as dword The SSE4 path loads this as `movd`. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/x86/ops.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 42ab0f67e21..599baf7981b 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -270,7 +270,7 @@ static int setup_scale(const SwsImplParams *params, SwsImplResult *out) { const SwsUOp *uop = params->uop; switch (uop->type) { - case SWS_PIXEL_U8: out->priv.u16[0] = uop->data.scalar.u8; break; /* for pmullw */ + case SWS_PIXEL_U8: out->priv.u32[0] = uop->data.scalar.u8; break; /* for movd */ case SWS_PIXEL_U16: out->priv.u16[0] = uop->data.scalar.u16; break; case SWS_PIXEL_U32: out->priv.u32[0] = uop->data.scalar.u32; break; case SWS_PIXEL_F32: out->priv.f32[0] = uop->data.scalar.f32; break; -- 2.54.0 >From 235fdb18e643303451d739b26a9b84baea7b79d1 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 21:13:35 +0100 Subject: [PATCH 14/24] swscale/x86/ops: pack the remaining private data Signed-off-by: Niklas Haas <[email protected]> --- libswscale/x86/ops.c | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 599baf7981b..4e77c4188f5 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -268,12 +268,16 @@ static int setup_filter_h_4x4(const SwsImplParams *params, SwsImplResult *out) static int setup_scale(const SwsImplParams *params, SwsImplResult *out) { - const SwsUOp *uop = params->uop; - switch (uop->type) { - case SWS_PIXEL_U8: out->priv.u32[0] = uop->data.scalar.u8; break; /* for movd */ - case SWS_PIXEL_U16: out->priv.u16[0] = uop->data.scalar.u16; break; - case SWS_PIXEL_U32: out->priv.u32[0] = uop->data.scalar.u32; break; - case SWS_PIXEL_F32: out->priv.f32[0] = uop->data.scalar.f32; break; + const SwsPixel scalar = params->uop->data.scalar; + switch (params->uop->type) { + case SWS_PIXEL_U8: { + const uint32_t u32 = scalar.u8; /* for movd */ + SWS_OP_PRIV_PACK(out, u32); + break; + } + case SWS_PIXEL_U16: SWS_OP_PRIV_PACK(out, scalar.u16); break; + case SWS_PIXEL_U32: SWS_OP_PRIV_PACK(out, scalar.u32); break; + case SWS_PIXEL_F32: SWS_OP_PRIV_PACK(out, scalar.f32); break; default: return AVERROR(EINVAL); } @@ -284,13 +288,14 @@ static int setup_clear(const SwsImplParams *params, SwsImplResult *out) { const SwsUOp *uop = params->uop; for (int i = 0; i < 4; i++) - out->priv.u32[i] = uop->data.vec4[i].u32; + SWS_OP_PRIV_PACK(out, uop->data.vec4[i].u32); return 0; } static int setup_dither(const SwsImplParams *params, SwsImplResult *out) { - out->priv.ptr = out->ref = av_refstruct_ref(params->uop->data.ptr); + out->ref = av_refstruct_ref(params->uop->data.ptr); + SWS_OP_PRIV_PACK(out, out->ref); return 0; } @@ -315,17 +320,19 @@ static int setup_linear(const SwsImplParams *params, SwsImplResult *out) { const SwsUOp *uop = params->uop; if (uop->type == SWS_PIXEL_F32) { - out->priv.ptr = out->ref = av_refstruct_allocz(sizeof(uop->data.mat4x5)); - if (!out->priv.ptr) + out->ref = av_refstruct_allocz(sizeof(uop->data.mat4x5)); + if (!out->ref) return AVERROR(ENOMEM); - memcpy(out->priv.ptr, uop->data.mat4x5, sizeof(uop->data.mat4x5)); + memcpy(out->ref, uop->data.mat4x5, sizeof(uop->data.mat4x5)); + SWS_OP_PRIV_PACK(out, out->ref); return 0; } uint8_t *mat = av_refstruct_allocz(4 * 5 * 16); /* one lane per component */ if (!mat) return AVERROR(ENOMEM); - out->priv.ptr = out->ref = mat; + out->ref = mat; + SWS_OP_PRIV_PACK(out, out->ref); for (int i = 0; i < 4; i++) { for (int j = 0; j < 5; j++) { -- 2.54.0 >From 01e7c44799171b218492d52de3d0506810466d02 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 20:50:08 +0100 Subject: [PATCH 15/24] swscale/aarch64/ops: pack kernel private data The clear/min/max kernels always load a full 128-bit vector, so pad the packed constants to 16 bytes. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 41 ++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 7fb46e76b91..6d374f9569a 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -99,14 +99,16 @@ static int aarch64_setup_linear(const SwsAArch64OpImplParams *p, } } - res->priv.ptr = res->ref = coeffs; + SWS_OP_PRIV_PACK(res, coeffs); + res->ref = coeffs; return 0; } /*********************************************************************/ static int aarch64_setup_dither(const SwsUOp *uop, SwsImplResult *res) { - res->priv.ptr = res->ref = av_refstruct_ref(uop->data.ptr); + res->ref = av_refstruct_ref(uop->data.ptr); + SWS_OP_PRIV_PACK(res, res->ref); return 0; } @@ -115,28 +117,35 @@ static int aarch64_setup(const SwsUOp *uop, const SwsAArch64OpImplParams *p, SwsImplResult *out) { switch (uop->uop) { - case SWS_UOP_READ_BIT: + case SWS_UOP_READ_BIT: { /* Negative shift values to perform right shift using ushl. */ - out->priv = (SwsOpPriv) { - .u8 = { - -7, -6, -5, -4, -3, -2, -1, 0, - -7, -6, -5, -4, -3, -2, -1, 0, - } + static const int8_t shift[16] = { + -7, -6, -5, -4, -3, -2, -1, 0, + -7, -6, -5, -4, -3, -2, -1, 0, }; + SWS_OP_PRIV_PACK(out, shift); break; - case SWS_UOP_WRITE_BIT: + } + case SWS_UOP_WRITE_BIT: { /* Shift values for ushl. */ - out->priv = (SwsOpPriv) { - .u8 = { - 7, 6, 5, 4, 3, 2, 1, 0, - 7, 6, 5, 4, 3, 2, 1, 0, - } + static const uint8_t shift[16] = { + 7, 6, 5, 4, 3, 2, 1, 0, + 7, 6, 5, 4, 3, 2, 1, 0, }; + SWS_OP_PRIV_PACK(out, shift); break; + } case SWS_UOP_CLEAR: case SWS_UOP_MIN: - case SWS_UOP_MAX: - return ff_sws_setup_vec4(&(const SwsImplParams) { .uop = uop }, out); + case SWS_UOP_MAX: { + int ret = ff_sws_setup_vec4(&(const SwsImplParams) { .uop = uop }, out); + if (ret < 0) + return ret; + /* Pad to the full vector register loaded by the kernel */ + static const uint8_t zero[16]; + ff_op_priv_pack(out, zero, sizeof(zero) - out->priv_size); + return 0; + } case SWS_UOP_SCALE: return ff_sws_setup_scalar(&(const SwsImplParams) { .uop = uop }, out); case SWS_UOP_LINEAR: -- 2.54.0 >From 168f4042cd778dc8e40a52e0232982c36c004585 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 21:54:56 +0100 Subject: [PATCH 16/24] swscale/uops_tmpl: parametrize CONTINUE() by size of priv data So that CONTINUE() can, in a following commit, start skipping over the data that each kernel consumes. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/uops_backend.c | 2 +- libswscale/uops_tmpl.c | 53 ++++++++++++++++++++------------------- libswscale/uops_tmpl.h | 5 ++-- 3 files changed, 31 insertions(+), 29 deletions(-) diff --git a/libswscale/uops_backend.c b/libswscale/uops_backend.c index d92c989058b..a499515eca5 100644 --- a/libswscale/uops_backend.c +++ b/libswscale/uops_backend.c @@ -125,7 +125,7 @@ static void process_c(const SwsOpExec *exec, const void *priv, for (iter->y = y_start; iter->y < y_end; iter->y++) { for (int block = bx_start; block < bx_end; block++) { iter->x = block * SWS_BLOCK_SIZE; - CONTINUE(&x, &y, &z, &w); + CONTINUE(0, &x, &y, &z, &w); } const int y_bump = exec->in_bump_y ? exec->in_bump_y[iter->y] : 0; diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index 7d36b44d22e..3f05daeb1c5 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -83,7 +83,7 @@ DECL_READ(read_planar, const SwsCompMask mask) if (Z) iter->in[2] += SIZEOF_BLOCK; if (W) iter->in[3] += SIZEOF_BLOCK; - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_READ(read_packed, const SwsCompMask mask) @@ -99,7 +99,7 @@ DECL_READ(read_packed, const SwsCompMask mask) } iter->in[0] += SIZEOF_BLOCK * elems; - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_WRITE(write_planar, const SwsCompMask mask) @@ -153,7 +153,7 @@ DECL_READ(read_bit, const SwsCompMask mask) } iter->in[0] += SIZEOF_BLOCK >> 3; - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_READ(read_nibble, const SwsCompMask mask) @@ -168,7 +168,7 @@ DECL_READ(read_nibble, const SwsCompMask mask) } iter->in[0] += SIZEOF_BLOCK >> 1; - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_READ(read_palette, const SwsCompMask mask) @@ -186,7 +186,7 @@ DECL_READ(read_palette, const SwsCompMask mask) } iter->in[0] += SIZEOF_BLOCK; - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_WRITE(write_bit, const SwsCompMask mask) @@ -308,7 +308,7 @@ DECL_READ(read_planar_fv, const SwsCompMask mask, const SwsPixelType type) if (Z) iter->in[2] += SIZEOF_BLOCK; if (W) iter->in[3] += SIZEOF_BLOCK; - CONTINUE(&xs, &ys, &zs, &ws); + CONTINUE(sizeof(*priv), &xs, &ys, &zs, &ws); } DECL_SETUP(setup_filter_h, params, out) @@ -367,7 +367,7 @@ DECL_READ(read_planar_fh, const SwsCompMask mask, const SwsPixelType type) weights += filter_size; } - CONTINUE(&xs, &ys, &zs, &ws); + CONTINUE(sizeof(*priv), &xs, &ys, &zs, &ws); } SWS_FOR(PX, READ_PLANAR_FV, DECL_IMPL_READ, read_planar_fv) @@ -393,7 +393,8 @@ DECL_FUNC(permute, const SwsCompMask mask, int num_moves, /* The unneeded registers may still alias the used ones, so point them * back at the stack to avoid collisions */ block_t xx, yy, zz, ww; - CONTINUE(X ? ptr[1] : xx.px, + CONTINUE(0, + X ? ptr[1] : xx.px, Y ? ptr[2] : yy.px, Z ? ptr[3] : zz.px, W ? ptr[4] : ww.px); @@ -421,7 +422,7 @@ DECL_FUNC(copy, const SwsCompMask mask, int num_moves, memcpy(z, &data[3].px, block_size); memcpy(w, &data[4].px, block_size); - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } SWS_FOR(PX, PERMUTE, DECL_IMPL, permute) @@ -446,7 +447,7 @@ SWS_FOR_STRUCT(PX, COPY, DECL_ENTRY) if (W) ww.dst[i] = w[i]; \ } \ \ - CONTINUE(&xx, &yy, &zz, &ww); \ + CONTINUE(0, &xx, &yy, &zz, &ww); \ } \ \ SWS_FOR(PX, TO_##DST, DECL_IMPL, to_##dst) \ @@ -472,7 +473,7 @@ DECL_FUNC(lshift, const SwsCompMask mask, const uint8_t amount) if (W) w[i] <<= amount; } - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_FUNC(rshift, const SwsCompMask mask, const uint8_t amount) @@ -485,7 +486,7 @@ DECL_FUNC(rshift, const SwsCompMask mask, const uint8_t amount) if (W) w[i] >>= amount; } - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } #endif @@ -506,7 +507,7 @@ DECL_FUNC(swap_bytes, const SwsCompMask mask) if (W) w[i] = PIXEL_SWAP(w[i]); } - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } #endif /* PIXEL_SWAP */ @@ -524,7 +525,7 @@ DECL_FUNC(expand_bit, const SwsCompMask mask) if (W) w[i] = w[i] ? PIXEL_MAX : 0; } - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } #endif @@ -541,7 +542,7 @@ DECL_FUNC(expand_pair, const SwsCompMask mask) if (W) w16.u16[i] = w[i] << 8 | w[i]; } - CONTINUE(&x16, &y16, &z16, &w16); + CONTINUE(0, &x16, &y16, &z16, &w16); } DECL_FUNC(expand_quad, const SwsCompMask mask) @@ -556,7 +557,7 @@ DECL_FUNC(expand_quad, const SwsCompMask mask) if (W) w32.u32[i] = (uint32_t) w[i] << 24 | w[i] << 16 | w[i] << 8 | w[i]; } - CONTINUE(&x32, &y32, &z32, &w32); + CONTINUE(0, &x32, &y32, &z32, &w32); } #endif /* BIT_DEPTH == 8 */ @@ -595,7 +596,7 @@ DECL_FUNC(unpack, const SwsCompMask mask, if (W) w[i] = (val >> sw) & mw; } - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } DECL_FUNC(pack, const SwsCompMask mask, @@ -617,7 +618,7 @@ DECL_FUNC(pack, const SwsCompMask mask, x[i] = val; } - CONTINUE(x, y, z, w); + CONTINUE(0, x, y, z, w); } #endif /* !IS_FLOAT */ @@ -649,7 +650,7 @@ DECL_FUNC(clear, const SwsCompMask mask, const SwsCompMask one, if (W) w[i] = cw; } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(pixel_t[4]), x, y, z, w); } #endif @@ -672,7 +673,7 @@ DECL_FUNC(scale, const SwsCompMask mask) if (W) w[i] *= scale; } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(scale), x, y, z, w); } DECL_FUNC(add, const SwsCompMask mask) @@ -685,7 +686,7 @@ DECL_FUNC(add, const SwsCompMask mask) if (W) w[i] += impl->priv.px[3]; } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(pixel_t[4]), x, y, z, w); } DECL_FUNC(min, const SwsCompMask mask) @@ -698,7 +699,7 @@ DECL_FUNC(min, const SwsCompMask mask) if (W) w[i] = FFMIN(w[i], impl->priv.px[3]); } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(pixel_t[4]), x, y, z, w); } DECL_FUNC(max, const SwsCompMask mask) @@ -711,7 +712,7 @@ DECL_FUNC(max, const SwsCompMask mask) if (W) w[i] = FFMAX(w[i], impl->priv.px[3]); } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(pixel_t[4]), x, y, z, w); } SWS_FOR(PX, SCALE, DECL_IMPL, scale) @@ -786,7 +787,7 @@ DECL_FUNC(dither, const SwsCompMask mask, if (W) w[i] += row3[i]; } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(matrix), x, y, z, w); } SWS_FOR(PX, DITHER, DECL_IMPL, dither) @@ -856,7 +857,7 @@ DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zer if (W) LIN_ROW(3, w); } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(impl->priv.ptr), x, y, z, w); } SWS_FOR(PX, LINEAR, DECL_IMPL, linear) @@ -1022,7 +1023,7 @@ DECL_FUNC(lut3d, const SwsCompMask mask, const int dynamic) z[i] = c.z; } - CONTINUE(x, y, z, w); + CONTINUE(sizeof(lut3d), x, y, z, w); } #endif /* IS_FLOAT */ diff --git a/libswscale/uops_tmpl.h b/libswscale/uops_tmpl.h index 73815fd4130..c054972c23c 100644 --- a/libswscale/uops_tmpl.h +++ b/libswscale/uops_tmpl.h @@ -117,8 +117,9 @@ static_assert(alignof(SwsFilterPriv) <= alignof(SwsOpPriv), #define CALL(NAME, ...) fn(NAME)(iter, impl, x, y, z, w, __VA_ARGS__) -/* Helper macro to call into the next continuation with a given type */ -#define CONTINUE(...) \ +/* Helper macro to call into the next continuation with a given type; N gives + * the size of private data consumed by this op */ +#define CONTINUE(N, ...) \ ((void (*)(SwsOpIter *, const SwsOpImpl *, \ void *restrict x, void *restrict y, \ void *restrict z, void *restrict w)) impl->cont) \ -- 2.54.0 >From 10bc2ef62ef4159b6f3b25bcb175c51eeda76404 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 21:59:20 +0100 Subject: [PATCH 17/24] swscale/x86/ops: parametrize CONTINUE by size of priv data See previous commit. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/x86/ops_float.asm | 18 +++++++++--------- libswscale/x86/ops_include.asm | 7 ++++--- libswscale/x86/ops_int.asm | 8 +++++--- 3 files changed, 18 insertions(+), 15 deletions(-) diff --git a/libswscale/x86/ops_float.asm b/libswscale/x86/ops_float.asm index fe46cffe9da..a1f5845260f 100644 --- a/libswscale/x86/ops_float.asm +++ b/libswscale/x86/ops_float.asm @@ -125,7 +125,7 @@ IF W, add in3q, FILTER_SIZE IF Z, add in2q, FILTER_SIZE IF Y, add in1q, FILTER_SIZE IF X, add in0q, FILTER_SIZE - CONTINUE tmp0q + CONTINUE tmp0q, 2 * gprsize %undef weights %undef fltsize %endmacro @@ -285,7 +285,7 @@ IF X, mulps mx2, m12 IF Y, mulps my2, m12 IF Z, mulps mz2, m12 IF W, mulps mw2, m12 - CONTINUE tmp0q + CONTINUE tmp0q, 2 * gprsize %undef weights %undef fltsize %endmacro @@ -453,7 +453,7 @@ IF X, mulps mx2, m10 IF Y, mulps my2, m10 IF Z, mulps mz2, m10 IF W, mulps mw2, m10 - CONTINUE tmp0q + CONTINUE tmp0q, 2 * gprsize %undef offset0q %undef offset1q %undef offset2q @@ -477,7 +477,7 @@ IF X, mulps mx2, m8 IF Y, mulps my2, m8 IF Z, mulps mz2, m8 IF W, mulps mw2, m8 - CONTINUE tmp0q + CONTINUE tmp0q, BYTES %endmacro %macro ADD 0 @@ -494,7 +494,7 @@ IF X, addps mx2, m8 IF Y, addps my2, m9 IF Z, addps mz2, m10 IF W, addps mw2, m11 - CONTINUE tmp0q + CONTINUE tmp0q, 4 * BYTES %endmacro %macro MIN 0 @@ -511,7 +511,7 @@ IF X, minps mx2, m8 IF Y, minps my2, m9 IF Z, minps mz2, m10 IF W, minps mw2, m11 - CONTINUE tmp0q + CONTINUE tmp0q, 4 * BYTES %endmacro %macro MAX 0 @@ -528,7 +528,7 @@ IF X, maxps mx2, m8 IF Y, maxps my2, m9 IF Z, maxps mz2, m10 IF W, maxps mw2, m11 - CONTINUE tmp0q + CONTINUE tmp0q, 4 * BYTES %endmacro ;--------------------------------------------------------- @@ -598,7 +598,7 @@ IF X, mova mx2, m8 IF Y, mova my2, m9 IF Z, mova mz2, m10 IF W, mova mw2, m11 - CONTINUE tmp1q + CONTINUE tmp1q, gprsize %endmacro %macro LINEAR 2 @@ -670,7 +670,7 @@ IF Y, addps my2, m9 IF Z, addps mz2, m10 IF W, addps mw2, m11 %endif - CONTINUE tmp0q + CONTINUE tmp0q, gprsize %endmacro ;--------------------------------------------------------- diff --git a/libswscale/x86/ops_include.asm b/libswscale/x86/ops_include.asm index b8f41ca07b2..fe7995e5b4a 100644 --- a/libswscale/x86/ops_include.asm +++ b/libswscale/x86/ops_include.asm @@ -306,8 +306,9 @@ endstruc mov %1, [implq + SwsOpImpl.cont] %endmacro -; Tail call into the next operation kernel, given that kernel's address -%macro CONTINUE 1 ; reg +; Tail call into the next operation kernel, given that kernel's address and +; the size of the private data consumed by this kernel (optional) +%macro CONTINUE 1-2 0 ; reg, [priv_size] add implq, SwsOpImpl.next jmp %1 annotate_function_size @@ -316,7 +317,7 @@ endstruc ; Convenience macro to load and continue to the next kernel in one step %macro CONTINUE 0 LOAD_CONT tmp0q - CONTINUE tmp0q + CONTINUE tmp0q, 0 %endmacro ;--------------------------------------------------------- diff --git a/libswscale/x86/ops_int.asm b/libswscale/x86/ops_int.asm index c5d38b69ff3..229f1d89425 100644 --- a/libswscale/x86/ops_int.asm +++ b/libswscale/x86/ops_int.asm @@ -600,11 +600,12 @@ IF V2, mova %3, %2 %macro CLEAR 2 %assign ONE_MASK %1 %assign ZERO_MASK %2 + LOAD_CONT tmp0q IF1 X, clear 0, mx, mx2 IF1 Y, clear 1, my, my2 IF1 Z, clear 2, mz, mz2 IF1 W, clear 3, mw, mw2 - CONTINUE + CONTINUE tmp0q, 4 * 4 %endmacro ;--------------------------------------------------------- @@ -745,12 +746,13 @@ IF W, packuswb %4, m11 pxor m15, m15 scale8 mx, my, mz, mw, m12 IF1 V2, scale8 mx2, my2, mz2, mw2, m12 + CONTINUE tmp0q, 4 ; packed as dword %else vpbroadcastx m12, [implq + SwsOpImpl.priv] pmull4 x, mx, my, mz, mw, m12 IF1 V2, pmull4 x, mx2, my2, mz2, mw2, m12 + CONTINUE tmp0q, BYTES %endif - CONTINUE tmp0q %endmacro %macro ADD 0 @@ -877,7 +879,7 @@ IF W, mova IN3, m11 LOAD_CONT tmp1q linear_pass IF2 V2, linear_pass 2 - CONTINUE tmp1q + CONTINUE tmp1q, gprsize %endmacro ;--------------------------------------------------------- -- 2.54.0 >From 64bcb3dd86db960f406d6af51b2cf854ea5dc7c9 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 22:00:03 +0100 Subject: [PATCH 18/24] swscale/aarch64/ops: parametrize the impl increment by size of priv data So that the generated kernels can, in a following commit, skip over the data that each kernel consumes. aarch64_priv_size() must match what aarch64_setup() packs, which is now also asserted. The generated code is unchanged. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 1 + libswscale/aarch64/ops_impl.h | 24 ++++++++++++++++++++++++ libswscale/aarch64/ops_static.c | 10 ++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 6d374f9569a..3c035546279 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -195,6 +195,7 @@ static int compile_uops_aarch64(SwsContext *ctx, const SwsUOpList *uops, ret = aarch64_setup(&uops->ops[i], ¶ms, &res); if (ret < 0) goto error; + av_assert1(res.priv_size == aarch64_priv_size(¶ms)); ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv, sizeof(res.priv)); av_refstruct_unref(&res.ref); if (ret < 0) diff --git a/libswscale/aarch64/ops_impl.h b/libswscale/aarch64/ops_impl.h index 2553805967c..5622e2976a8 100644 --- a/libswscale/aarch64/ops_impl.h +++ b/libswscale/aarch64/ops_impl.h @@ -82,6 +82,30 @@ static inline int linear_num_vregs(const SwsAArch64OpImplParams *params) return (count + nelems - 1) / nelems; } +/** + * Number of bytes of private data consumed by each kernel; must match what + * aarch64_setup() packs. + */ +static inline int aarch64_priv_size(const SwsAArch64OpImplParams *params) +{ + switch (params->uop) { + case SWS_UOP_READ_BIT: + case SWS_UOP_WRITE_BIT: + case SWS_UOP_CLEAR: + case SWS_UOP_MIN: + case SWS_UOP_MAX: + return 16; /* one full vector register */ + case SWS_UOP_SCALE: + return ff_sws_pixel_type_size(params->type); + case SWS_UOP_LINEAR: + case SWS_UOP_LINEAR_FMA: + case SWS_UOP_DITHER: + return 8; /* pointer */ + default: + return 0; + } +} + /** * These values will be used by ops_asmgen to access fields inside of * SwsOpExec and SwsOpImpl. The sizes are checked in aarch64/ops.c when diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index 2a7933d6752..3a9d222ac49 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -130,6 +130,12 @@ static const SwsAArch64OpEntry ops_entries[] = { /*********************************************************************/ #define IMPL_PRIV(s) a64op_off(s->impl, offsetof_impl_priv) +/* Offset of the next kernel's SwsOpImpl, given this kernel's priv data size */ +static int impl_next(int priv_size) +{ + return sizeof_impl; +} + /** * Set node where the continuation address will be loaded and impl will * be incremented. This should be done right after impl->priv has been @@ -521,7 +527,7 @@ static void asmgen_process_cps(SwsAArch64Context *s, SwsCompMask mask) rasm_set_current_node(r, s->setup); RasmOp impl_cont = a64op_off(s->impl, offsetof_impl_cont); i_ldr(r, s->op0_func, impl_cont); CMT("SwsFuncPtr op0_func = impl->cont;"); - i_add(r, s->op1_impl, s->impl, IMM(sizeof_impl)); CMT("SwsOpImpl *op1_impl = impl + 1;"); + i_add(r, s->op1_impl, s->impl, IMM(impl_next(0))); CMT("SwsOpImpl *op1_impl = impl + 1;"); /* Reset impl and call first kernel. */ rasm_set_current_node(r, s->loop); @@ -635,7 +641,7 @@ static void asmgen_op_cps(SwsAArch64Context *s, const SwsAArch64OpEntry *entry) } else { /* Load continuation address and increment impl pointer. */ RasmNode *node = rasm_set_current_node(r, s->load_cont_node); - RasmOp impl_post = a64op_post(s->impl, sizeof_impl); + RasmOp impl_post = a64op_post(s->impl, impl_next(aarch64_priv_size(p))); i_ldr(r, s->cont, impl_post); CMT("SwsFuncPtr cont = (impl++)->cont;"); rasm_set_current_node(r, node); /* Common end for remaining CPS functions. */ -- 2.54.0 >From 47c711bb9d8196553586d300efa9a4ccacb6a079 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 22:18:57 +0100 Subject: [PATCH 19/24] swscale/ops: only skip over consumed data in SwsOpImpl This commit turns on the mechanism built up by the previous commits. This allows ops to have dynamically sized priv data. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 2 +- libswscale/aarch64/ops_static.c | 2 +- libswscale/ops_chain.c | 4 ++-- libswscale/uops_tmpl.h | 2 +- libswscale/x86/ops_common.asm | 2 +- libswscale/x86/ops_include.asm | 5 ++--- 6 files changed, 8 insertions(+), 9 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index 3c035546279..dcd0a3d79b6 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -196,7 +196,7 @@ static int compile_uops_aarch64(SwsContext *ctx, const SwsUOpList *uops, if (ret < 0) goto error; av_assert1(res.priv_size == aarch64_priv_size(¶ms)); - ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv, sizeof(res.priv)); + ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv, res.priv_size); av_refstruct_unref(&res.ref); if (ret < 0) goto error; diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index 3a9d222ac49..143db3f0c1f 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -133,7 +133,7 @@ static const SwsAArch64OpEntry ops_entries[] = { /* Offset of the next kernel's SwsOpImpl, given this kernel's priv data size */ static int impl_next(int priv_size) { - return sizeof_impl; + return offsetof_impl_priv + ((priv_size + 7) & ~7); } /** diff --git a/libswscale/ops_chain.c b/libswscale/ops_chain.c index 96645259c4e..02b11ee3af7 100644 --- a/libswscale/ops_chain.c +++ b/libswscale/ops_chain.c @@ -31,7 +31,7 @@ SwsOpChain *ff_sws_op_chain_alloc(void) return NULL; /* Reserve extra space for the entrypoint */ - chain->tail = sizeof(SwsOpImpl); + chain->tail = offsetof(SwsOpImpl, priv); return chain; } @@ -126,7 +126,7 @@ int ff_sws_uop_lookup(SwsContext *ctx, const SwsUOpTable *const tables[], } ret = ff_sws_op_chain_append(chain, res.func ? res.func : match->func, - res.ref, &res.priv, sizeof(res.priv)); + res.ref, &res.priv, res.priv_size); av_refstruct_unref(&res.ref); if (ret < 0) return ret; diff --git a/libswscale/uops_tmpl.h b/libswscale/uops_tmpl.h index c054972c23c..edf84fcb1d8 100644 --- a/libswscale/uops_tmpl.h +++ b/libswscale/uops_tmpl.h @@ -123,7 +123,7 @@ static_assert(alignof(SwsFilterPriv) <= alignof(SwsOpPriv), ((void (*)(SwsOpIter *, const SwsOpImpl *, \ void *restrict x, void *restrict y, \ void *restrict z, void *restrict w)) impl->cont) \ - (iter, &impl[1], __VA_ARGS__) + (iter, (const void *) &impl->priv.data[FFALIGN(N, alignof(SwsOpImpl))], __VA_ARGS__) /* Helper macros for common op setup code */ #define DECL_SETUP(NAME, PARAMS, OUT) \ diff --git a/libswscale/x86/ops_common.asm b/libswscale/x86/ops_common.asm index 9053147dcab..8a576db6c93 100644 --- a/libswscale/x86/ops_common.asm +++ b/libswscale/x86/ops_common.asm @@ -43,7 +43,7 @@ cglobal sws_process%1_x86, 6, 7 + 2 * %1, 16 mov [rsp + 20], tmp0d ; bx_end mov [rsp + 24], tmp1d ; y_end mov tmp0q, [implq + SwsOpImpl.cont] - add implq, SwsOpImpl.next + add implq, SwsOpImpl.priv mov [rsp + 0], tmp0q mov [rsp + 8], implq movsxdifnidn bxq, bxd diff --git a/libswscale/x86/ops_include.asm b/libswscale/x86/ops_include.asm index fe7995e5b4a..7bdd965f35c 100644 --- a/libswscale/x86/ops_include.asm +++ b/libswscale/x86/ops_include.asm @@ -135,8 +135,7 @@ endstruc struc SwsOpImpl .cont resq 1 - .priv resb 16 - .next resb 0 + .priv resb 0 endstruc %define SWS_COMP_NONE 0 @@ -309,7 +308,7 @@ endstruc ; Tail call into the next operation kernel, given that kernel's address and ; the size of the private data consumed by this kernel (optional) %macro CONTINUE 1-2 0 ; reg, [priv_size] - add implq, SwsOpImpl.next + add implq, SwsOpImpl.priv + ((%2 + gprsize - 1) & ~(gprsize - 1)) jmp %1 annotate_function_size %endmacro -- 2.54.0 >From 29da373731de3de1797b1faaf666fdbcb30dcd8c Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 22:20:39 +0100 Subject: [PATCH 20/24] swscale/ops_chain: replace SwsImplResult.priv by abstract buffer Now that all op kernels use ff_op_priv_pack(), we can use a generic buffer here. Pre-emptively size it at 128 bytes to allow storing linear kernel data here as well. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 2 +- libswscale/ops_chain.c | 2 +- libswscale/ops_chain.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index dcd0a3d79b6..ca469045109 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -196,7 +196,7 @@ static int compile_uops_aarch64(SwsContext *ctx, const SwsUOpList *uops, if (ret < 0) goto error; av_assert1(res.priv_size == aarch64_priv_size(¶ms)); - ret = ff_sws_op_chain_append(chain, func, res.ref, &res.priv, res.priv_size); + ret = ff_sws_op_chain_append(chain, func, res.ref, res.priv, res.priv_size); av_refstruct_unref(&res.ref); if (ret < 0) goto error; diff --git a/libswscale/ops_chain.c b/libswscale/ops_chain.c index 02b11ee3af7..cef50c0a949 100644 --- a/libswscale/ops_chain.c +++ b/libswscale/ops_chain.c @@ -126,7 +126,7 @@ int ff_sws_uop_lookup(SwsContext *ctx, const SwsUOpTable *const tables[], } ret = ff_sws_op_chain_append(chain, res.func ? res.func : match->func, - res.ref, &res.priv, res.priv_size); + res.ref, res.priv, res.priv_size); av_refstruct_unref(&res.ref); if (ret < 0) return ret; diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index 6ea369a31fc..5fe26e10c80 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -130,8 +130,8 @@ typedef struct SwsImplResult { void *ref; /* external data backing `priv`, ownership passes to SwsOpChain */ /* Private data for this kernel */ - SwsOpPriv priv; size_t priv_size; + uint8_t priv[128]; } SwsImplResult; typedef struct SwsUOpEntry { @@ -162,7 +162,7 @@ static inline void *ff_refstruct_calloc(size_t n, size_t size) static inline void ff_op_priv_pack(SwsImplResult *out, const void *data, size_t size) { av_assert0(out->priv_size + size <= sizeof(out->priv)); - memcpy(&out->priv.data[out->priv_size], data, size); + memcpy(&out->priv[out->priv_size], data, size); out->priv_size += size; } -- 2.54.0 >From e152c83320f2a69afd39cd3518e224cc93088124 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 21:44:40 +0100 Subject: [PATCH 21/24] swscale/ops_chain: make SwsOpImpl.priv generic SwsOpPriv is now only used by the generic C code to access op data set up by the generic pack helpers, so move it there. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 1 - libswscale/aarch64/ops_impl.h | 1 - libswscale/ops_chain.h | 28 ++--------------- libswscale/uops_tmpl.c | 57 +++++++++++++++++++++-------------- libswscale/uops_tmpl.h | 14 ++++++++- 5 files changed, 49 insertions(+), 52 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index ca469045109..ff223f982f3 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -39,7 +39,6 @@ static_assert(offsetof_exec_in_bump == offsetof(SwsOpExec, in_bump), "SwsOpExe static_assert(offsetof_exec_out_bump == offsetof(SwsOpExec, out_bump), "SwsOpExec layout mismatch"); static_assert(offsetof_impl_cont == offsetof(SwsOpImpl, cont), "SwsOpImpl layout mismatch"); static_assert(offsetof_impl_priv == offsetof(SwsOpImpl, priv), "SwsOpImpl layout mismatch"); -static_assert(sizeof_impl == sizeof(SwsOpImpl), "SwsOpImpl layout mismatch"); /*********************************************************************/ /* Forward-declare exported functions. */ diff --git a/libswscale/aarch64/ops_impl.h b/libswscale/aarch64/ops_impl.h index 5622e2976a8..b951e8b1c3f 100644 --- a/libswscale/aarch64/ops_impl.h +++ b/libswscale/aarch64/ops_impl.h @@ -117,6 +117,5 @@ static inline int aarch64_priv_size(const SwsAArch64OpImplParams *params) #define offsetof_exec_out_bump 160 #define offsetof_impl_cont 0 #define offsetof_impl_priv 8 -#define sizeof_impl 24 #endif /* SWSCALE_AARCH64_OPS_IMPL_H */ diff --git a/libswscale/ops_chain.h b/libswscale/ops_chain.h index 5fe26e10c80..95291c19935 100644 --- a/libswscale/ops_chain.h +++ b/libswscale/ops_chain.h @@ -41,29 +41,6 @@ typedef struct SwsUOpTable SwsUOpTable; -/** - * Private data for each kernel. - */ -typedef union SwsOpPriv { - char data[16]; - - /* Common types */ - void *ptr; - uint8_t u8[16]; - int8_t i8[16]; - uint16_t u16[8]; - int16_t i16[8]; - uint32_t u32[4]; - int32_t i32[4]; - float f32[4]; - uint64_t u64[2]; - int64_t i64[2]; - uintptr_t uptr[2]; - intptr_t iptr[2]; -} SwsOpPriv; - -static_assert(sizeof(SwsOpPriv) == 16, "SwsOpPriv size mismatch"); - /** * Per-kernel execution context. * @@ -71,12 +48,11 @@ static_assert(sizeof(SwsOpPriv) == 16, "SwsOpPriv size mismatch"); */ typedef void (*SwsFuncPtr)(void); typedef struct SwsOpImpl { - SwsFuncPtr cont; /* Continuation for this operation. */ - SwsOpPriv priv; /* Private data for this operation. */ + SwsFuncPtr cont; /* Continuation for this operation. */ + char priv[]; /* Private data for this operation. */ } SwsOpImpl; #if ARCH_X86_64 || ARCH_AARCH64 -static_assert(sizeof(SwsOpImpl) == 24, "SwsOpImpl layout mismatch"); static_assert(offsetof(SwsOpImpl, priv) == 8, "SwsOpImpl layout mismatch"); static_assert(alignof(SwsOpImpl) == sizeof(void *), "SwsOpImpl alignment mismatch"); #endif diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index 3f05daeb1c5..3f159fcc6d7 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -275,7 +275,7 @@ DECL_READ(read_planar_fv, const SwsCompMask mask, const SwsPixelType type) { av_assert2(type == SWS_PIXEL_F32); const SwsOpExec *exec = iter->exec; - const SwsFilterPriv *priv = (const SwsFilterPriv *) impl->priv.data; + const SwsFilterPriv *priv = SWS_OP_PRIV(impl); const float *restrict weights = priv->weights; const int filter_size = priv->filter_size; weights += filter_size * iter->y; @@ -335,7 +335,7 @@ DECL_READ(read_planar_fh, const SwsCompMask mask, const SwsPixelType type) { av_assert2(type == SWS_PIXEL_F32); const SwsOpExec *exec = iter->exec; - const SwsFilterPriv *priv = (const SwsFilterPriv *) impl->priv.data; + const SwsFilterPriv *priv = SWS_OP_PRIV(impl); const int *restrict weights = priv->weights; const int filter_size = priv->filter_size; const float scale = 1.0f / SWS_FILTER_SCALE; @@ -637,10 +637,11 @@ DECL_FUNC(clear, const SwsCompMask mask, const SwsCompMask one, { #define ONE(N) SWS_COMP_TEST(one, N) #define ZERO(N) SWS_COMP_TEST(zero, N) - const pixel_t cx = ONE(0) ? PIXEL_MAX : ZERO(0) ? 0 : impl->priv.px[0]; - const pixel_t cy = ONE(1) ? PIXEL_MAX : ZERO(1) ? 0 : impl->priv.px[1]; - const pixel_t cz = ONE(2) ? PIXEL_MAX : ZERO(2) ? 0 : impl->priv.px[2]; - const pixel_t cw = ONE(3) ? PIXEL_MAX : ZERO(3) ? 0 : impl->priv.px[3]; + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + const pixel_t cx = ONE(0) ? PIXEL_MAX : ZERO(0) ? 0 : priv->px[0]; + const pixel_t cy = ONE(1) ? PIXEL_MAX : ZERO(1) ? 0 : priv->px[1]; + const pixel_t cz = ONE(2) ? PIXEL_MAX : ZERO(2) ? 0 : priv->px[2]; + const pixel_t cw = ONE(3) ? PIXEL_MAX : ZERO(3) ? 0 : priv->px[3]; SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { @@ -663,7 +664,8 @@ SWS_FOR_STRUCT(PX, CLEAR, DECL_ENTRY, .setup = ff_sws_setup_vec4) DECL_FUNC(scale, const SwsCompMask mask) { - const pixel_t scale = impl->priv.px[0]; + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + const pixel_t scale = priv->px[0]; SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { @@ -678,12 +680,14 @@ DECL_FUNC(scale, const SwsCompMask mask) DECL_FUNC(add, const SwsCompMask mask) { + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { - if (X) x[i] += impl->priv.px[0]; - if (Y) y[i] += impl->priv.px[1]; - if (Z) z[i] += impl->priv.px[2]; - if (W) w[i] += impl->priv.px[3]; + if (X) x[i] += priv->px[0]; + if (Y) y[i] += priv->px[1]; + if (Z) z[i] += priv->px[2]; + if (W) w[i] += priv->px[3]; } CONTINUE(sizeof(pixel_t[4]), x, y, z, w); @@ -691,12 +695,14 @@ DECL_FUNC(add, const SwsCompMask mask) DECL_FUNC(min, const SwsCompMask mask) { + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { - if (X) x[i] = FFMIN(x[i], impl->priv.px[0]); - if (Y) y[i] = FFMIN(y[i], impl->priv.px[1]); - if (Z) z[i] = FFMIN(z[i], impl->priv.px[2]); - if (W) w[i] = FFMIN(w[i], impl->priv.px[3]); + if (X) x[i] = FFMIN(x[i], priv->px[0]); + if (Y) y[i] = FFMIN(y[i], priv->px[1]); + if (Z) z[i] = FFMIN(z[i], priv->px[2]); + if (W) w[i] = FFMIN(w[i], priv->px[3]); } CONTINUE(sizeof(pixel_t[4]), x, y, z, w); @@ -704,12 +710,14 @@ DECL_FUNC(min, const SwsCompMask mask) DECL_FUNC(max, const SwsCompMask mask) { + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { - if (X) x[i] = FFMAX(x[i], impl->priv.px[0]); - if (Y) y[i] = FFMAX(y[i], impl->priv.px[1]); - if (Z) z[i] = FFMAX(z[i], impl->priv.px[2]); - if (W) w[i] = FFMAX(w[i], impl->priv.px[3]); + if (X) x[i] = FFMAX(x[i], priv->px[0]); + if (Y) y[i] = FFMAX(y[i], priv->px[1]); + if (Z) z[i] = FFMAX(z[i], priv->px[2]); + if (W) w[i] = FFMAX(w[i], priv->px[3]); } CONTINUE(sizeof(pixel_t[4]), x, y, z, w); @@ -770,7 +778,8 @@ DECL_FUNC(dither, const SwsCompMask mask, const int size = 1 << size_log2; const int stride = FFMAX(size, SWS_BLOCK_SIZE); - const pixel_t *matrix = impl->priv.ptr; + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + const pixel_t *matrix = priv->ptr; matrix += (iter->y & (size - 1)) * stride; matrix += (iter->x & (size - 1)) & ~(SWS_BLOCK_SIZE - 1); @@ -830,7 +839,8 @@ DECL_SETUP(setup_linear, params, out) */ DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zero) { - const fn(LinCoeffs) c = *(const fn(LinCoeffs) *) impl->priv.ptr; + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + const fn(LinCoeffs) c = *(const fn(LinCoeffs) *) priv->ptr; SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { @@ -857,7 +867,7 @@ DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zer if (W) LIN_ROW(3, w); } - CONTINUE(sizeof(impl->priv.ptr), x, y, z, w); + CONTINUE(sizeof(priv->ptr), x, y, z, w); } SWS_FOR(PX, LINEAR, DECL_IMPL, linear) @@ -1009,7 +1019,8 @@ vec3_t fn(lut3d_dynamic)(const SwsLut3D *restrict lut3d, vec3_t rgb) DECL_FUNC(lut3d, const SwsCompMask mask, const int dynamic) { - const SwsLut3D *restrict lut3d = impl->priv.ptr; + const SwsOpPriv *priv = SWS_OP_PRIV(impl); + const SwsLut3D *restrict lut3d = priv->ptr; SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { diff --git a/libswscale/uops_tmpl.h b/libswscale/uops_tmpl.h index edf84fcb1d8..046645a14bf 100644 --- a/libswscale/uops_tmpl.h +++ b/libswscale/uops_tmpl.h @@ -63,6 +63,17 @@ typedef struct SwsOpIter { const SwsOpExec *exec; } SwsOpIter; +/* Private data for generic kernels. */ +typedef union SwsOpPriv { + uint8_t u8[4]; + uint16_t u16[4]; + uint32_t u32[4]; + float f32[4]; + void *ptr; +} SwsOpPriv; + +#define SWS_OP_PRIV(impl) ((const void *) (impl)->priv) + /* Private data for filtered reads */ typedef struct SwsFilterPriv { void *weights; @@ -123,7 +134,8 @@ static_assert(alignof(SwsFilterPriv) <= alignof(SwsOpPriv), ((void (*)(SwsOpIter *, const SwsOpImpl *, \ void *restrict x, void *restrict y, \ void *restrict z, void *restrict w)) impl->cont) \ - (iter, (const void *) &impl->priv.data[FFALIGN(N, alignof(SwsOpImpl))], __VA_ARGS__) + (iter, (const void *) &impl->priv[FFALIGN(N, alignof(SwsOpImpl))], \ + __VA_ARGS__) /* Helper macros for common op setup code */ #define DECL_SETUP(NAME, PARAMS, OUT) \ -- 2.54.0 >From b6f24ec0f9e57f5ac7ea120e952963a9f220c48f Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 22:29:34 +0100 Subject: [PATCH 22/24] swscale/uops_tmpl: store the linear coefficients in the op chain Avoids a separate allocation, and the pointer indirection in the kernel. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/uops_tmpl.c | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/libswscale/uops_tmpl.c b/libswscale/uops_tmpl.c index 3f159fcc6d7..bf30acc53c9 100644 --- a/libswscale/uops_tmpl.c +++ b/libswscale/uops_tmpl.c @@ -823,11 +823,7 @@ DECL_SETUP(setup_linear, params, out) c.k[i] = uop->data.mat4x5[i][4].px; } - out->ref = av_refstruct_allocz(sizeof(c)); - if (!out->ref) - return AVERROR(ENOMEM); - memcpy(out->ref, &c, sizeof(c)); - SWS_OP_PRIV_PACK(out, out->ref); + SWS_OP_PRIV_PACK(out, c); return 0; } @@ -839,8 +835,7 @@ DECL_SETUP(setup_linear, params, out) */ DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zero) { - const SwsOpPriv *priv = SWS_OP_PRIV(impl); - const fn(LinCoeffs) c = *(const fn(LinCoeffs) *) priv->ptr; + const fn(LinCoeffs) c = *(const fn(LinCoeffs) *) SWS_OP_PRIV(impl); SWS_LOOP for (int i = 0; i < SWS_BLOCK_SIZE; i++) { @@ -867,7 +862,7 @@ DECL_FUNC(linear, const SwsCompMask mask, const uint32_t one, const uint32_t zer if (W) LIN_ROW(3, w); } - CONTINUE(sizeof(priv->ptr), x, y, z, w); + CONTINUE(sizeof(c), x, y, z, w); } SWS_FOR(PX, LINEAR, DECL_IMPL, linear) -- 2.54.0 >From e5da32d8186f7551d61157f1128aaef1d0e4e02f Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 22:30:39 +0100 Subject: [PATCH 23/24] swscale/x86/ops: store the f32 linear coefficients in the op chain The integer kernels keep using a separate allocation, since their pre-splatted coefficients (4x5 full lanes) are too large. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/x86/ops.c | 9 ++++----- libswscale/x86/ops_float.asm | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/libswscale/x86/ops.c b/libswscale/x86/ops.c index 4e77c4188f5..99ffa657a3b 100644 --- a/libswscale/x86/ops.c +++ b/libswscale/x86/ops.c @@ -320,11 +320,10 @@ static int setup_linear(const SwsImplParams *params, SwsImplResult *out) { const SwsUOp *uop = params->uop; if (uop->type == SWS_PIXEL_F32) { - out->ref = av_refstruct_allocz(sizeof(uop->data.mat4x5)); - if (!out->ref) - return AVERROR(ENOMEM); - memcpy(out->ref, uop->data.mat4x5, sizeof(uop->data.mat4x5)); - SWS_OP_PRIV_PACK(out, out->ref); + /* Layout hard-coded in ops_float.asm */ + static_assert(sizeof(uop->data.mat4x5) == 4 * 5 * sizeof(float), + "mat4x5 layout mismatch"); + SWS_OP_PRIV_PACK(out, uop->data.mat4x5); return 0; } diff --git a/libswscale/x86/ops_float.asm b/libswscale/x86/ops_float.asm index a1f5845260f..b7e03a722ab 100644 --- a/libswscale/x86/ops_float.asm +++ b/libswscale/x86/ops_float.asm @@ -580,25 +580,26 @@ IF NEED(3), linear_muladdps %1, mw%4, LOAD(3), m15, FMA(3) %assign ZERO_MASK %2 %assign EXACT_MASK %3 - mov tmp0q, [implq + SwsOpImpl.priv] ; address of matrix LOAD_CONT tmp1q -IF1 X, linear_row m8, tmp0q + 0 * BYTES, 0 -IF1 Y, linear_row m9, tmp0q + 5 * BYTES, 1 -IF1 Z, linear_row m10, tmp0q + 10 * BYTES, 2 -IF1 W, linear_row m11, tmp0q + 15 * BYTES, 3 +%define MATRIX implq + SwsOpImpl.priv +IF1 X, linear_row m8, MATRIX + 0 * BYTES, 0 +IF1 Y, linear_row m9, MATRIX + 5 * BYTES, 1 +IF1 Z, linear_row m10, MATRIX + 10 * BYTES, 2 +IF1 W, linear_row m11, MATRIX + 15 * BYTES, 3 IF X, mova mx, m8 IF Y, mova my, m9 IF Z, mova mz, m10 IF W, mova mw, m11 -IF1 X, linear_row m8, tmp0q + 0 * BYTES, 0, 2 -IF1 Y, linear_row m9, tmp0q + 5 * BYTES, 1, 2 -IF1 Z, linear_row m10, tmp0q + 10 * BYTES, 2, 2 -IF1 W, linear_row m11, tmp0q + 15 * BYTES, 3, 2 +IF1 X, linear_row m8, MATRIX + 0 * BYTES, 0, 2 +IF1 Y, linear_row m9, MATRIX + 5 * BYTES, 1, 2 +IF1 Z, linear_row m10, MATRIX + 10 * BYTES, 2, 2 +IF1 W, linear_row m11, MATRIX + 15 * BYTES, 3, 2 IF X, mova mx2, m8 IF Y, mova my2, m9 IF Z, mova mz2, m10 IF W, mova mw2, m11 - CONTINUE tmp1q, gprsize + CONTINUE tmp1q, 4 * 5 * BYTES +%undef MATRIX %endmacro %macro LINEAR 2 -- 2.54.0 >From 65f1a9be811f5c838c063d358908ea6abfc66a18 Mon Sep 17 00:00:00 2001 From: Niklas Haas <[email protected]> Date: Tue, 3 Nov 2026 22:31:05 +0100 Subject: [PATCH 24/24] swscale/aarch64/ops: store the linear coefficients in the op chain Avoids a separate allocation, and the dependent load of the coefficient pointer in the kernel. Signed-off-by: Niklas Haas <[email protected]> --- libswscale/aarch64/ops.c | 20 +++++++++++--------- libswscale/aarch64/ops_impl.h | 1 + libswscale/aarch64/ops_static.c | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/libswscale/aarch64/ops.c b/libswscale/aarch64/ops.c index ff223f982f3..e10d44eddc6 100644 --- a/libswscale/aarch64/ops.c +++ b/libswscale/aarch64/ops.c @@ -74,9 +74,12 @@ static int aarch64_setup_linear(const SwsAArch64OpImplParams *p, */ const int num_vregs = linear_num_vregs(p); av_assert0(num_vregs <= 4); - void *coeffs = av_refstruct_allocz(num_vregs * 16); - if (!coeffs) - return AVERROR(ENOMEM); + union { + uint8_t u8[4 * 16]; + uint16_t u16[4 * 8]; + uint32_t u32[4 * 4]; + float f32[4 * 4]; + } coeffs = { 0 }; /** * Copy non-zero coefficients, packed in sequential order, offset first. @@ -89,17 +92,16 @@ static int aarch64_setup_linear(const SwsAArch64OpImplParams *p, if (!(p->par.lin.zero & SWS_MASK(i, jj))) { const SwsPixel px = uop->data.mat4x5[i][jj]; switch (p->type) { - case SWS_PIXEL_U8: ((uint8_t *) coeffs)[i_coeff++] = px.u8; break; - case SWS_PIXEL_U16: ((uint16_t *) coeffs)[i_coeff++] = px.u16; break; - case SWS_PIXEL_U32: ((uint32_t *) coeffs)[i_coeff++] = px.u32; break; - case SWS_PIXEL_F32: ((float *) coeffs)[i_coeff++] = px.f32; break; + case SWS_PIXEL_U8: coeffs.u8[i_coeff++] = px.u8; break; + case SWS_PIXEL_U16: coeffs.u16[i_coeff++] = px.u16; break; + case SWS_PIXEL_U32: coeffs.u32[i_coeff++] = px.u32; break; + case SWS_PIXEL_F32: coeffs.f32[i_coeff++] = px.f32; break; } } } } - SWS_OP_PRIV_PACK(res, coeffs); - res->ref = coeffs; + ff_op_priv_pack(res, &coeffs, num_vregs * 16); return 0; } diff --git a/libswscale/aarch64/ops_impl.h b/libswscale/aarch64/ops_impl.h index b951e8b1c3f..8059d7dea4b 100644 --- a/libswscale/aarch64/ops_impl.h +++ b/libswscale/aarch64/ops_impl.h @@ -99,6 +99,7 @@ static inline int aarch64_priv_size(const SwsAArch64OpImplParams *params) return ff_sws_pixel_type_size(params->type); case SWS_UOP_LINEAR: case SWS_UOP_LINEAR_FMA: + return 16 * linear_num_vregs(params); /* packed coefficients */ case SWS_UOP_DITHER: return 8; /* pointer */ default: diff --git a/libswscale/aarch64/ops_static.c b/libswscale/aarch64/ops_static.c index 143db3f0c1f..e71246afc4f 100644 --- a/libswscale/aarch64/ops_static.c +++ b/libswscale/aarch64/ops_static.c @@ -309,7 +309,7 @@ static void asmgen_setup_linear(SwsAArch64Context *s, const SwsAArch64OpImplPara case 3: coeff_veclist = vv_3(vc[0], vc[1], vc[2]); break; case 4: coeff_veclist = vv_4(vc[0], vc[1], vc[2], vc[3]); break; } - i_ldr(r, ptr, IMPL_PRIV(s)); CMT("v128 *vcoeff_ptr = impl->priv.ptr;"); + i_add(r, ptr, s->impl, IMM(offsetof_impl_priv)); CMT("v128 *vcoeff_ptr = &impl->priv;"); asmgen_set_load_cont_node(s); i_ld1(r, coeff_veclist, a64op_base(ptr)); CMT("coeff_veclist = *vcoeff_ptr;"); -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
