PR #25125 opened by romainhedouin URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25125 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/25125.patch
Each predictor model held eight pointers (softmax and elliott for both weights and biases of both quality networks) The code that evaluates and normalises a network was duplicated This clean-up keeps one weights block & one bias block per quality To follow the logic of the weights file itself, I store the softmax neurons followed by the elliott neurons We can then loop over the qualities in predictor() and subtract_mean_predictor() A handful of helper functions that computed the per-neuron offsets are no longer needed Also, the model data has the same layout in memory as before and the dot products are done in the same order so the output does not change I've tested these changes extensively to ensure parity with the existing code: | Test | Size | Result | |---|---|---| | `x * 0.5f` / `x * 1.f` against `x / qual` | all 2^32 floats | identical | | Loaded and normalised weights of all 70 models | 13.6 MB | byte-identical | | Every dot-product call (inputs and result) | 8.6 billion calls | identical sequence | | Old vs. new predictor, bit for bit | 202 million pixels, 16 input patterns | identical | | Whole ffmpeg runs, frame checksums | 3,056 runs of varying formats/model/quality/... | identical | | ASan + UBSan | 159 runs, 4.4 million pixels | no reports | Issue spotted during a review sweep of ffmpeg using an LLM <!-- If this PR requires new FATE test samples, attach them to the PR and list their target paths below (relative to the fate-suite root). Attached filenames must match the sample's filename: ```fate-samples # e.g. vorbis/new-sample.ogg ``` --> >From 1d6a3e2bff3ee8fc5da53ec4112dd345dd9b1a6e Mon Sep 17 00:00:00 2001 From: romainhedouin <[email protected]> Date: Fri, 16 Oct 2026 20:18:04 +0200 Subject: [PATCH] avfilter/vf_nnedi: merge the duplicated quality 1 and 2 predictor code Each predictor model held eight pointers, for the softmax and elliott weights and biases of the two quality networks, and the code that evaluates and normalises a network was written out twice. Keep one weights block and one bias block per quality, with the softmax neurons followed by the elliott neurons as in the weights file, and loop over the qualities in predictor() and subtract_mean_predictor(). The helpers that computed the per-neuron offsets are no longer needed. The model data has the same layout in memory as before and the dot products are done in the same order, so the output does not change. Found during a code review sweep of ffmpeg using an LLM. --- libavfilter/vf_nnedi.c | 168 +++++++++++------------------------------ 1 file changed, 45 insertions(+), 123 deletions(-) diff --git a/libavfilter/vf_nnedi.c b/libavfilter/vf_nnedi.c index 148791f804..9826afdc01 100644 --- a/libavfilter/vf_nnedi.c +++ b/libavfilter/vf_nnedi.c @@ -52,14 +52,10 @@ typedef struct PrescreenerCoefficients { typedef struct PredictorCoefficients { int xdim, ydim, nns, nsize; float *data; - float *softmax_q1; - float *elliott_q1; - float *softmax_bias_q1; - float *elliott_bias_q1; - float *softmax_q2; - float *elliott_q2; - float *softmax_bias_q2; - float *elliott_bias_q2; + /* [0]: quality 1 network, [1]: additional network used for quality 2; + * nns softmax neurons followed by nns elliott neurons */ + float *weights[2]; + float *bias[2]; } PredictorCoefficients; typedef struct NNEDIContext { @@ -297,35 +293,6 @@ static void process_new(AVFilterContext *ctx, } } -static int filter_offset(int nn, const PredictorCoefficients *const model) -{ - return nn * model->nsize; -} - -static const float *softmax_q1_filter(int nn, - const PredictorCoefficients *const model) -{ - return model->softmax_q1 + filter_offset(nn, model); -} - -static const float *elliott_q1_filter(int nn, - const PredictorCoefficients *const model) -{ - return model->elliott_q1 + filter_offset(nn, model); -} - -static const float *softmax_q2_filter(int nn, - const PredictorCoefficients *const model) -{ - return model->softmax_q2 + filter_offset(nn, model); -} - -static const float *elliott_q2_filter(int nn, - const PredictorCoefficients *const model) -{ - return model->elliott_q2 + filter_offset(nn, model); -} - static void gather_input(const float *src, ptrdiff_t src_stride, float *buf, float mstd[4], const PredictorCoefficients *const model) @@ -392,7 +359,7 @@ static void wae5(const float *softmax, const float *el, static void predictor(AVFilterContext *ctx, const void *src, ptrdiff_t src_stride, void *dst, const uint8_t *prescreen, int N, - const PredictorCoefficients *const model, int use_q2) + const PredictorCoefficients *const model, int qual) { const NNEDIContext *const s = ctx->priv; const float *src_p = src; @@ -415,27 +382,18 @@ static void predictor(AVFilterContext *ctx, gather_input(window + i, src_stride, input, mstd, model); scale = mstd[2]; - for (int nn = 0; nn < nns; nn++) - activation[nn] = dot_dsp(s, softmax_q1_filter(nn, model), input, filter_size, scale, model->softmax_bias_q1[nn]); + for (int q = 0; q < qual; q++) { + const float *weights = model->weights[q]; + const float *bias = model->bias[q]; - for (int nn = 0; nn < nns; nn++) - activation[nns + nn] = dot_dsp(s, elliott_q1_filter(nn, model), input, filter_size, scale, model->elliott_bias_q1[nn]); - - transform_softmax_exp(activation, nns); - wae5(activation, activation + nns, nns, mstd); - - if (use_q2) { - for (int nn = 0; nn < nns; nn++) - activation[nn] = dot_dsp(s, softmax_q2_filter(nn, model), input, filter_size, scale, model->softmax_bias_q2[nn]); - - for (int nn = 0; nn < nns; nn++) - activation[nns + nn] = dot_dsp(s, elliott_q2_filter(nn, model), input, filter_size, scale, model->elliott_bias_q2[nn]); + for (int nn = 0; nn < 2 * nns; nn++) + activation[nn] = dot_dsp(s, weights + nn * filter_size, input, filter_size, scale, bias[nn]); transform_softmax_exp(activation, nns); wae5(activation, activation + nns, nns, mstd); } - dst_p[i] = mstd[3] * (use_q2 ? 0.5f : 1.f); + dst_p[i] = mstd[3] / qual; } } @@ -642,7 +600,7 @@ static int filter_slice(AVFilterContext *ctx, void *arg, int jobnr, int nb_jobs) srcbuf_stride, dstbuf + (y / 2) * dstbuf_stride, prescreen_buf, width, - &s->coeffs[s->etype][s->nnsparam][s->nsize], s->qual == 2); + &s->coeffs[s->etype][s->nnsparam][s->nsize], s->qual); if (s->pscrn > 0) interpolation(srcbuf + (y / 2) * srcbuf_stride + 32, @@ -774,15 +732,10 @@ static int allocate_model(PredictorCoefficients *coeffs, int xdim, int ydim, int coeffs->nsize = xdim * ydim; coeffs->nns = nns; - coeffs->softmax_q1 = allocate(&data, filter_size); - coeffs->elliott_q1 = allocate(&data, filter_size); - coeffs->softmax_bias_q1 = allocate(&data, bias_size); - coeffs->elliott_bias_q1 = allocate(&data, bias_size); - - coeffs->softmax_q2 = allocate(&data, filter_size); - coeffs->elliott_q2 = allocate(&data, filter_size); - coeffs->softmax_bias_q2 = allocate(&data, bias_size); - coeffs->elliott_bias_q2 = allocate(&data, bias_size); + for (int q = 0; q < 2; q++) { + coeffs->weights[q] = allocate(&data, 2 * filter_size); + coeffs->bias[q] = allocate(&data, 2 * bias_size); + } return 0; } @@ -836,21 +789,13 @@ static int read_weights(AVFilterContext *ctx, const float *bdata) if (ret < 0) return ret; - // Quality 1 model. NNS[i] * (XDIM[j] * YDIM[j]) * 2 coefficients. - copy_weights(model->softmax_q1, nns * filter_size, &bdata); - copy_weights(model->elliott_q1, nns * filter_size, &bdata); - - // Quality 1 model bias. NNS[i] * 2 coefficients. - copy_weights(model->softmax_bias_q1, nns, &bdata); - copy_weights(model->elliott_bias_q1, nns, &bdata); - - // Quality 2 model. NNS[i] * (XDIM[j] * YDIM[j]) * 2 coefficients. - copy_weights(model->softmax_q2, nns * filter_size, &bdata); - copy_weights(model->elliott_q2, nns * filter_size, &bdata); - - // Quality 2 model bias. NNS[i] * 2 coefficients. - copy_weights(model->softmax_bias_q2, nns, &bdata); - copy_weights(model->elliott_bias_q2, nns, &bdata); + // Quality 1 then quality 2 model. + for (int q = 0; q < 2; q++) { + // NNS[i] * (XDIM[j] * YDIM[j]) * 2 coefficients. + copy_weights(model->weights[q], 2 * nns * filter_size, &bdata); + // NNS[i] * 2 biases. + copy_weights(model->bias[q], 2 * nns, &bdata); + } } } } @@ -898,57 +843,34 @@ static void subtract_mean_predictor(PredictorCoefficients *model) const int nns = model->nns; const float scale = 1.f / nns; - double softmax_means[256]; // Average of individual softmax filters. - double elliott_means[256]; // Average of individual elliott filters. - double mean_filter[48 * 6] = { 0 }; // Pointwise average of all softmax filters. - double mean_bias; + for (int q = 0; q < 2; q++) { + float *softmax = model->weights[q]; + float *elliott = model->weights[q] + nns * filter_size; + double softmax_means[256]; // Average of individual softmax filters. + double elliott_means[256]; // Average of individual elliott filters. + double mean_filter[48 * 6] = { 0 }; // Pointwise average of all softmax filters. + double mean_bias; - // Quality 1. - for (int nn = 0; nn < nns; nn++) { - softmax_means[nn] = mean(model->softmax_q1 + nn * filter_size, filter_size); - elliott_means[nn] = mean(model->elliott_q1 + nn * filter_size, filter_size); + for (int nn = 0; nn < nns; nn++) { + softmax_means[nn] = mean(softmax + nn * filter_size, filter_size); + elliott_means[nn] = mean(elliott + nn * filter_size, filter_size); + + for (int k = 0; k < filter_size; k++) + mean_filter[k] += softmax[nn * filter_size + k] - softmax_means[nn]; + } for (int k = 0; k < filter_size; k++) - mean_filter[k] += model->softmax_q1[nn * filter_size + k] - softmax_means[nn]; - } + mean_filter[k] *= scale; - for (int k = 0; k < filter_size; k++) - mean_filter[k] *= scale; + mean_bias = mean(model->bias[q], nns); - mean_bias = mean(model->softmax_bias_q1, nns); - - for (int nn = 0; nn < nns; nn++) { - for (int k = 0; k < filter_size; k++) { - model->softmax_q1[nn * filter_size + k] -= softmax_means[nn] + mean_filter[k]; - model->elliott_q1[nn * filter_size + k] -= elliott_means[nn]; + for (int nn = 0; nn < nns; nn++) { + for (int k = 0; k < filter_size; k++) { + softmax[nn * filter_size + k] -= softmax_means[nn] + mean_filter[k]; + elliott[nn * filter_size + k] -= elliott_means[nn]; + } + model->bias[q][nn] -= mean_bias; } - model->softmax_bias_q1[nn] -= mean_bias; - } - - // Quality 2. - memset(mean_filter, 0, sizeof(mean_filter)); - - for (int nn = 0; nn < nns; nn++) { - softmax_means[nn] = mean(model->softmax_q2 + nn * filter_size, filter_size); - elliott_means[nn] = mean(model->elliott_q2 + nn * filter_size, filter_size); - - for (int k = 0; k < filter_size; k++) { - mean_filter[k] += model->softmax_q2[nn * filter_size + k] - softmax_means[nn]; - } - } - - for (int k = 0; k < filter_size; k++) - mean_filter[k] *= scale; - - mean_bias = mean(model->softmax_bias_q2, nns); - - for (int nn = 0; nn < nns; nn++) { - for (int k = 0; k < filter_size; k++) { - model->softmax_q2[nn * filter_size + k] -= softmax_means[nn] + mean_filter[k]; - model->elliott_q2[nn * filter_size + k] -= elliott_means[nn]; - } - - model->softmax_bias_q2[nn] -= mean_bias; } } -- 2.54.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
