PR #23292 opened by CyberShadow URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23292 Patch URL: https://code.ffmpeg.org/FFmpeg/FFmpeg/pulls/23292.patch
A few years ago I made these two improvements to the vf_photosensitivity filter (which I've submitted earlier). Since then, they've been tested "in the field"; users have submitted positive feedback about these patches and asked me to upstream them, so that they can reach downstream distributions of ffmpeg. If these patches are acceptable, please consider including them. Thank you >From f27015b3b43a0338d83273c4b985240aa6f4f2b2 Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev <[email protected]> Date: Tue, 3 Mar 2020 06:37:57 +0000 Subject: [PATCH 1/2] vf_photosensitivity: Scale badness threshold with history size This commit attempts to address feedback from this filter's users, by improving the filter's behavior at the start of playback or immediately after seeking. In these situations, the history buffer is empty, but because we did not previously track its size, we were calculating the weighted average as if the corresponding frames had zero badness. This caused the filter to behave differently and possibly produce false negatives when the history was not fully populated. Address this by instead taking into account the history size when calculating cumulative badness. To accomplish this: - Add a history_size field to to PhotosensitivityContext, tracking how much of the history buffer is populated. - Change the semantics of PhotosensitivityContext::badness_threshold. Previously, it was premultiplied by the maximum history size. This is no longer done, so that we can multiply it to the live history_size on the fly instead. - Calculate the badness threshold on a per-frame basis. The result is now stored in a badness_threshold local variable. This commit only changes the filter's behavior for the first PhotosensitivityContext::nb_frames (configurable as the "frames" filter option) frames. For successively filtered frames, the behavior is unchanged. --- libavfilter/vf_photosensitivity.c | 40 ++++++++++++++++++------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/libavfilter/vf_photosensitivity.c b/libavfilter/vf_photosensitivity.c index b8d3667ff8..1ef89e0242 100644 --- a/libavfilter/vf_photosensitivity.c +++ b/libavfilter/vf_photosensitivity.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019 Vladimir Panteleev + * Copyright (c) 2019, 2020 Vladimir Panteleev * * This file is part of FFmpeg. * @@ -46,7 +46,7 @@ typedef struct PhotosensitivityContext { /* Circular buffer */ int history[MAX_FRAMES]; - int history_pos; + int history_pos, history_size; PhotosensitivityFrame last_frame_e; AVFrame *last_frame_av; @@ -196,14 +196,14 @@ static int config_input(AVFilterLink *inlink) AVFilterContext *ctx = inlink->dst; PhotosensitivityContext *s = ctx->priv; - s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->nb_frames * s->threshold_multiplier / 128); + s->badness_threshold = (int)(GRID_SIZE * GRID_SIZE * 4 * 256 * s->threshold_multiplier / 128); return 0; } static int filter_frame(AVFilterLink *inlink, AVFrame *in) { - int this_badness, current_badness, fixed_badness, new_badness, i, res; + int this_badness, current_badness, fixed_badness, new_badness, badness_threshold, i, res, start; PhotosensitivityFrame ef; AVFrame *src, *out; int free_in = 0; @@ -216,26 +216,32 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) /* weighted moving average */ current_badness = 0; - for (i = 1; i < s->nb_frames; i++) - current_badness += i * s->history[(s->history_pos + i) % s->nb_frames]; - current_badness /= s->nb_frames; + if (s->history_size) { + start = s->nb_frames + s->history_pos - s->history_size; + for (i = 1; i < s->history_size; i++) + current_badness += i * s->history[(start + i) % s->nb_frames]; + current_badness /= s->history_size; + badness_threshold = s->badness_threshold * s->history_size; + } else { + badness_threshold = INT_MAX; + } convert_frame(ctx, in, &ef, s->skip); this_badness = get_badness(&ef, &s->last_frame_e); new_badness = current_badness + this_badness; av_log(ctx, AV_LOG_VERBOSE, "badness: %6d -> %6d / %6d (%3d%% - %s)\n", - current_badness, new_badness, s->badness_threshold, - 100 * new_badness / s->badness_threshold, new_badness < s->badness_threshold ? "OK" : "EXCEEDED"); + current_badness, new_badness, badness_threshold, + 100 * new_badness / badness_threshold, new_badness < badness_threshold ? "OK" : "EXCEEDED"); fixed_badness = new_badness; - if (new_badness < s->badness_threshold || !s->last_frame_av || s->bypass) { + if (new_badness < badness_threshold || !s->last_frame_av || s->bypass) { factor = 1; /* for metadata */ av_frame_free(&s->last_frame_av); s->last_frame_av = src = in; s->last_frame_e = ef; s->history[s->history_pos] = this_badness; } else { - factor = (float)(s->badness_threshold - current_badness) / (new_badness - current_badness); + factor = (float)(badness_threshold - current_badness) / (new_badness - current_badness); if (factor <= 0) { /* just duplicate the frame */ s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */ @@ -251,8 +257,8 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) this_badness = get_badness(&ef, &s->last_frame_e); fixed_badness = current_badness + this_badness; av_log(ctx, AV_LOG_VERBOSE, " fixed: %6d -> %6d / %6d (%3d%%) factor=%5.3f\n", - current_badness, fixed_badness, s->badness_threshold, - 100 * new_badness / s->badness_threshold, factor); + current_badness, fixed_badness, badness_threshold, + 100 * new_badness / badness_threshold, factor); s->last_frame_e = ef; s->history[s->history_pos] = this_badness; } @@ -260,6 +266,8 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) free_in = 1; } s->history_pos = (s->history_pos + 1) % s->nb_frames; + if (s->history_size < s->nb_frames) + s->history_size++; out = ff_get_video_buffer(outlink, in->width, in->height); if (!out) { @@ -272,13 +280,13 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) if (metadata) { char value[128]; - snprintf(value, sizeof(value), "%f", (float)new_badness / s->badness_threshold); + snprintf(value, sizeof(value), "%f", (float)new_badness / badness_threshold); av_dict_set(metadata, "lavfi.photosensitivity.badness", value, 0); - snprintf(value, sizeof(value), "%f", (float)fixed_badness / s->badness_threshold); + snprintf(value, sizeof(value), "%f", (float)fixed_badness / badness_threshold); av_dict_set(metadata, "lavfi.photosensitivity.fixed-badness", value, 0); - snprintf(value, sizeof(value), "%f", (float)this_badness / s->badness_threshold); + snprintf(value, sizeof(value), "%f", (float)this_badness / badness_threshold); av_dict_set(metadata, "lavfi.photosensitivity.frame-badness", value, 0); snprintf(value, sizeof(value), "%f", factor); -- 2.52.0 >From ebcbe885f511c0dbb108c075a805f95e93eb6468 Mon Sep 17 00:00:00 2001 From: Vladimir Panteleev <[email protected]> Date: Sat, 19 Nov 2022 20:23:59 +0000 Subject: [PATCH 2/2] vf_photosensitivity: Add blend option This commit attempts to address feedback from this filter's users, by introducing a new option which controls the amelioration mechanism. The "blend" option is a factor which is multiplied by the difference in badness (between the threshold and the currently accumulated badness). This difference normally controls how much of the next frame we can let through without making it exceed the badness threshold. Setting the option to zero effectively puts the filter into a mode where it always duplicates the last frame (which did not put accumulated badness over the threshold) instead of attempting to blend in new frames. I have received reports that this mode is preferable to users for some types of media. --- libavfilter/vf_photosensitivity.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavfilter/vf_photosensitivity.c b/libavfilter/vf_photosensitivity.c index 1ef89e0242..cd15027140 100644 --- a/libavfilter/vf_photosensitivity.c +++ b/libavfilter/vf_photosensitivity.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2019, 2020 Vladimir Panteleev + * Copyright (c) 2019, 2020, 2022 Vladimir Panteleev * * This file is part of FFmpeg. * @@ -39,7 +39,7 @@ typedef struct PhotosensitivityContext { int nb_frames; int skip; - float threshold_multiplier; + float threshold_multiplier, blend_factor; int bypass; int badness_threshold; @@ -60,6 +60,7 @@ static const AVOption photosensitivity_options[] = { { "f", "set how many frames to use", OFFSET(nb_frames), AV_OPT_TYPE_INT, {.i64=30}, 2, MAX_FRAMES, FLAGS }, { "threshold", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0.1, FLT_MAX, FLAGS }, { "t", "set detection threshold factor (lower is stricter)", OFFSET(threshold_multiplier), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0.1, FLT_MAX, FLAGS }, + { "blend", "set blending factor (0 always duplicates frames)", OFFSET(blend_factor), AV_OPT_TYPE_FLOAT, {.dbl=1}, 0, FLT_MAX, FLAGS }, { "skip", "set pixels to skip when sampling frames", OFFSET(skip), AV_OPT_TYPE_INT, {.i64=1}, 1, 1024, FLAGS }, { "bypass", "leave frames unchanged", OFFSET(bypass), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, FLAGS }, { NULL } @@ -241,7 +242,7 @@ static int filter_frame(AVFilterLink *inlink, AVFrame *in) s->last_frame_e = ef; s->history[s->history_pos] = this_badness; } else { - factor = (float)(badness_threshold - current_badness) / (new_badness - current_badness); + factor = (float)(badness_threshold - current_badness) / (new_badness - current_badness) * s->blend_factor; if (factor <= 0) { /* just duplicate the frame */ s->history[s->history_pos] = 0; /* frame was duplicated, thus, delta is zero */ -- 2.52.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
