From 0972d337c7408b943b5a38dbaacb813dc51c593d Mon Sep 17 00:00:00 2001 From: Vittorio Giovara Date: Mon, 14 Jul 2014 21:26:55 -0400 Subject: [PATCH] Add new vf_tiltandshift filter Signed-off-by: Vittorio Giovara --- Changelog | 1 + doc/filters.texi | 63 +++++ libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/vf_tiltandshift.c | 360 +++++++++++++++++++++++++++++ tests/fate/filter-video.mak | 2 + tests/ref/fate/filter-tiltandshift | 1 + 7 files changed, 429 insertions(+) create mode 100644 libavfilter/vf_tiltandshift.c create mode 100644 tests/ref/fate/filter-tiltandshift diff --git a/Changelog b/Changelog index f00bc27ca4..0ca867cc47 100644 --- a/Changelog +++ b/Changelog @@ -7,6 +7,7 @@ version : - EVC encoding using external library libxeve - QOA decoder and demuxer - aap filter +- tiltandshift filter version 6.1: - libaribcaption decoder diff --git a/doc/filters.texi b/doc/filters.texi index 6d00ba2c3f..1d1634bf06 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -23186,6 +23186,69 @@ tile=3x2:nb_frames=5:padding=7:margin=2 @end example @end itemize +@section tiltandshift + +What happens when you invert time and space? + +Normally a video is composed of several frames that represent a different +instant of time and shows a scence that evolves in the space captured by the +frame. This filter is the antipode of that concept, taking inspiration by +tilt and shift photography. + +A filtered frame contains the whole timeline of events composing the sequence, +and this is obtained by placing a slice of pixels from each frame into a single +one. However, since there are no infinite-width frames, this is done up the +width of the input frame, and a video is recomposed by shifting away one +column for each subsequent frame. In order to map space to time, the filter +tilts each input frame as well, so that motion is preseved. This is accomplished +by progressively selecting a different column from each input frame. + +The end result is a sort of inverted parralax, so that far away objects move +much faster that the ones in the front. The ideal conditions for this video +effect are when there is either very little motion and the backgroud is static, +or when there is a lot of motion and a very wide depth of field (eg. wide +panorama, while moving on a train). + +The filter accepts the following parameters: + +@table @option + +@item tilt +Tilt video while shifting (default). When unset, video will be sliding a +static image, composed of the first column of each frame. + +@item start +What to do at the start of filtering (see below). + +@item end +What to do at the end of filtering (see below). + +@item hold +How many columns should pass through before start of filtering. + +@item pad +How many columns should be inserted before end of filtering. + +@end table + +Normally the filter shifts and tils from the very first frame, and stops when +the last one is received. However, before filtering starts, normal video may +be preseved, so that the effect is slowly shifted in its place. Similarly, +the last video frame may be reconstructed at the end. Alternatively it is +possible to just start and end with black. + +@table @samp +@item none +Filtering is starts immediately and ends when the last frame is received. + +@item frame +The first frames or the very last frame are kept intact during processing. + +@item black +Black is padded at the beginning or at the end of filtering. + +@end table + @section tinterlace Perform various types of temporal field interlacing. diff --git a/libavfilter/Makefile b/libavfilter/Makefile index 63725f91b4..afc7bc1566 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -514,6 +514,7 @@ OBJS-$(CONFIG_THUMBNAIL_FILTER) += vf_thumbnail.o OBJS-$(CONFIG_THUMBNAIL_CUDA_FILTER) += vf_thumbnail_cuda.o vf_thumbnail_cuda.ptx.o \ cuda/load_helper.o OBJS-$(CONFIG_TILE_FILTER) += vf_tile.o +OBJS-$(CONFIG_TILTANDSHIFT_FILTER) += vf_tiltandshift.o OBJS-$(CONFIG_TINTERLACE_FILTER) += vf_tinterlace.o OBJS-$(CONFIG_TLUT2_FILTER) += vf_lut2.o framesync.o OBJS-$(CONFIG_TMEDIAN_FILTER) += vf_xmedian.o framesync.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index ed7c32be94..135794ba36 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -483,6 +483,7 @@ extern const AVFilter ff_vf_threshold; extern const AVFilter ff_vf_thumbnail; extern const AVFilter ff_vf_thumbnail_cuda; extern const AVFilter ff_vf_tile; +extern const AVFilter ff_vf_tiltandshift; extern const AVFilter ff_vf_tinterlace; extern const AVFilter ff_vf_tlut2; extern const AVFilter ff_vf_tmedian; diff --git a/libavfilter/vf_tiltandshift.c b/libavfilter/vf_tiltandshift.c new file mode 100644 index 0000000000..243105a9e8 --- /dev/null +++ b/libavfilter/vf_tiltandshift.c @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2014 Vittorio Giovara + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +/** + * @file vf_tiltandshift.c + * Simple time and space inverter. + */ + +#include + +#include "libavutil/common.h" +#include "libavutil/imgutils.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "libavutil/rational.h" + +#include "avfilter.h" +#include "formats.h" +#include "internal.h" +#include "video.h" + +#define TILT_NONE -1 +#define TILT_FRAME 0 +#define TILT_BLACK 1 + +typedef struct FrameList { + AVFrame *frame; + struct FrameList *next; +} FrameList; + +typedef struct TiltandshiftContext { + const AVClass *class; + + /* set when all input frames have been processed and we have to + * empty buffers, pad and then return */ + int eof_recv; + + /* live or static sliding */ + int tilt; + + /* initial or final actions to perform (pad/hold a frame/black/nothing) */ + int start; + int end; + + /* columns to hold or pad at the beginning or at the end (respectively) */ + int hold; + int pad; + + /* buffers for black columns */ + uint8_t *black_buffers[4]; + int black_linesizes[4]; + + /* list containing all input frames */ + int list_size; + FrameList *input; + FrameList *prev; + + const AVPixFmtDescriptor *desc; +} TiltandshiftContext; + +static int list_add_frame(TiltandshiftContext *s, AVFrame *frame) +{ + FrameList *element = av_mallocz(sizeof(FrameList)); + if (!element) + return AVERROR(ENOMEM); + + element->frame = frame; + + if (s->input == NULL) { + s->input = element; + } else { + FrameList *head = s->input; + while (head->next) + head = head->next; + head->next = element; + } + + s->list_size++; + return 0; +} + +static void list_remove_head(TiltandshiftContext *s) +{ + FrameList *head = s->input; + + if (head) { + av_frame_free(&head->frame); + s->input = head->next; + av_freep(&head); + } + + s->list_size--; +} + +static const enum AVPixelFormat formats_supported[] = { + AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P, + AV_PIX_FMT_YUV410P, + AV_PIX_FMT_YUVJ420P, AV_PIX_FMT_YUVJ422P, AV_PIX_FMT_YUVJ444P, + AV_PIX_FMT_YUVJ440P, + AV_PIX_FMT_NONE +}; + +static int query_formats(AVFilterContext *ctx) +{ + return ff_set_common_formats(ctx, ff_make_format_list(formats_supported)); +} + +static av_cold void uninit(AVFilterContext *ctx) +{ + TiltandshiftContext *s = ctx->priv; + while (s->input) + list_remove_head(s); + av_freep(&s->black_buffers); +} + +static int config_props(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = ctx->priv; + + outlink->w = ctx->inputs[0]->w; + outlink->h = ctx->inputs[0]->h; + outlink->format = ctx->inputs[0]->format; + + // when we have to pad black or a frame at the start, skip navigating + // the list and use either the frame or black for the requested value + if (s->start != TILT_NONE && !s->hold) + s->hold = outlink->w; + + // Init black buffers if we pad with black at the start or at the end. + // For the end, we always have to init on NONE and BLACK because we never + // know if there are going to be enough input frames to fill an output one. + if (s->start == TILT_BLACK || s->end != TILT_FRAME) { + int i, j, ret; + uint8_t black_data[] = { 0x10, 0x80, 0x80, 0x10 }; + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(outlink->format); + if (!desc) + return AVERROR_BUG; + + if (outlink->format == AV_PIX_FMT_YUVJ420P || + outlink->format == AV_PIX_FMT_YUVJ422P || + outlink->format == AV_PIX_FMT_YUVJ444P || + outlink->format == AV_PIX_FMT_YUVJ440P) + black_data[0] = black_data[3] = 0; + + ret = av_image_alloc(s->black_buffers, s->black_linesizes, 1, + outlink->h, outlink->format, 1); + if (ret < 0) + return ret; + + for (i = 0; i < FFMIN(desc->nb_components, 4); i++) + for (j = 0; j < (!i ? outlink->h + : -((-outlink->h) >> desc->log2_chroma_h)); j++) + memset(s->black_buffers[i] + j * s->black_linesizes[i], + black_data[i], 1); + + av_log(ctx, AV_LOG_VERBOSE, "Padding buffers initialized.\n"); + } + + s->desc = av_pix_fmt_desc_get(outlink->format); + if (!s->desc) + return AVERROR_BUG; + + return 0; +} + +// This function just polls for new frames and queues them on a list +static int filter_frame(AVFilterLink *inlink, AVFrame *frame) +{ + TiltandshiftContext *s = inlink->dst->priv; + return list_add_frame(s, frame); +} + +static void copy_column(AVFilterLink *outlink, + uint8_t *dst_data[4], int dst_linesizes[4], + const uint8_t *src_data[4], const int src_linesizes[4], + int ncol, int tilt) +{ + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = ctx->priv; + uint8_t *dst[4]; + const uint8_t *src[4]; + + dst[0] = dst_data[0] + ncol; + dst[1] = dst_data[1] + (ncol >> s->desc->log2_chroma_h); + dst[2] = dst_data[2] + (ncol >> s->desc->log2_chroma_h); + + if (!tilt) + ncol = 0; + src[0] = src_data[0] + ncol; + src[1] = src_data[1] + (ncol >> s->desc->log2_chroma_h); + src[2] = src_data[2] + (ncol >> s->desc->log2_chroma_h); + + av_image_copy(dst, dst_linesizes, src, src_linesizes, outlink->format, 1, outlink->h); +} + +static int request_frame(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + TiltandshiftContext *s = ctx->priv; + FrameList *head = s->input; + int ret, ncol; + AVFrame *dst; + + // signal job finished when list is empty or when padding is either + // limited or disabled and eof was received + if ((s->list_size <= 0 || s->list_size == outlink->w - s->pad || s->end == TILT_NONE) && s->eof_recv) { + return AVERROR_EOF; + } + + ret = ff_request_frame(ctx->inputs[0]); + if (ret == AVERROR_EOF) { + av_log(ctx, AV_LOG_VERBOSE, "Last frame, emptying buffers (%d/%d).\n", s->list_size, outlink->w - s->pad); + s->eof_recv = 1; + } else if (ret < 0) { + return ret; + } + + // load up enough frames to fill a frame and keep the queue filled on subsequent + // calls, until we receive EOF, and then we either pad or end + if (!s->eof_recv && s->list_size < outlink->w - s->pad) { + av_log(ctx, AV_LOG_VERBOSE, "Not enough frames in the list (%d/%d), polling for more.\n", + s->list_size, outlink->w - s->pad); + return AVERROR(EAGAIN); + } + + // new frame + ncol = 0; + dst = ff_get_video_buffer(outlink, outlink->w, outlink->h); + if (!dst) + return AVERROR(ENOMEM); + + // in case we have to do any initial black padding + if (s->start == TILT_BLACK) { + for ( ; ncol < s->hold; ncol++) + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)s->black_buffers, s->black_linesizes, + ncol, 0); + } + + // copy a column from each input frame + for ( ; ncol < s->list_size; ncol++) { + AVFrame *src = head->frame; + + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)src->data, src->linesize, + ncol, s->tilt); + + // keep track of the last known frame in case we need it below + s->prev = head; + // advance to the next frame unless we have to hold it + if (s->hold <= ncol) + head = head->next; + } + + // pad any remaining space with black or last frame + if (s->end == TILT_FRAME) { + for ( ; ncol < outlink->w; ncol++) + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)s->prev->frame->data, + s->prev->frame->linesize, ncol, 1); + } else { // TILT_BLACK and TILT_NONE + for ( ; ncol < outlink->w; ncol++) + copy_column(outlink, dst->data, dst->linesize, + (const uint8_t **)s->black_buffers, s->black_linesizes, + ncol, 0); + } + + // set correct timestamps and props as long as there is proper input + ret = av_frame_copy_props(dst, s->input->frame); + if (ret < 0) + return ret; + + // discard frame at the top of the list since it has been fully processed + list_remove_head(s); + // and it is safe to reduce the hold value (even if unused) + s->hold--; + + // output + return ff_filter_frame(outlink, dst); +} + +#define OFFSET(x) offsetof(TiltandshiftContext, x) +#define V AV_OPT_FLAG_VIDEO_PARAM +static const AVOption tiltandshift_options[] = { + { "tilt", "Tilt the video horizontally while shifting", OFFSET(tilt), AV_OPT_TYPE_INT, + { .i64 = 1 }, 0, 1, .flags = V, .unit = "tilt" }, + + { "start", "Action at the start of input", OFFSET(start), AV_OPT_TYPE_INT, + { .i64 = TILT_NONE }, -1, 1, .flags = V, .unit = "start" }, + { "none", "Start immediately (default)", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "start" }, + { "frame", "Use the first frames", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "start" }, + { "black", "Fill with black", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "start" }, + + { "end", "Action at the end of input", OFFSET(end), AV_OPT_TYPE_INT, + { .i64 = TILT_NONE }, -1, 1, .flags = V, .unit = "end" }, + { "none", "Do not pad at the end (default)", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_NONE }, INT_MIN, INT_MAX, .flags = V, .unit = "end" }, + { "frame", "Use the last frame", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_FRAME }, INT_MIN, INT_MAX, .flags = V, .unit = "end" }, + { "black", "Fill with black", 0, AV_OPT_TYPE_CONST, + { .i64 = TILT_BLACK }, INT_MIN, INT_MAX, .flags = V, .unit = "end" }, + + { "hold", "Number of columns to hold at the start of the video", OFFSET(hold), AV_OPT_TYPE_INT, + { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "hold" }, + { "pad", "Number of columns to pad at the end of the video", OFFSET(pad), AV_OPT_TYPE_INT, + { .i64 = 0 }, 0, INT_MAX, .flags = V, .unit = "pad" }, + + { NULL }, +}; + +AVFILTER_DEFINE_CLASS(tiltandshift); + +static const AVFilterPad tiltandshift_inputs[] = { + { + .name = "in", + .type = AVMEDIA_TYPE_VIDEO, + .filter_frame = filter_frame, + }, +}; + +static const AVFilterPad tiltandshift_outputs[] = { + { + .name = "out", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = config_props, + .request_frame = request_frame, + }, +}; + +AVFilter ff_vf_tiltandshift = { + .name = "tiltandshift", + .description = NULL_IF_CONFIG_SMALL("Generate a tilt-and-shift'd video."), + .priv_size = sizeof(TiltandshiftContext), + .priv_class = &tiltandshift_class, + .uninit = uninit, + FILTER_INPUTS(tiltandshift_inputs), + FILTER_OUTPUTS(tiltandshift_outputs), + FILTER_QUERY_FUNC(query_formats), +}; diff --git a/tests/fate/filter-video.mak b/tests/fate/filter-video.mak index e4bdf59db9..25bc47b393 100644 --- a/tests/fate/filter-video.mak +++ b/tests/fate/filter-video.mak @@ -532,6 +532,8 @@ fate-filter-thumbnail: CMD = video_filter "scale,thumbnail=10" FATE_FILTER_VSYNTH_VIDEO_FILTER-$(CONFIG_TILE_FILTER) += fate-filter-tile fate-filter-tile: CMD = video_filter "tile=3x3:nb_frames=5:padding=7:margin=2" +FATE_FILTER_VSYNTH_VIDEO_FILTER-$(CONFIG_TILTANDSHIFT_FILTER) += fate-filter-tiltandshift +fate-filter-tiltandshift: CMD = video_filter "tiltandshift" tests/pixfmts.mak: TAG = GEN tests/pixfmts.mak: ffmpeg$(PROGSSUF)$(EXESUF) | tests diff --git a/tests/ref/fate/filter-tiltandshift b/tests/ref/fate/filter-tiltandshift new file mode 100644 index 0000000000..e639922e3b --- /dev/null +++ b/tests/ref/fate/filter-tiltandshift @@ -0,0 +1 @@ +tiltandshift eed1caa0c2551fddad2e45ce84b465f0 -- 2.42.0