From 72ce447ea2bf3c0fbf98a3324b56c1875d3dcb5f Mon Sep 17 00:00:00 2001 From: Roman Arzumanyan Date: Wed, 29 Aug 2018 12:53:02 +0300 Subject: [PATCH] Adding NPP transpose filter --- libavfilter/Makefile | 1 + libavfilter/allfilters.c | 1 + libavfilter/vf_transpose_npp.c | 515 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 517 insertions(+) create mode 100644 libavfilter/vf_transpose_npp.c diff --git a/libavfilter/Makefile b/libavfilter/Makefile index e5d3a57af7..b756a3cae8 100644 --- a/libavfilter/Makefile +++ b/libavfilter/Makefile @@ -373,6 +373,7 @@ OBJS-$(CONFIG_TONEMAP_FILTER) += vf_tonemap.o colorspace.o OBJS-$(CONFIG_TONEMAP_OPENCL_FILTER) += vf_tonemap_opencl.o colorspace.o opencl.o \ opencl/tonemap.o opencl/colorspace_common.o OBJS-$(CONFIG_TRANSPOSE_FILTER) += vf_transpose.o +OBJS-$(CONFIG_TRANSPOSE_NPP_FILTER) += vf_transpose_npp.o OBJS-$(CONFIG_TRIM_FILTER) += trim.o OBJS-$(CONFIG_UNPREMULTIPLY_FILTER) += vf_premultiply.o framesync.o OBJS-$(CONFIG_UNSHARP_FILTER) += vf_unsharp.o diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c index 9732ae5345..260038e030 100644 --- a/libavfilter/allfilters.c +++ b/libavfilter/allfilters.c @@ -355,6 +355,7 @@ extern AVFilter ff_vf_tmix; extern AVFilter ff_vf_tonemap; extern AVFilter ff_vf_tonemap_opencl; extern AVFilter ff_vf_transpose; +extern AVFilter ff_vf_transpose_npp; extern AVFilter ff_vf_trim; extern AVFilter ff_vf_unpremultiply; extern AVFilter ff_vf_unsharp; diff --git a/libavfilter/vf_transpose_npp.c b/libavfilter/vf_transpose_npp.c new file mode 100644 index 0000000000..355991e5eb --- /dev/null +++ b/libavfilter/vf_transpose_npp.c @@ -0,0 +1,515 @@ +/* + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ +/** + * @file + * transpose npp video filter + */ +#include +#include +#include +#include "libavutil/avstring.h" +#include "libavutil/common.h" +#include "libavutil/eval.h" +#include "libavutil/hwcontext.h" +#include "libavutil/hwcontext_cuda_internal.h" +#include "libavutil/internal.h" +#include "libavutil/opt.h" +#include "libavutil/pixdesc.h" +#include "avfilter.h" +#include "formats.h" +#include "internal.h" +#include "video.h" + +static const enum AVPixelFormat supported_formats[] = { + AV_PIX_FMT_YUV420P, + AV_PIX_FMT_NV12, + AV_PIX_FMT_YUV444P, +}; + +static const enum AVPixelFormat deinterleaved_formats[][2] = { + { AV_PIX_FMT_NV12, AV_PIX_FMT_YUV420P }, +}; + +enum TransposeStage { + STAGE_DEINTERLEAVE, + STAGE_TRANSPOSE, + STAGE_INTERLEAVE, + STAGE_NB, +}; + +enum Transpose { + NPP_TRANSPOSE_NONE, + NPP_TRANSPOSE_CW_90, + NPP_TRANSPOSE_CW_180, + NPP_TRANSPOSE_CW_270, + NPP_TRANSPOSE_INVALID, +}; + +typedef struct NPPTransposeStageContext { + int stage_needed; + enum AVPixelFormat in_fmt; + enum AVPixelFormat out_fmt; + struct { + int width; + int height; + } planes_in[3], planes_out[3]; + AVBufferRef *frames_ctx; + AVFrame *frame; +} NPPTransposeStageContext; + +typedef struct NPPTransposeContext { + const AVClass *class; + NPPTransposeStageContext stages[STAGE_NB]; + AVFrame *tmp_frame; + int passthrough; + int shift_width, shift_height; + /** + * Transpose. Values are: + * 0 = original orientation + * 1 = Rotate clockwise 90 + * 2 = Rotate clockwise 180 + * 3 = Rotate clockwise 270 + */ + int t; + /** + * Output sw format. AV_PIX_FMT_NONE for no conversion. + */ + enum AVPixelFormat format; + char *t_expr; ///< transpose expression string + char *format_str; + int interp_algo; +} NPPTransposeContext; + +static int npptranspose_init(AVFilterContext *ctx) +{ + NPPTransposeContext *s = ctx->priv; + int i; + if (!strcmp(s->format_str, "same")) { + s->format = AV_PIX_FMT_NONE; + } else { + s->format = av_get_pix_fmt(s->format_str); + if (s->format == AV_PIX_FMT_NONE) { + av_log(ctx, AV_LOG_ERROR, "Unrecognized pixel format: %s\n", s->format_str); + return AVERROR(EINVAL); + } + } + for (i = 0; i < FF_ARRAY_ELEMS(s->stages); i++) { + s->stages[i].frame = av_frame_alloc(); + if (!s->stages[i].frame) + return AVERROR(ENOMEM); + } + s->tmp_frame = av_frame_alloc(); + if (!s->tmp_frame) + return AVERROR(ENOMEM); + return 0; +} + +static void npptranspose_uninit(AVFilterContext *ctx) +{ + NPPTransposeContext *s = ctx->priv; + int i; + for (i = 0; i < FF_ARRAY_ELEMS(s->stages); i++) { + av_frame_free(&s->stages[i].frame); + av_buffer_unref(&s->stages[i].frames_ctx); + } + av_frame_free(&s->tmp_frame); +} + +static int npptranspose_query_formats(AVFilterContext *ctx) +{ + static const enum AVPixelFormat pixel_formats[] = { + AV_PIX_FMT_CUDA, AV_PIX_FMT_NONE, + }; + AVFilterFormats *pix_fmts = ff_make_format_list(pixel_formats); + return ff_set_common_formats(ctx, pix_fmts); +} + +static int init_stage(NPPTransposeStageContext *stage, AVBufferRef *device_ctx) +{ + AVBufferRef *out_ref = NULL; + AVHWFramesContext *out_ctx; + int in_sw, in_sh, out_sw, out_sh; + int ret, i; + av_pix_fmt_get_chroma_sub_sample(stage->in_fmt, &in_sw, &in_sh); + av_pix_fmt_get_chroma_sub_sample(stage->out_fmt, &out_sw, &out_sh); + if (!stage->planes_out[0].width) { + stage->planes_out[0].width = stage->planes_in[0].width; + stage->planes_out[0].height = stage->planes_in[0].height; + } + for (i = 1; i < FF_ARRAY_ELEMS(stage->planes_in); i++) { + stage->planes_in[i].width = stage->planes_in[0].width >> in_sw; + stage->planes_in[i].height = stage->planes_in[0].height >> in_sh; + stage->planes_out[i].width = stage->planes_out[0].width >> out_sw; + stage->planes_out[i].height = stage->planes_out[0].height >> out_sh; + } + out_ref = av_hwframe_ctx_alloc(device_ctx); + if (!out_ref) + return AVERROR(ENOMEM); + out_ctx = (AVHWFramesContext*)out_ref->data; + out_ctx->format = AV_PIX_FMT_CUDA; + out_ctx->sw_format = stage->out_fmt; + out_ctx->width = FFALIGN(stage->planes_out[0].width, 32); + out_ctx->height = FFALIGN(stage->planes_out[0].height, 32); + ret = av_hwframe_ctx_init(out_ref); + if (ret < 0) + goto fail; + av_frame_unref(stage->frame); + ret = av_hwframe_get_buffer(out_ref, stage->frame, 0); + if (ret < 0) + goto fail; + stage->frame->width = stage->planes_out[0].width; + stage->frame->height = stage->planes_out[0].height; + av_buffer_unref(&stage->frames_ctx); + stage->frames_ctx = out_ref; + return 0; +fail: + av_buffer_unref(&out_ref); + return ret; +} + +static int format_is_supported(enum AVPixelFormat fmt) +{ + int i; + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) + if (supported_formats[i] == fmt) + return 1; + return 0; +} + +static enum AVPixelFormat get_deinterleaved_format(enum AVPixelFormat fmt) +{ + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(fmt); + int i, planes; + planes = av_pix_fmt_count_planes(fmt); + if (planes == desc->nb_components) + return fmt; + for (i = 0; i < FF_ARRAY_ELEMS(deinterleaved_formats); i++) + if (deinterleaved_formats[i][0] == fmt) + return deinterleaved_formats[i][1]; + return AV_PIX_FMT_NONE; +} + +static int init_processing_chain(AVFilterContext *ctx, int in_width, int in_height, + int out_width, int out_height) +{ + NPPTransposeContext *s = ctx->priv; + AVHWFramesContext *in_frames_ctx; + enum AVPixelFormat in_format; + enum AVPixelFormat out_format; + enum AVPixelFormat in_deinterleaved_format; + enum AVPixelFormat out_deinterleaved_format; + int i, ret, last_stage = -1; + /* check that we have a hw context */ + if (!ctx->inputs[0]->hw_frames_ctx) { + av_log(ctx, AV_LOG_ERROR, "No hw context provided on input\n"); + return AVERROR(EINVAL); + } + in_frames_ctx = (AVHWFramesContext*)ctx->inputs[0]->hw_frames_ctx->data; + in_format = in_frames_ctx->sw_format; + out_format = (s->format == AV_PIX_FMT_NONE) ? in_format : s->format; + if (!format_is_supported(in_format)) { + av_log(ctx, AV_LOG_ERROR, "Unsupported input format: %s\n", + av_get_pix_fmt_name(in_format)); + return AVERROR(ENOSYS); + } + if (!format_is_supported(out_format)) { + av_log(ctx, AV_LOG_ERROR, "Unsupported output format: %s\n", + av_get_pix_fmt_name(out_format)); + return AVERROR(ENOSYS); + } + in_deinterleaved_format = get_deinterleaved_format(in_format); + out_deinterleaved_format = get_deinterleaved_format(out_format); + if (in_deinterleaved_format == AV_PIX_FMT_NONE || + out_deinterleaved_format == AV_PIX_FMT_NONE) + return AVERROR_BUG; + if ((s->t > NPP_TRANSPOSE_NONE) && (s->t < NPP_TRANSPOSE_INVALID)) { + s->stages[STAGE_TRANSPOSE].stage_needed = 1; + } + if (in_format == out_format && !s->stages[STAGE_TRANSPOSE].stage_needed) + s->passthrough = 1; + if (!s->passthrough) { + if (in_format != in_deinterleaved_format) + s->stages[STAGE_DEINTERLEAVE].stage_needed = 1; + if (out_format != out_deinterleaved_format) + s->stages[STAGE_INTERLEAVE].stage_needed = 1; + } + s->stages[STAGE_DEINTERLEAVE].in_fmt = in_format; + s->stages[STAGE_DEINTERLEAVE].out_fmt = in_deinterleaved_format; + s->stages[STAGE_DEINTERLEAVE].planes_in[0].width = in_width; + s->stages[STAGE_DEINTERLEAVE].planes_in[0].height = in_height; + s->stages[STAGE_TRANSPOSE].in_fmt = in_deinterleaved_format; + s->stages[STAGE_TRANSPOSE].out_fmt = out_deinterleaved_format; + s->stages[STAGE_TRANSPOSE].planes_in[0].width = in_width; + s->stages[STAGE_TRANSPOSE].planes_in[0].height = in_height; + s->stages[STAGE_TRANSPOSE].planes_out[0].width = out_width; + s->stages[STAGE_TRANSPOSE].planes_out[0].height = out_height; + s->stages[STAGE_INTERLEAVE].in_fmt = out_deinterleaved_format; + s->stages[STAGE_INTERLEAVE].out_fmt = out_format; + s->stages[STAGE_INTERLEAVE].planes_in[0].width = out_width; + s->stages[STAGE_INTERLEAVE].planes_in[0].height = out_height; + /* init the hardware contexts */ + for (i = 0; i < FF_ARRAY_ELEMS(s->stages); i++) { + if (!s->stages[i].stage_needed) + continue; + ret = init_stage(&s->stages[i], in_frames_ctx->device_ref); + if (ret < 0) + return ret; + last_stage = i; + } + if (last_stage >= 0) + ctx->outputs[0]->hw_frames_ctx = av_buffer_ref(s->stages[last_stage].frames_ctx); + else + ctx->outputs[0]->hw_frames_ctx = av_buffer_ref(ctx->inputs[0]->hw_frames_ctx); + if (!ctx->outputs[0]->hw_frames_ctx) + return AVERROR(ENOMEM); + return 0; +} + +static int npptranspose_config_props(AVFilterLink *outlink) +{ + AVFilterContext *ctx = outlink->src; + AVFilterLink *inlink = outlink->src->inputs[0]; + NPPTransposeContext *s = ctx->priv; + int ret; + const char *expr; + double res; + av_expr_parse_and_eval(&res, (expr = s->t_expr), + NULL, NULL, + NULL, NULL, NULL, NULL, NULL, 0, ctx); + s->t = (int) res; + if (s->t < NPP_TRANSPOSE_NONE || s->t >= NPP_TRANSPOSE_INVALID) + { + av_log(ctx, AV_LOG_ERROR, "Invalid value for transpose: %d.\n",s->t); + goto fail; + } + // Swap width and height for 90 or 270 deg rotation + outlink->w = s->t % 2 ? inlink->h : inlink->w; + outlink->h = s->t % 2 ? inlink->w : inlink->h; + ret = init_processing_chain(ctx, inlink->w, inlink->h, outlink->w, outlink->h); + if (ret < 0) + return ret; + av_log(ctx, AV_LOG_VERBOSE, "w:%d h:%d -transpose-> w:%d h:%d\n", + inlink->w, inlink->h, outlink->w, outlink->h); + // invert aspect ratio for 90 or 270 deg rotation + if ( s->t % 2 ) + outlink->sample_aspect_ratio = (AVRational){outlink->sample_aspect_ratio.den, outlink->sample_aspect_ratio.num}; + + return 0; +fail: + return ret; +} + +static int npptranspose_deinterleave(AVFilterContext *ctx, NPPTransposeStageContext *stage, + AVFrame *out, AVFrame *in) +{ + AVHWFramesContext *in_frames_ctx = (AVHWFramesContext*)in->hw_frames_ctx->data; + NppStatus err; + switch (in_frames_ctx->sw_format) { + case AV_PIX_FMT_NV12: + err = nppiYCbCr420_8u_P2P3R(in->data[0], in->linesize[0], + in->data[1], in->linesize[1], + out->data, out->linesize, + (NppiSize){ in->width, in->height }); + break; + default: + return AVERROR_BUG; + } + if (err != NPP_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "NPP deinterleave error: %d\n", err); + return AVERROR_UNKNOWN; + } + return 0; +} + +static int npptranspose_rotate(AVFilterContext *ctx, NPPTransposeStageContext *stage, + AVFrame *out, AVFrame *in) +{ + NPPTransposeContext *s = ctx->priv; + NppStatus err; + int i; + for (i = 0; i < FF_ARRAY_ELEMS(stage->planes_in) && i < FF_ARRAY_ELEMS(in->data) && in->data[i]; i++) { + int iw = stage->planes_in[i].width; + int ih = stage->planes_in[i].height; + int ow = stage->planes_out[i].width; + int oh = stage->planes_out[i].height; + // nppRotate uses 0,0 as the rotation point + // need to shift the image accordingly after rotation + // need to substract 1 to get the correct coordinates + int shiftw = (s->t==NPP_TRANSPOSE_CW_90 || s->t==NPP_TRANSPOSE_CW_180) ? ow-1 : 0; + int shifth = (s->t==NPP_TRANSPOSE_CW_180 || s->t==NPP_TRANSPOSE_CW_270) ? oh-1 : 0; + err = nppiRotate_8u_C1R(in->data[i], (NppiSize){ iw, ih }, + in->linesize[i], (NppiRect){0,0,iw,ih}, + out->data[i], out->linesize[i], + (NppiRect){ 0, 0, ow, oh }, + s->t*-90.0, shiftw, shifth, s->interp_algo); + + if (err != NPP_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "NPP rotate error: %d\n", err); + return AVERROR_UNKNOWN; + } + } + return 0; +} + +static int npptranspose_interleave(AVFilterContext *ctx, NPPTransposeStageContext *stage, + AVFrame *out, AVFrame *in) +{ + AVHWFramesContext *out_frames_ctx = (AVHWFramesContext*)out->hw_frames_ctx->data; + NppStatus err; + switch (out_frames_ctx->sw_format) { + case AV_PIX_FMT_NV12: + err = nppiYCbCr420_8u_P3P2R((const uint8_t**)in->data, + in->linesize, + out->data[0], out->linesize[0], + out->data[1], out->linesize[1], + (NppiSize){ in->width, in->height }); + break; + default: + return AVERROR_BUG; + } + if (err != NPP_SUCCESS) { + av_log(ctx, AV_LOG_ERROR, "NPP deinterleave error: %d\n", err); + return AVERROR_UNKNOWN; + } + return 0; +} + +static int (*const npptranspose_process[])(AVFilterContext *ctx, NPPTransposeStageContext *stage, + AVFrame *out, AVFrame *in) = { + [STAGE_DEINTERLEAVE] = npptranspose_deinterleave, + [STAGE_TRANSPOSE] = npptranspose_rotate, + [STAGE_INTERLEAVE] = npptranspose_interleave, +}; + +static int npptranspose_transpose(AVFilterContext *ctx, AVFrame *out, AVFrame *in) +{ + NPPTransposeContext *s = ctx->priv; + AVFrame *src = in; + int i, ret, last_stage = -1; + for (i = 0; i < FF_ARRAY_ELEMS(s->stages); i++) { + if (!s->stages[i].stage_needed) + continue; + ret = npptranspose_process[i](ctx, &s->stages[i], s->stages[i].frame, src); + if (ret < 0) + return ret; + src = s->stages[i].frame; + last_stage = i; + } + if (last_stage < 0) + return AVERROR_BUG; + ret = av_hwframe_get_buffer(src->hw_frames_ctx, s->tmp_frame, 0); + if (ret < 0) + return ret; + av_frame_move_ref(out, src); + av_frame_move_ref(src, s->tmp_frame); + ret = av_frame_copy_props(out, in); + if (ret < 0) + return ret; + return 0; +} + +static int npptranspose_filter_frame(AVFilterLink *link, AVFrame *in) +{ + AVFilterContext *ctx = link->dst; + NPPTransposeContext *s = ctx->priv; + AVFilterLink *outlink = ctx->outputs[0]; + AVHWFramesContext *frames_ctx = (AVHWFramesContext*)outlink->hw_frames_ctx->data; + AVCUDADeviceContext *device_hwctx = frames_ctx->device_ctx->hwctx; + AVFrame *out = NULL; + CUresult err; + CUcontext dummy; + int ret = 0; + if (s->passthrough) + return ff_filter_frame(outlink, in); + out = av_frame_alloc(); + if (!out) { + ret = AVERROR(ENOMEM); + goto fail; + } + err = device_hwctx->internal->cuda_dl->cuCtxPushCurrent(device_hwctx->cuda_ctx); + if (err != CUDA_SUCCESS) { + ret = AVERROR_UNKNOWN; + goto fail; + } + ret = npptranspose_transpose(ctx, out, in); + device_hwctx->internal->cuda_dl->cuCtxPopCurrent(&dummy); + if (ret < 0) + goto fail; + av_reduce(&out->sample_aspect_ratio.num, &out->sample_aspect_ratio.den, + (int64_t)in->sample_aspect_ratio.num * outlink->h * link->w, + (int64_t)in->sample_aspect_ratio.den * outlink->w * link->h, + INT_MAX); + av_frame_free(&in); + return ff_filter_frame(outlink, out); +fail: + av_frame_free(&in); + av_frame_free(&out); + return ret; +} + +#define OFFSET(x) offsetof(NPPTransposeContext, x) +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM) + +static const AVOption options[] = { + { "t", "Output video transpose", OFFSET(t_expr), AV_OPT_TYPE_STRING, { .str = "it" }, .flags = FLAGS }, + { "format", "Output pixel format", OFFSET(format_str), AV_OPT_TYPE_STRING, { .str = "same" }, .flags = FLAGS }, + { "interp_algo", "Interpolation algorithm used for transposing", OFFSET(interp_algo), AV_OPT_TYPE_INT, { .i64 = NPPI_INTER_CUBIC }, 0, INT_MAX, FLAGS, "interp_algo" }, + { "nn", "nearest neighbour", 0, AV_OPT_TYPE_CONST, { .i64 = NPPI_INTER_NN }, 0, 0, FLAGS, "interp_algo" }, + { "linear", "linear", 0, AV_OPT_TYPE_CONST, { .i64 = NPPI_INTER_LINEAR}, 0, 0, FLAGS, "interp_algo" }, + { "cubic", "cubic", 0, AV_OPT_TYPE_CONST, { .i64 = NPPI_INTER_CUBIC }, 0, 0, FLAGS, "interp_algo" }, + { NULL }, +}; + +static const AVClass npptranspose_class = { + .class_name = "npptranspose", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; + +static const AVFilterPad npptranspose_inputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .filter_frame = npptranspose_filter_frame, + }, + { NULL } +}; + +static const AVFilterPad npptranspose_outputs[] = { + { + .name = "default", + .type = AVMEDIA_TYPE_VIDEO, + .config_props = npptranspose_config_props, + }, + { NULL } +}; + +AVFilter ff_vf_transpose_npp = { + .name = "transpose_npp", + .description = NULL_IF_CONFIG_SMALL("NVIDIA Performance Primitives video " + "transpose and format conversion"), + .init = npptranspose_init, + .uninit = npptranspose_uninit, + .query_formats = npptranspose_query_formats, + .priv_size = sizeof(NPPTransposeContext), + .priv_class = &npptranspose_class, + .inputs = npptranspose_inputs, + .outputs = npptranspose_outputs, + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, +}; \ No newline at end of file -- 2.15.1.windows.2