--- Incomplete, but enough to work with the encoder in patch 3. Todo: * Superframe splitting. * More options to control recombination. * Better error handling, especially for slightly broken streams.
libavcodec/Makefile | 1 + libavcodec/bitstream_filters.c | 1 + libavcodec/vp9_recombine_bsf.c | 557 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 559 insertions(+) create mode 100644 libavcodec/vp9_recombine_bsf.c diff --git a/libavcodec/Makefile b/libavcodec/Makefile index f0752a2..28cea1b 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -760,6 +760,7 @@ OBJS-$(CONFIG_MOV2TEXTSUB_BSF) += movsub_bsf.o OBJS-$(CONFIG_NOISE_BSF) += noise_bsf.o OBJS-$(CONFIG_REMOVE_EXTRADATA_BSF) += remove_extradata_bsf.o OBJS-$(CONFIG_TEXT2MOVSUB_BSF) += movsub_bsf.o +OBJS-$(CONFIG_VP9_RECOMBINE_BSF) += vp9_recombine_bsf.o # thread libraries OBJS-$(HAVE_LIBC_MSVCRT) += file_open.o diff --git a/libavcodec/bitstream_filters.c b/libavcodec/bitstream_filters.c index 8a5379e..20ab281 100644 --- a/libavcodec/bitstream_filters.c +++ b/libavcodec/bitstream_filters.c @@ -37,6 +37,7 @@ extern const AVBitStreamFilter ff_mov2textsub_bsf; extern const AVBitStreamFilter ff_text2movsub_bsf; extern const AVBitStreamFilter ff_noise_bsf; extern const AVBitStreamFilter ff_remove_extradata_bsf; +extern const AVBitStreamFilter ff_vp9_recombine_bsf; #include "libavcodec/bsf_list.c" diff --git a/libavcodec/vp9_recombine_bsf.c b/libavcodec/vp9_recombine_bsf.c new file mode 100644 index 0000000..cd5b968 --- /dev/null +++ b/libavcodec/vp9_recombine_bsf.c @@ -0,0 +1,557 @@ +/* + * This file is part of Libav. + * + * Libav is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * Libav is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with Libav; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "libavutil/avassert.h" +#include "libavutil/intmath.h" +#include "libavutil/log.h" +#include "libavutil/mem.h" +#include "libavutil/opt.h" + +#include "bitstream.h" +#include "bsf.h" +#include "put_bits.h" + +#define MAX_BUFFERING 16 + +typedef struct VP9RecombineFrame { + AVPacket *packet; + int output; + int64_t pts; + unsigned int slots; + + unsigned int profile; + + unsigned int show_existing_frame; + unsigned int frame_to_show; + + unsigned int frame_type; + unsigned int show_frame; + unsigned int refresh_frame_flags; +} VP9RecombineFrame; + +typedef struct VP9RecombineContext { + AVClass *class; + + int implicit_show; + int super_only; + + int nb_frames; + VP9RecombineFrame *frames[MAX_BUFFERING]; + + VP9RecombineFrame *next_frame; +} VP9RecombineContext; + +static void vp9_recombine_frame_free(VP9RecombineFrame *frame) +{ + av_packet_free(&frame->packet); + av_free(frame); +} + +static int vp9_recombine_frame_parse(AVBSFContext *bsf, VP9RecombineFrame *frame) +{ + BitstreamContext bc; + int err; + + unsigned int frame_marker; + unsigned int profile_low_bit, profile_high_bit, reserved_zero; + unsigned int error_resilient_mode; + unsigned int frame_sync_code; + + err = bitstream_init8(&bc, frame->packet->data, frame->packet->size); + if (err) + return err; + + frame_marker = bitstream_read(&bc, 2); + if (frame_marker != 2) { + av_log(bsf, AV_LOG_ERROR, "Invalid frame marker: %u.\n", + frame_marker); + return AVERROR_INVALIDDATA; + } + + profile_low_bit = bitstream_read_bit(&bc); + profile_high_bit = bitstream_read_bit(&bc); + frame->profile = (profile_high_bit << 1) | profile_low_bit; + if (frame->profile == 3) { + reserved_zero = bitstream_read_bit(&bc); + if (reserved_zero != 0) { + av_log(bsf, AV_LOG_ERROR, "Profile reserved_zero bit set: " + "unsupported profile or invalid bitstream.\n"); + return AVERROR_INVALIDDATA; + } + } + + frame->show_existing_frame = bitstream_read_bit(&bc); + if (frame->show_existing_frame) { + frame->frame_to_show = bitstream_read(&bc, 3); + return 0; + } + frame->frame_type = bitstream_read_bit(&bc); + frame->show_frame = bitstream_read_bit(&bc); + error_resilient_mode = bitstream_read_bit(&bc); + + if (frame->frame_type == 0) { + frame_sync_code = bitstream_read(&bc, 24); + if (frame_sync_code != 0x498342) { + av_log(bsf, AV_LOG_ERROR, "Invalid frame sync code: %06x.\n", + frame_sync_code); + return AVERROR_INVALIDDATA; + } + frame->refresh_frame_flags = 0xff; + } else { + unsigned int intra_only; + + if (frame->show_frame == 0) + intra_only = bitstream_read_bit(&bc); + else + intra_only = 0; + if (error_resilient_mode == 0) { + // reset_frame_context + bitstream_skip(&bc, 2); + } + if (intra_only) { + frame_sync_code = bitstream_read(&bc, 24); + if (frame_sync_code != 0x498342) { + av_log(bsf, AV_LOG_ERROR, "Invalid frame sync code: %06x.\n", + frame_sync_code); + return AVERROR_INVALIDDATA; + } + if (frame->profile > 0) { + unsigned int color_space; + if (frame->profile >= 2) { + // ten_or_twelve_bit + bitstream_skip(&bc, 1); + } + color_space = bitstream_read(&bc, 3); + if (color_space != 7 /* CS_RGB */) { + // color_range + bitstream_skip(&bc, 1); + if (frame->profile == 1 || frame->profile == 3) { + // subsampling + bitstream_skip(&bc, 3); + } + } else { + if (frame->profile == 1 || frame->profile == 3) + bitstream_skip(&bc, 1); + } + } + frame->refresh_frame_flags = bitstream_read(&bc, 8); + } else { + frame->refresh_frame_flags = bitstream_read(&bc, 8); + } + } + + if (frame->show_existing_frame) + av_log(bsf, AV_LOG_DEBUG, "Old frame: index %u pts %"PRId64 + ".\n", frame->frame_to_show, frame->pts); + else + av_log(bsf, AV_LOG_DEBUG, "New frame: type %u show %u " + "refresh %02x pts %"PRId64".\n", + frame->frame_type, frame->show_frame, + frame->refresh_frame_flags, frame->pts); + + return 0; +} + +static int vp9_recombine_make_superframe(AVBSFContext *bsf, AVPacket *out, + VP9RecombineFrame **frames, int count) +{ + PutBitContext pb; + size_t size = 0, pos; + unsigned int bytes = 1; + size_t frame_size, index_size; + int err, i; + + av_assert0(count > 0 && count <= 8); + + for (i = 0; i < count; i++) { + av_assert0(frames[i]->packet); + + frame_size = frames[i]->packet->size; + size += frame_size; + + while (bytes < 4 && frame_size > 1 << 8 * bytes) + ++bytes; + } + index_size = 2 + bytes * count; + size += index_size; + + err = av_new_packet(out, size); + if (err < 0) + return err; + + out->dts = out->pts = frames[count - 1]->pts; + + init_put_bits(&pb, out->data + size - index_size, index_size); + + // superframe_marker + put_bits(&pb, 3, 6); + // bytes_per_framesize_minus_1 + put_bits(&pb, 2, bytes - 1); + // frames_in_superframe_minus_1 + put_bits(&pb, 3, count - 1); + + pos = 0; + for (i = 0; i < count; i++) { + frame_size = frames[i]->packet->size; + av_log(bsf, AV_LOG_DEBUG, "Combine frame of %zu bytes.\n", + frame_size); + + memcpy(out->data + pos, frames[i]->packet->data, frame_size); + pos += frame_size; + + av_packet_free(&frames[i]->packet); + + // These bytes are written in little-endian order. + switch (bytes) { + case 4: + put_bits(&pb, 8, frame_size & 0xff); + frame_size >>= 8; + case 3: + put_bits(&pb, 8, frame_size & 0xff); + frame_size >>= 8; + case 2: + put_bits(&pb, 8, frame_size & 0xff); + frame_size >>= 8; + case 1: + put_bits(&pb, 8, frame_size & 0xff); + break; + default: + av_assert0(0); + } + } + + // superframe_marker + put_bits(&pb, 3, 6); + // bytes_per_framesize_minus_1 + put_bits(&pb, 2, bytes - 1); + // frames_in_superframe_minus_1 + put_bits(&pb, 3, count - 1); + + flush_put_bits(&pb); + + av_log(bsf, AV_LOG_DEBUG, "Made superframe of %zu bytes from " + "%d frames.\n", size, count); + + return 0; +} + +static int vp9_recombine_make_output(AVBSFContext *bsf, + AVPacket *out, + int64_t next_pts) +{ + VP9RecombineContext *ctx = bsf->priv_data; + VP9RecombineFrame *frame; + int i, err; + + if (ctx->nb_frames == 0) + return AVERROR(EAGAIN); + + // Find the frame with the lowest pts amongst the frames we + // haven't yet sent and try to send it. + frame = NULL; + for (i = 0; i < ctx->nb_frames; i++) { + if (ctx->frames[i]->output) + continue; + if (!frame || ctx->frames[i]->pts < frame->pts) + frame = ctx->frames[i]; + } + + if (!frame) { + //av_log(bsf, AV_LOG_DEBUG, "No frames ready yet.\n"); + return AVERROR(EAGAIN); + } + + if (frame->show_existing_frame) { + VP9RecombineFrame *old; + int i; + + av_log(bsf, AV_LOG_DEBUG, "Explicit show existing frame " + "from slot %d (%"PRId64").\n", frame->frame_to_show, + frame->pts); + + for (i = 0; i < ctx->nb_frames; i++) { + old = ctx->frames[i]; + if (old->slots & (1 << frame->frame_to_show)) + break; + } + if (i == ctx->nb_frames) { + av_log(bsf, AV_LOG_ERROR, "Attempting to show frame " + "which does not exist.\n"); + // Broken bitstream? We could just pass this through. + return AVERROR_INVALIDDATA; + } + if (old->packet) { + av_log(bsf, AV_LOG_ERROR, "Attempting to show frame " + "which has not yet been output.\n"); + // This could work (edit the packet to set show_frame), + // but shouldn't occur in any real case so don't bother. + return AVERROR_INVALIDDATA; + } + + if (ctx->super_only) { + err = vp9_recombine_make_superframe(bsf, out, &frame, 1); + if (err < 0) + return err; + } else { + av_packet_move_ref(out, frame->packet); + } + + frame->packet = NULL; + frame->output = 1; + + } else if (frame->show_frame) { + int count; + + av_assert0(frame->packet); + count = 0; + for (i = 0; i < ctx->nb_frames; i++) { + if (ctx->frames[i]->packet) + ++count; + if (ctx->frames[i] == frame) + break; + } + av_assert0(i < ctx->nb_frames); + + if (count > 8) { + av_log(bsf, AV_LOG_ERROR, "Attempting to make superframe " + "out of too many frames? (%d)\n", count); + return AVERROR_INVALIDDATA; + } + + if (count > 1) + av_log(bsf, AV_LOG_DEBUG, "Show frame as %d combined " + "frames (%"PRId64").\n", count, frame->pts); + else + av_log(bsf, AV_LOG_DEBUG, "Show frame as singleton " + "(%"PRId64").\n", frame->pts); + + if (count > 1 || ctx->super_only) { + VP9RecombineFrame *frame_list[8]; + int k; + + for (i = k = 0; i < ctx->nb_frames; i++) { + if (!ctx->frames[i]->packet) + continue; + frame_list[k++] = ctx->frames[i]; + if (ctx->frames[i] == frame) + break; + } + av_assert0(k == count); + + err = vp9_recombine_make_superframe(bsf, out, + frame_list, count); + if (err < 0) + return err; + + frame->output = 1; + } else { + av_packet_move_ref(out, frame->packet); + frame->packet = NULL; + frame->output = 1; + } + + } else if (ctx->implicit_show) { + PutBitContext pb; + int slot; + + if (frame->pts > next_pts) { + av_log(bsf, AV_LOG_DEBUG, "Waiting with existing " + "unshown frame (%"PRId64").\n", frame->pts); + return AVERROR(EAGAIN); + } + + if (!frame->slots) { + av_log(bsf, AV_LOG_ERROR, "Trying to show frame which " + "no longer exists?\n"); + return AVERROR_INVALIDDATA; + } + slot = ff_ctz(frame->slots); + av_assert0(slot < 8); + + av_log(bsf, AV_LOG_DEBUG, "Implicit show existing frame " + "from slot %d (%"PRId64").\n", slot, frame->pts); + + frame->packet = av_packet_alloc(); + if (!frame->packet) + return AVERROR(ENOMEM); + + err = av_new_packet(frame->packet, 2); + if (err < 0) + return err; + + init_put_bits(&pb, frame->packet->data, 2); + + // frame_marker + put_bits(&pb, 2, 2); + // profile_low_bit + put_bits(&pb, 1, frame->profile & 1); + // profile_high_bit + put_bits(&pb, 1, (frame->profile >> 1) & 1); + if (frame->profile == 3) { + // reserved_zero + put_bits(&pb, 1, 0); + } + // show_existing_frame + put_bits(&pb, 1, 1); + // frame_to_show_map_idx + put_bits(&pb, 3, slot); + + while (put_bits_count(&pb) < 16) + put_bits(&pb, 1, 0); + + flush_put_bits(&pb); + + err = vp9_recombine_make_superframe(bsf, out, &frame, 1); + if (err < 0) + return err; + + frame->packet = NULL; + frame->output = 1; + + } else { + av_log(bsf, AV_LOG_DEBUG, "Skipping existing unshown frame " + "(%"PRId64").\n", frame->pts); + frame->output = 1; + + return vp9_recombine_make_output(bsf, out, next_pts); + } + + return 0; +} + +static int vp9_recombine_filter(AVBSFContext *bsf, AVPacket *out) +{ + VP9RecombineContext *ctx = bsf->priv_data; + VP9RecombineFrame *frame; + AVPacket *in; + int err; + + err = vp9_recombine_make_output(bsf, out, INT64_MIN); + if (err != AVERROR(EAGAIN)) + return err; + + if (ctx->next_frame) { + frame = ctx->next_frame; + + } else { + err = ff_bsf_get_packet(bsf, &in); + if (err < 0) { + if (err == AVERROR_EOF) + return vp9_recombine_make_output(bsf, out, INT64_MAX); + return err; + } + + if (in->pts == AV_NOPTS_VALUE) { + av_log(bsf, AV_LOG_ERROR, "Timestamps are not set.\n"); + return AVERROR_INVALIDDATA; + } + + if (ctx->nb_frames == MAX_BUFFERING) { + av_log(bsf, AV_LOG_ERROR, "Too much buffering required.\n"); + return AVERROR_INVALIDDATA; + } + + frame = av_mallocz(sizeof(*frame)); + if (!frame) + return AVERROR(ENOMEM); + + frame->packet = in; + frame->pts = in->pts; + err = vp9_recombine_frame_parse(bsf, frame); + if (err) { + av_log(bsf, AV_LOG_ERROR, "Failed to parse input frame.\n"); + goto fail; + } + + ctx->next_frame = frame; + } + + if (frame->show_frame) { + // Since this frame is immediately visible, we want to + // output all previous frames up to this point. + err = vp9_recombine_make_output(bsf, out, frame->pts); + if (err != AVERROR(EAGAIN)) + return err; + } + + if (!frame->show_existing_frame) { + int i, j; + for (i = 0; i < ctx->nb_frames; i++) + ctx->frames[i]->slots &= ~frame->refresh_frame_flags; + frame->slots = frame->refresh_frame_flags; + + for (i = j = 0; i < ctx->nb_frames; i++) { + if (!ctx->frames[i]->packet && + ctx->frames[i]->output && + !ctx->frames[i]->slots) { + av_log(bsf, AV_LOG_DEBUG, "Free frame (%"PRId64").\n", + ctx->frames[i]->pts); + vp9_recombine_frame_free(ctx->frames[i]); + } else { + ctx->frames[j++] = ctx->frames[i]; + } + } + ctx->nb_frames = j; + } + ctx->frames[ctx->nb_frames++] = frame; + ctx->next_frame = NULL; + + return vp9_recombine_make_output(bsf, out, INT64_MIN); + +fail: + vp9_recombine_frame_free(frame); + return err; +} + +static void vp9_recombine_close(AVBSFContext *bsf) +{ + VP9RecombineContext *ctx = bsf->priv_data; + int i; + + for (i = 0; i < ctx->nb_frames; i++) + vp9_recombine_frame_free(ctx->frames[i]); +} + +#define OFFSET(x) offsetof(VP9RecombineContext, x) +static const AVOption vp9_recombine_options[] = { + { "implicit_show", "Add implicit show_existing_frame packets for reordered frames", + OFFSET(implicit_show), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1 }, + { "super_only", "Output superframes only", + OFFSET(super_only), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1 }, + { NULL } +}; + +static const enum AVCodecID vp9_recombine_codec_ids[] = { + AV_CODEC_ID_VP9, AV_CODEC_ID_NONE, +}; + +static const AVClass vp9_recombine_class = { + .class_name = "vp9_recombine bsf", + .item_name = av_default_item_name, + .option = vp9_recombine_options, + .version = LIBAVUTIL_VERSION_MAJOR, +}; + +const AVBitStreamFilter ff_vp9_recombine_bsf = { + .name = "vp9_recombine", + .priv_data_size = sizeof(VP9RecombineContext), + .priv_class = &vp9_recombine_class, + .close = &vp9_recombine_close, + .filter = &vp9_recombine_filter, + .codec_ids = vp9_recombine_codec_ids, +}; -- 2.10.2 _______________________________________________ libav-devel mailing list libav-devel@libav.org https://lists.libav.org/mailman/listinfo/libav-devel