Add parsing of subset_seq_parameter_set_rbsp() (H.7.3.2.1.3) and the seq_parameter_set_mvc_extension() it carries (H.7.3.2.1.4), which is where an MVC stream declares its views and the inter-view dependency lists later used to build the inter-view reference picture lists.
Subset SPSs live in an id space of their own, separate from that of ordinary SPSs (H.7.4.1.2.1), so they get a separate list. A PPS referenced by a coded slice extension refers to a subset SPS, but nothing in the PPS itself says which space is meant, so resolve it against both and let the slice header pick by NAL type. This also stops a PPS pointing at a subset SPS from being rejected outright, which previously spammed "sps_id N out of range" on every MVC stream. Only the MVC extension is handled; SVC (Annex G) and MVCD/3D-AVC subset SPSs are stored but their extensions are not parsed. Streams declaring more views than are supported are likewise stored without a usable view list, so that base view decoding is unaffected either way. Also add H264_NAL_SUB_SPS to the extract_extradata bsf, which would otherwise drop it and leave the decoder unable to resolve a dependent view PPS arriving via extradata. Signed-off-by: Dom Cobley <[email protected]> --- libavcodec/bsf/extract_extradata.c | 2 +- libavcodec/h264_parse.c | 4 + libavcodec/h264_parser.c | 3 + libavcodec/h264_ps.c | 157 +++++++++++++++++++++++++++-- libavcodec/h264_ps.h | 57 +++++++++++ libavcodec/h264_slice.c | 4 +- libavcodec/h264dec.c | 5 + 7 files changed, 222 insertions(+), 10 deletions(-) diff --git a/libavcodec/bsf/extract_extradata.c b/libavcodec/bsf/extract_extradata.c index 3c66f392b2..c6220bd7a2 100644 --- a/libavcodec/bsf/extract_extradata.c +++ b/libavcodec/bsf/extract_extradata.c @@ -173,7 +173,7 @@ static int extract_extradata_h2645(AVBSFContext *ctx, AVPacket *pkt, HEVC_NAL_VPS, HEVC_NAL_SPS, HEVC_NAL_PPS, }; static const int extradata_nal_types_h264[] = { - H264_NAL_SPS, H264_NAL_PPS, + H264_NAL_SPS, H264_NAL_SUB_SPS, H264_NAL_PPS, }; ExtractExtradataContext *s = ctx->priv_data; diff --git a/libavcodec/h264_parse.c b/libavcodec/h264_parse.c index c6f9b3cea7..d472d9a86c 100644 --- a/libavcodec/h264_parse.c +++ b/libavcodec/h264_parse.c @@ -396,6 +396,10 @@ static int decode_extradata_ps(const uint8_t *data, int size, H264ParamSets *ps, goto fail; break; } + case H264_NAL_SUB_SPS: + /* Only affects multiview decoding, so a failure is not fatal. */ + ff_h264_decode_subset_seq_parameter_set(&nal->gb, logctx, ps); + break; case H264_NAL_PPS: ret = ff_h264_decode_picture_parameter_set(&nal->gb, logctx, ps, nal->size_bits); diff --git a/libavcodec/h264_parser.c b/libavcodec/h264_parser.c index af43cad609..1b9cf9fe36 100644 --- a/libavcodec/h264_parser.c +++ b/libavcodec/h264_parser.c @@ -344,6 +344,9 @@ static inline int parse_nal_units(AVCodecParserContext *s, case H264_NAL_SPS: ff_h264_decode_seq_parameter_set(&nal.gb, avctx, &p->ps, 0); break; + case H264_NAL_SUB_SPS: + ff_h264_decode_subset_seq_parameter_set(&nal.gb, avctx, &p->ps); + break; case H264_NAL_PPS: ff_h264_decode_picture_parameter_set(&nal.gb, avctx, &p->ps, nal.size_bits); diff --git a/libavcodec/h264_ps.c b/libavcodec/h264_ps.c index d59fd2667e..269702c0bc 100644 --- a/libavcodec/h264_ps.c +++ b/libavcodec/h264_ps.c @@ -271,8 +271,10 @@ void ff_h264_ps_uninit(H264ParamSets *ps) { int i; - for (i = 0; i < MAX_SPS_COUNT; i++) + for (i = 0; i < MAX_SPS_COUNT; i++) { av_refstruct_unref(&ps->sps_list[i]); + av_refstruct_unref(&ps->subset_sps_list[i]); + } for (i = 0; i < MAX_PPS_COUNT; i++) av_refstruct_unref(&ps->pps_list[i]); @@ -281,8 +283,103 @@ void ff_h264_ps_uninit(H264ParamSets *ps) ps->sps = NULL; } -int ff_h264_decode_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, - H264ParamSets *ps, int ignore_truncation) +/** + * Parse seq_parameter_set_mvc_extension(), H.7.3.2.1.4. + * + * On success sps->mvc.num_views is the number of views; on unsupported input it + * is left at 0 and 0 is still returned, so that the subset SPS remains usable + * for base-view-only decoding. + */ +static int decode_sps_mvc_extension(GetBitContext *gb, AVCodecContext *avctx, + SPS *sps) +{ + SPSMVCExt *mvc = &sps->mvc; + unsigned num_views, num_level_values; + + num_views = get_ue_golomb_long(gb) + 1; + if (num_views > H264_MAX_MVC_VIEWS) { + avpriv_report_missing_feature(avctx, "MVC with %u views", num_views); + return 0; + } + + for (int i = 0; i < num_views; i++) { + unsigned view_id = get_ue_golomb_long(gb); + if (view_id > 1023) { + av_log(avctx, AV_LOG_ERROR, "Invalid view_id %u\n", view_id); + return AVERROR_INVALIDDATA; + } + mvc->view_id[i] = view_id; + } + + /* The anchor and non-anchor inter-view dependency lists. Note l0 and l1 are + * interleaved within each iteration of i. */ + for (int i = 1; i < num_views; i++) { + for (int list = 0; list < 2; list++) { + unsigned n = get_ue_golomb_31(gb); + if (n >= H264_MAX_MVC_REFS) { + av_log(avctx, AV_LOG_ERROR, + "Invalid num_anchor_refs_l%d[%d] %u\n", list, i, n); + return AVERROR_INVALIDDATA; + } + mvc->num_anchor_refs[list][i] = n; + for (int j = 0; j < n; j++) + mvc->anchor_ref[list][i][j] = get_ue_golomb_long(gb); + } + } + for (int i = 1; i < num_views; i++) { + for (int list = 0; list < 2; list++) { + unsigned n = get_ue_golomb_31(gb); + if (n >= H264_MAX_MVC_REFS) { + av_log(avctx, AV_LOG_ERROR, + "Invalid num_non_anchor_refs_l%d[%d] %u\n", list, i, n); + return AVERROR_INVALIDDATA; + } + mvc->num_non_anchor_refs[list][i] = n; + for (int j = 0; j < n; j++) + mvc->non_anchor_ref[list][i][j] = get_ue_golomb_long(gb); + } + } + + /* The level / operation point table is parsed only to consume it; we have no + * use for the operation points. */ + num_level_values = get_ue_golomb_31(gb) + 1; + for (int i = 0; i < num_level_values; i++) { + unsigned num_ops; + + skip_bits(gb, 8); // level_idc + num_ops = get_ue_golomb_long(gb) + 1; + for (int j = 0; j < num_ops; j++) { + unsigned num_target_views; + + skip_bits(gb, 3); // applicable_op_temporal_id + num_target_views = get_ue_golomb_long(gb) + 1; + for (int k = 0; k < num_target_views; k++) + get_ue_golomb_long(gb); // applicable_op_target_view_id + get_ue_golomb_long(gb); // applicable_op_num_views_minus1 + } + if (get_bits_left(gb) < 0) { + av_log(avctx, AV_LOG_ERROR, "Overread MVC SPS extension\n"); + return AVERROR_INVALIDDATA; + } + } + + mvc->num_views = num_views; + + if (avctx->debug & FF_DEBUG_PICT_INFO) { + for (int i = 0; i < num_views; i++) + av_log(avctx, AV_LOG_DEBUG, + "mvc: voidx:%d view_id:%u anchor_refs:%d/%d non_anchor_refs:%d/%d\n", + i, mvc->view_id[i], + mvc->num_anchor_refs[0][i], mvc->num_anchor_refs[1][i], + mvc->num_non_anchor_refs[0][i], mvc->num_non_anchor_refs[1][i]); + } + + return 0; +} + +static int decode_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, + H264ParamSets *ps, int ignore_truncation, + int subset) { int profile_idc, level_idc, constraint_set_flags = 0; unsigned int sps_id; @@ -532,6 +629,21 @@ int ff_h264_decode_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, goto fail; } + /* The remainder of subset_seq_parameter_set_rbsp(), H.7.3.2.1.3. Only the + * MVC extension is handled; SVC (profile 83/86) and MVCD/3D-AVC (138) are + * parsed no further, which just leaves mvc.num_views at 0. */ + sps->is_subset = subset; + if (subset && (sps->profile_idc == 118 || sps->profile_idc == 128)) { + if (!get_bits1(gb)) { // bit_equal_to_one + av_log(avctx, AV_LOG_ERROR, + "Invalid subset SPS: bit_equal_to_one is 0\n"); + goto fail; + } + ret = decode_sps_mvc_extension(gb, avctx, sps); + if (ret < 0) + goto fail; + } + if (get_bits_left(gb) < 0) { av_log_once(avctx, ignore_truncation ? AV_LOG_WARNING : AV_LOG_ERROR, AV_LOG_DEBUG, &ps->overread_warning_printed[sps->vui_parameters_present_flag], @@ -578,8 +690,16 @@ int ff_h264_decode_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, /* check if this is a repeat of an already parsed SPS, then keep the * original one. * otherwise drop all PPSes that depend on it */ - if (ps->sps_list[sps_id] && - !memcmp(ps->sps_list[sps_id], sps, sizeof(*sps))) { + if (subset) { + if (ps->subset_sps_list[sps_id] && + !memcmp(ps->subset_sps_list[sps_id], sps, sizeof(*sps))) { + av_refstruct_unref(&sps); + } else { + av_refstruct_unref(&ps->subset_sps_list[sps_id]); + ps->subset_sps_list[sps_id] = sps; + } + } else if (ps->sps_list[sps_id] && + !memcmp(ps->sps_list[sps_id], sps, sizeof(*sps))) { av_refstruct_unref(&sps); } else { remove_sps(ps, sps_id); @@ -593,6 +713,18 @@ fail: return AVERROR_INVALIDDATA; } +int ff_h264_decode_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, + H264ParamSets *ps, int ignore_truncation) +{ + return decode_seq_parameter_set(gb, avctx, ps, ignore_truncation, 0); +} + +int ff_h264_decode_subset_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, + H264ParamSets *ps) +{ + return decode_seq_parameter_set(gb, avctx, ps, 0, 1); +} + static void init_dequant8_coeff_table(PPS *pps, const SPS *sps) { int i, j, q, x; @@ -693,6 +825,7 @@ static void pps_free(AVRefStructOpaque unused, void *obj) PPS *pps = obj; av_refstruct_unref(&pps->sps); + av_refstruct_unref(&pps->sps_mvc); } int ff_h264_decode_picture_parameter_set(GetBitContext *gb, AVCodecContext *avctx, @@ -729,14 +862,22 @@ int ff_h264_decode_picture_parameter_set(GetBitContext *gb, AVCodecContext *avct pps->pps_id = pps_id; pps->sps_id = get_ue_golomb_31(gb); + /* A PPS referenced by a coded slice extension (NAL 20) refers to a subset + * SPS, which lives in its own id space. As the two spaces are disjoint in + * practice we cannot tell from the PPS alone which one is meant, so resolve + * against both and let the slice header pick by NAL type. */ if ((unsigned)pps->sps_id >= MAX_SPS_COUNT || - !ps->sps_list[pps->sps_id]) { + (!ps->sps_list[pps->sps_id] && !ps->subset_sps_list[pps->sps_id])) { av_log(avctx, AV_LOG_ERROR, "sps_id %u out of range\n", pps->sps_id); ret = AVERROR_INVALIDDATA; goto fail; } - pps->sps = av_refstruct_ref_c(ps->sps_list[pps->sps_id]); - sps = pps->sps; + /* av_refstruct_replace() rather than av_refstruct_ref_c(), as either list + * entry may be NULL here. */ + av_refstruct_replace(&pps->sps_mvc, ps->subset_sps_list[pps->sps_id]); + av_refstruct_replace(&pps->sps, ps->sps_list[pps->sps_id] ? + ps->sps_list[pps->sps_id] : pps->sps_mvc); + sps = pps->sps; if (sps->bit_depth_luma > 14) { av_log(avctx, AV_LOG_ERROR, diff --git a/libavcodec/h264_ps.h b/libavcodec/h264_ps.h index f216e4989f..41d776a3a6 100644 --- a/libavcodec/h264_ps.h +++ b/libavcodec/h264_ps.h @@ -38,6 +38,36 @@ #define MAX_PPS_COUNT 256 #define MAX_LOG2_MAX_FRAME_NUM (12 + 4) +/** + * Maximum number of MVC views we support decoding. The syntax allows up to + * 1024; as for MV-HEVC only stereo is implemented. + */ +#define H264_MAX_MVC_VIEWS 2 +/** num_anchor_refs_lX / num_non_anchor_refs_lX are in [0, 15], cf. H.7.4.2.1.4 */ +#define H264_MAX_MVC_REFS 16 + +/** + * MVC extension of the subset sequence parameter set, cf. H.7.3.2.1.4 + * seq_parameter_set_mvc_extension(). + * + * Arrays documented as [voidx] are indexed by view order index (VOIdx), i.e. + * the position of a view in view_id[]. The inter-view reference arrays hold + * view_id values, not VOIdx values, and are only meaningful for voidx >= 1. + */ +typedef struct SPSMVCExt { + /** + * num_views_minus1 + 1, or 0 if this subset SPS carries no usable MVC + * extension (absent, or more views than we support). + */ + int num_views; + uint16_t view_id[H264_MAX_MVC_VIEWS]; + + uint8_t num_anchor_refs [2][H264_MAX_MVC_VIEWS]; + uint16_t anchor_ref [2][H264_MAX_MVC_VIEWS][H264_MAX_MVC_REFS]; + uint8_t num_non_anchor_refs[2][H264_MAX_MVC_VIEWS]; + uint16_t non_anchor_ref [2][H264_MAX_MVC_VIEWS][H264_MAX_MVC_REFS]; +} SPSMVCExt; + /** * Sequence parameter set */ @@ -102,6 +132,9 @@ typedef struct SPS { int constraint_set_flags; ///< constraint_set[0-3]_flag uint8_t data[4096]; size_t data_size; + + int is_subset; ///< came from a NAL_SUB_SPS (H.7.3.2.1.3) + SPSMVCExt mvc; ///< only valid when is_subset is set } SPS; /** @@ -139,10 +172,24 @@ typedef struct PPS { uint32_t(*dequant8_coeff[6])[64]; const SPS *sps; ///< RefStruct reference + + /** + * RefStruct reference to the subset SPS with the same seq_parameter_set_id, + * if one exists, else NULL. Slices in a coded slice extension (NAL 20) + * activate this instead of ::sps. + * + * The derived tables above (chroma_qp_table, dequant*_coeff) are built from + * ::sps; a subset SPS is required to agree with the base SPS on bit depth + * and chroma format for multiview decoding to be attempted, so they are + * valid for both. + */ + const SPS *sps_mvc; ///< RefStruct reference } PPS; typedef struct H264ParamSets { const SPS *sps_list[MAX_SPS_COUNT]; ///< RefStruct references + /** subset SPSs (NAL 15); a separate id space from sps_list, cf. H.7.4.1.2.1 */ + const SPS *subset_sps_list[MAX_SPS_COUNT]; ///< RefStruct references const PPS *pps_list[MAX_PPS_COUNT]; ///< RefStruct references /* currently active parameters sets */ @@ -163,6 +210,16 @@ int ff_h264_get_profile(const SPS *sps); int ff_h264_decode_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, H264ParamSets *ps, int ignore_truncation); +/** + * Decode a subset SPS (NAL_SUB_SPS), cf. H.7.3.2.1.3 + * subset_seq_parameter_set_rbsp(). + * + * The result is stored in ps->subset_sps_list, which is a separate id space + * from ps->sps_list. + */ +int ff_h264_decode_subset_seq_parameter_set(GetBitContext *gb, AVCodecContext *avctx, + H264ParamSets *ps); + /** * Decode PPS */ diff --git a/libavcodec/h264_slice.c b/libavcodec/h264_slice.c index c96d99d242..aeddf49db4 100644 --- a/libavcodec/h264_slice.c +++ b/libavcodec/h264_slice.c @@ -365,8 +365,10 @@ int ff_h264_update_thread_context(AVCodecContext *dst, memcpy(h->block_offset, h1->block_offset, sizeof(h->block_offset)); // SPS/PPS - for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.sps_list); i++) + for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.sps_list); i++) { av_refstruct_replace(&h->ps.sps_list[i], h1->ps.sps_list[i]); + av_refstruct_replace(&h->ps.subset_sps_list[i], h1->ps.subset_sps_list[i]); + } for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.pps_list); i++) av_refstruct_replace(&h->ps.pps_list[i], h1->ps.pps_list[i]); diff --git a/libavcodec/h264dec.c b/libavcodec/h264dec.c index b78b7989ea..8636001270 100644 --- a/libavcodec/h264dec.c +++ b/libavcodec/h264dec.c @@ -714,6 +714,11 @@ static int decode_nal_units(H264Context *h, AVBufferRef *buf_ref, ff_h264_decode_seq_parameter_set(&nal->gb, avctx, &h->ps, 1); break; } + case H264_NAL_SUB_SPS: + /* Failure here only means multiview decoding will not be offered; + * the base view is unaffected, so do not propagate the error. */ + ff_h264_decode_subset_seq_parameter_set(&nal->gb, avctx, &h->ps); + break; case H264_NAL_PPS: if (FF_HW_HAS_CB(avctx, decode_params)) { ret = FF_HW_CALL(avctx, decode_params, -- 2.53.0 _______________________________________________ ffmpeg-devel mailing list -- [email protected] To unsubscribe send an email to [email protected]
