From e7ff03ed52c709d647a112833427b44c41e3ed12 Mon Sep 17 00:00:00 2001 From: "Ronak Patel (Audible)" Date: Tue, 31 Jul 2018 19:05:18 -0400 Subject: [PATCH] libavformat/hlsenc: Fix HLS Manifest Generation from an N^2 algorithm to N. This fixes the creation of the hls manifest in hlsenc.c by writing the header once in hls_write_header, and the segments individually in the hls_window method. Files that would previously take over a week to fragment now take 1 minute on the same hardware. This was a 153 hour audio file (2.2GB of audio). Signed-off-by: Ronak Patel --- libavformat/hlsenc.c | 143 +++++++++++++++++++++++++++------------------------ 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/libavformat/hlsenc.c b/libavformat/hlsenc.c index b5644f0..b15645d 100644 --- a/libavformat/hlsenc.c +++ b/libavformat/hlsenc.c @@ -488,7 +488,6 @@ static int hls_delete_old_segments(AVFormatContext *s, HLSContext *hls, } p = (char *)av_basename(dirname); *p = '\0'; - } while (segment) { @@ -1365,63 +1364,37 @@ fail: return ret; } -static int hls_window(AVFormatContext *s, int last, VariantStream *vs) +static int hls_write_manifest_segment(AVFormatContext *s, int last, VariantStream *vs) { HLSContext *hls = s->priv_data; HLSSegment *en; - int target_duration = 0; int ret = 0; - char temp_filename[1024]; - int64_t sequence = FFMAX(hls->start_sequence, vs->sequence - vs->nb_entries); - const char *proto = avio_find_protocol_name(s->url); - int use_rename = proto && !strcmp(proto, "file"); - static unsigned warned_non_file; + int byterange_mode; char *key_uri = NULL; char *iv_string = NULL; AVDictionary *options = NULL; double prog_date_time = vs->initial_prog_date_time; double *prog_date_time_p = (hls->flags & HLS_PROGRAM_DATE_TIME) ? &prog_date_time : NULL; - int byterange_mode = (hls->flags & HLS_SINGLE_FILE) || (hls->max_seg_size > 0); - - hls->version = 3; - if (byterange_mode) { - hls->version = 4; - sequence = 0; - } - - if (hls->flags & HLS_INDEPENDENT_SEGMENTS) { - hls->version = 6; - } - if (hls->segment_type == SEGMENT_TYPE_FMP4) { - hls->version = 7; - } + if (last) { - if (!use_rename && !warned_non_file++) - av_log(s, AV_LOG_ERROR, "Cannot use rename on non file protocol, this may lead to races and temporary partial files\n"); + if ((hls->flags & HLS_OMIT_ENDLIST==0)) { + ff_hls_write_end_list(hls->m3u8_out); + } - set_http_options(s, &options, hls); - snprintf(temp_filename, sizeof(temp_filename), use_rename ? "%s.tmp" : "%s", vs->m3u8_name); - if ((ret = hlsenc_io_open(s, &hls->m3u8_out, temp_filename, &options)) < 0) - goto fail; + if (vs->vtt_m3u8_name) { + ff_hls_write_end_list(hls->sub_m3u8_out); + } + } else { - for (en = vs->segments; en; en = en->next) { - if (target_duration <= en->duration) - target_duration = lrint(en->duration); - } + byterange_mode = (hls->flags & HLS_SINGLE_FILE) || (hls->max_seg_size > 0); - vs->discontinuity_set = 0; - ff_hls_write_playlist_header(hls->m3u8_out, hls->version, hls->allowcache, - target_duration, sequence, hls->pl_type); + en = vs->last_segment; + if (en == NULL) { + ret = 1; + goto fail; + } - if((hls->flags & HLS_DISCONT_START) && sequence==hls->start_sequence && vs->discontinuity_set==0 ){ - avio_printf(hls->m3u8_out, "#EXT-X-DISCONTINUITY\n"); - vs->discontinuity_set = 1; - } - if (vs->has_video && (hls->flags & HLS_INDEPENDENT_SEGMENTS)) { - avio_printf(hls->m3u8_out, "#EXT-X-INDEPENDENT-SEGMENTS\n"); - } - for (en = vs->segments; en; en = en->next) { if ((hls->encrypt || hls->key_info_file) && (!key_uri || strcmp(en->key_uri, key_uri) || av_strcasecmp(en->iv_string, iv_string))) { avio_printf(hls->m3u8_out, "#EXT-X-KEY:METHOD=AES-128,URI=\"%s\"", en->key_uri); @@ -1444,17 +1417,8 @@ static int hls_window(AVFormatContext *s, int last, VariantStream *vs) if (ret < 0) { av_log(s, AV_LOG_WARNING, "ff_hls_write_file_entry get error\n"); } - } - - if (last && (hls->flags & HLS_OMIT_ENDLIST)==0) - ff_hls_write_end_list(hls->m3u8_out); - if( vs->vtt_m3u8_name ) { - if ((ret = hlsenc_io_open(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name, &options)) < 0) - goto fail; - ff_hls_write_playlist_header(hls->sub_m3u8_out, hls->version, hls->allowcache, - target_duration, sequence, PLAYLIST_TYPE_NONE); - for (en = vs->segments; en; en = en->next) { + if( vs->vtt_m3u8_name ) { ret = ff_hls_write_file_entry(hls->sub_m3u8_out, 0, byterange_mode, en->duration, 0, en->size, en->pos, vs->baseurl, en->sub_filename, NULL); @@ -1462,22 +1426,16 @@ static int hls_window(AVFormatContext *s, int last, VariantStream *vs) av_log(s, AV_LOG_WARNING, "ff_hls_write_file_entry get error\n"); } } - - if (last) - ff_hls_write_end_list(hls->sub_m3u8_out); - } fail: av_dict_free(&options); - hlsenc_io_close(s, &hls->m3u8_out, temp_filename); - hlsenc_io_close(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name); - if (ret >= 0 && use_rename) - ff_rename(temp_filename, vs->m3u8_name, s); - if (ret >= 0 && hls->master_pl_name) - if (create_master_playlist(s, vs) < 0) + if (ret >= 0 && hls->master_pl_name) { + if (create_master_playlist(s, vs) < 0) { av_log(s, AV_LOG_WARNING, "Master playlist creation failed\n"); + } + } return ret; } @@ -2078,6 +2036,11 @@ static int hls_write_header(AVFormatContext *s) int ret, i, j; AVDictionary *options = NULL; VariantStream *vs = NULL; + int target_duration = 0; + int byterange_mode = 0; + + target_duration = lrint(hls->time); + byterange_mode = (hls->flags & HLS_SINGLE_FILE) || (hls->max_seg_size > 0); for (i = 0; i < hls->nb_varstreams; i++) { vs = &hls->var_streams[i]; @@ -2091,7 +2054,7 @@ static int hls_write_header(AVFormatContext *s) goto fail; } av_dict_free(&options); - //av_assert0(s->nb_streams == hls->avf->nb_streams); + for (j = 0; j < vs->nb_streams; j++) { AVStream *inner_st; AVStream *outer_st = vs->streams[j]; @@ -2130,6 +2093,47 @@ static int hls_write_header(AVFormatContext *s) } } } + + // write the header for the main hls playlist + int64_t sequence = FFMAX(hls->start_sequence, vs->sequence - vs->nb_entries); + hls->version = 3; + if (byterange_mode) { + hls->version = 4; + sequence = 0; + } + + if (hls->flags & HLS_INDEPENDENT_SEGMENTS) { + hls->version = 6; + } + + if (hls->segment_type == SEGMENT_TYPE_FMP4) { + hls->version = 7; + } + + set_http_options(s, &options, hls); + if ((ret = hlsenc_io_open(s, &hls->m3u8_out, vs->m3u8_name, &options)) < 0) { + goto fail; + } + + vs->discontinuity_set = 0; + ff_hls_write_playlist_header(hls->m3u8_out, hls->version, hls->allowcache, target_duration, sequence, hls->pl_type); + + if((hls->flags & HLS_DISCONT_START) && sequence==hls->start_sequence && vs->discontinuity_set==0 ){ + avio_printf(hls->m3u8_out, "#EXT-X-DISCONTINUITY\n"); + vs->discontinuity_set = 1; + } + if (vs->has_video && (hls->flags & HLS_INDEPENDENT_SEGMENTS)) { + avio_printf(hls->m3u8_out, "#EXT-X-INDEPENDENT-SEGMENTS\n"); + } + + if( vs->vtt_m3u8_name ) { + if ((ret = hlsenc_io_open(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name, &options)) < 0) { + goto fail; + } + + ff_hls_write_playlist_header(hls->sub_m3u8_out, hls->version, hls->allowcache, target_duration, sequence, PLAYLIST_TYPE_NONE); + } + fail: return ret; @@ -2334,10 +2338,11 @@ static int hls_write_packet(AVFormatContext *s, AVPacket *pkt) return ret; } - if (!vs->fmp4_init_mode || byterange_mode) - if ((ret = hls_window(s, 0, vs)) < 0) { + if (!vs->fmp4_init_mode || byterange_mode) { + if ((ret = hls_write_manifest_segment(s, 0, vs)) < 0) { return ret; } + } } vs->packets_written++; @@ -2421,7 +2426,7 @@ failed: avformat_free_context(oc); vs->avf = NULL; - hls_window(s, 1, vs); + hls_write_manifest_segment(s, 1, vs); av_freep(&vs->fmp4_init_filename); if (vtt_oc) { @@ -2447,6 +2452,9 @@ failed: av_freep(&ccs->language); } + hlsenc_io_close(s, &hls->m3u8_out, vs->m3u8_name); + hlsenc_io_close(s, &hls->sub_m3u8_out, vs->vtt_m3u8_name); + ff_format_io_close(s, &hls->m3u8_out); ff_format_io_close(s, &hls->sub_m3u8_out); av_freep(&hls->key_basename); @@ -2884,3 +2892,4 @@ AVOutputFormat ff_hls_muxer = { .write_trailer = hls_write_trailer, .priv_class = &hls_class, }; + -- 2.6.3