Hi all,
I'm trying to get hardware-accelerated decoding with VideoToolbox working
on iOS. I'm using FFmpeg 3.2, and I'm trying to decode a live TV stream
(streamed from an HD Homerun) with mpegts & h264.
I've got a very simple toy program that works on macOS - see attached
sources. Poc.cpp/h creates and runs the decoder class, and exposes a
function that's callable from C (and Objective-C.) Decoder.cpp/hpp is a
simple C++ class which sets up the appropriate FFmpeg contexts and runs a
simple packet/frame pump. To enable HWA decoding, I set a custom get_format
function on the codec context before opening it; my get_format
implementation then watches to see if AV_PIX_FMT_VIDEOTOOLBOX is an
available option. If so, it calls av_videotoolbox_default_init on the codec
context and returns indicating selection of that format. Again, on macOS
this appears to work as expected, that is, my dummy frame handler function
gets a steady stream of frames, and CPU usage is much lower than it is
without my custom get_format implementation.
However, the same code fails on iOS, and then crashes. Specifically, the
setup code succeeds (or at least, it neither reports nor logs any errors),
then I enter my packet pump, and start pumping a few packets from the demux
to the codec. The first few times, this gets logged:
[h264 @ 0x101982400] Failed to decode frame (-12911)
[h264 @ 0x101982400] hardware accelerator failed to decode picture
...and avcodec_receive_frame returns AVERROR(EAGAIN) (not surprising, as
the codec buffer starts to fill.) Then after a few rounds of that (and
before avcodec_receive_frame actually returns any frames),
avcodec_send_packet crashes, EXC_BAD_ACCESS (code=1, address=0x0) with this
call stack:
#0 0x0000000100274f04 in ff_h264_decode_ref_pic_list_reordering ()
#1 0x000000010027bc58 in ff_h264_decode_slice_header ()
#2 0x000000010023a9c8 in h264_decode_frame ()
#3 0x0000000100596248 in avcodec_decode_video2 ()
#4 0x0000000100597824 in do_decode ()
#5 0x000000010158883c in decoder::run() at
/Users/josh/Git/ffmpeg-sandbox/ffmpeg-poc/ffmpeg-poc/decoder.cpp:85
#6 0x0000000101584e0c in ::run_poc() at
/Users/josh/Git/ffmpeg-sandbox/ffmpeg-poc/ffmpeg-poc/poc.cpp:42
So, is there something that I need to do differently on iOS vs macOS, or is
this a bug, or...?
Or, if nothing else, can anyone point me to an example of working source
code which successfully uses HWA decode on iOS?
*Josh Gustafson*
*Senior Software Engineer*
650.209.0952
www.simplecontrol.com
//
// decoder.cpp
// ffmpeg-poc
//
// Created by Josh Gustafson on 16/11/14.
// Copyright © 2016 Simple Control. All rights reserved.
//
#include "decoder.hpp"
#include <iostream>
#include <string>
#include <sstream>
using std::string;
decoder::decoder(string const& url) {
if (auto result = avformat_open_input(&formatContext, url.c_str(), nullptr, nullptr) < 0) {
std::ostringstream s;
s << "avformat_open_input failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
if (auto result = avformat_find_stream_info(formatContext, nullptr) < 0) {
std::ostringstream s;
s << "avformat_find_stream_info failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
auto videoStreamIndex = av_find_best_stream(formatContext, AVMEDIA_TYPE_VIDEO, -1, -1, &videoCodec, 0);
if (videoStreamIndex < 0) {
std::ostringstream s;
s << "av_find_best_stream(video) failed: " << av_err2str(videoStreamIndex);
throw std::runtime_error(s.str());
}
videoStream = formatContext->streams[videoStreamIndex];
assert(!!videoStream);
videoCodecContext = avcodec_alloc_context3(videoCodec);
if (!videoCodecContext) {
std::ostringstream s;
s << "avcodec_alloc_context3(video) failed";
throw std::runtime_error(s.str());
}
if (auto result = avcodec_parameters_to_context(videoCodecContext, videoStream->codecpar) < 0) {
std::ostringstream s;
s << "avcodec_parameters_to_context(video) failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
videoCodecContext->get_format = decoder::get_format;
if (auto result = avcodec_open2(videoCodecContext, videoCodec, nullptr) < 0) {
std::ostringstream s;
s << "avcodec_open2(video) failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
for (auto i = 0; i < formatContext->nb_streams; ++i) {
formatContext->streams[i]->discard = AVDISCARD_ALL;
}
videoStream->discard = AVDISCARD_DEFAULT;
}
decoder::~decoder() {
avcodec_free_context(&videoCodecContext);
avformat_close_input(&formatContext);
}
void decoder::run() {
AVPacket packet = { 0 };
auto frame = av_frame_alloc();
while (true) {
av_packet_unref(&packet);
auto result = av_read_frame(formatContext, &packet);
if (result < 0) {
if (result == AVERROR_EOF) {
avcodec_send_packet(videoCodecContext, NULL);
receive_video_frames(frame);
break;
} else {
std::ostringstream s;
s << "av_read_frame failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
} else {
if (packet.stream_index == videoStream->index) {
avcodec_send_packet(videoCodecContext, &packet);
if (!receive_video_frames(frame)) {
break;
}
}
}
}
av_packet_unref(&packet);
av_frame_free(&frame);
}
AVPixelFormat decoder::get_format(AVCodecContext* context, AVPixelFormat const formats[]) {
for (auto i = 0; formats[i] != AV_PIX_FMT_NONE; ++i) {
std::cerr << "get_format: format[" << i << "]: " << av_get_pix_fmt_name(formats[i]) << std::endl;
}
for (auto i = 0; formats[i] != AV_PIX_FMT_NONE; ++i)
{
if (formats[i] == AV_PIX_FMT_VIDEOTOOLBOX)
{
auto result = av_videotoolbox_default_init(context);
if (result < 0)
{
std::ostringstream s;
s << "av_videotoolbox_default_init failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
else
{
std::cerr << "get_format: selecting AV_PIX_FMT_VIDEOTOOLBOX" << std::endl;
return AV_PIX_FMT_VIDEOTOOLBOX;
}
}
}
return formats[0];
}
bool decoder::receive_video_frames(AVFrame* frame) {
while (true) {
auto result = avcodec_receive_frame(videoCodecContext, frame);
switch (result) {
case 0: handle_frame(frame); break;
case AVERROR(EAGAIN): return true;
case AVERROR_EOF: return false;
default:
{
std::ostringstream s;
s << "avcodec_receive_frame failed: " << av_err2str(result);
throw std::runtime_error(s.str());
}
}
}
}
void decoder::handle_frame(AVFrame const* frame) {
std::cerr << "Decoded frame: " << frame->width << "x" << frame->height << " " << av_get_pix_fmt_name(static_cast<AVPixelFormat>(frame->format)) << std::endl;
// TODO
}
//
// decoder.hpp
// ffmpeg-poc
//
// Created by Josh Gustafson on 16/11/14.
// Copyright © 2016 Simple Control. All rights reserved.
//
#ifndef decoder_hpp
#define decoder_hpp
#include <string>
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/videotoolbox.h>
#include <libavutil/pixdesc.h>
}
class decoder {
public:
decoder(std::string const& url);
~decoder();
void run();
private:
static AVPixelFormat get_format(AVCodecContext* context, AVPixelFormat const formats[]);
AVFormatContext* formatContext = nullptr;
AVStream* videoStream = nullptr;
AVCodec* videoCodec = nullptr;
AVCodecContext* videoCodecContext = nullptr;
bool receive_video_frames(AVFrame* frame);
void handle_frame(AVFrame const* frame);
};
#endif /* decoder_hpp */
//
// poc.cpp
// ffmpeg-poc-ios
//
// Created by Josh Gustafson on 16/11/14.
// Copyright © 2016 Simple Control. All rights reserved.
//
#include "poc.h"
#include <iostream>
#include <string>
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/videotoolbox.h>
#include <libavutil/pixdesc.h>
}
#include "decoder.hpp"
using namespace std::string_literals;
using std::string;
namespace {
auto const input_url = "http://192.168.3.32:5004/auto/v660"s;
class avformat_network_context {
public:
avformat_network_context() { avformat_network_init(); }
~avformat_network_context() { avformat_network_deinit(); }
};
}
int run_poc() {
av_register_all();
avformat_network_context c;
try {
decoder d(input_url);
d.run();
return 0;
} catch(std::runtime_error const& e) {
std::cerr << e.what() << std::endl;
return -1;
}
}
//
// poc.hpp
// ffmpeg-poc-ios
//
// Created by Josh Gustafson on 16/11/14.
// Copyright © 2016 Simple Control. All rights reserved.
//
#ifndef poc_hpp
#define poc_hpp
#ifdef __cplusplus
#define EXTERNC extern "C"
#else
#define EXTERNC
#endif
EXTERNC int run_poc();
#endif /* poc_hpp */
_______________________________________________
Libav-user mailing list
[email protected]
http://ffmpeg.org/mailman/listinfo/libav-user
