Hello Tanguy, Robert,
We've put one of our best debuggers to work on finding out why the video
stopped updating after 15-20 seconds (see previous messages in this
thread). He found the part of the code that made it stop, though the
deep-down reason for it still remains a bit of a mystery to us, but this
is at least a good clue.
In FFmpegDecoderVideo.cpp, in the publishFrame() method:
void FFmpegDecoderVideo::publishFrame(const double delay)
{
// ...
// If the display delay is too small, we better skip the frame.
if (delay < -0.010)
return;
// ...
It seems that the delay value comes from synchronizing the video to the
audio. However, in the case of our stream, there is no audio, and this
synchronization does not work well or gives bad results, because this is
the test that makes the video stop updating.
Removing this makes it work for us, though of course this is not the
right course of action. Now, the FFmpegClocks class already had an
m_audio_disabled member and a method audioDisable() to disable audio,
but no getter. So we added a getter, and now the publishFrame() method
takes this as an argument and does not try to synchronize with the audio
if there is no audio present:
// ... this is where publishFrame() is called:
publishFrame(frame_delay, m_clocks.audioDisabled());
// ...
void FFmpegDecoderVideo::publishFrame(const double delay,
bool audio_disabled)
{
// ...
// If the display delay is too small, we better skip the frame.
if (!audio_disabled && delay < -0.010)
return;
// ...
This also fixes the problem for us. To test it, compile the plugin with
the attached changed files, and run the following:
osgmovie -e ffmpeg http://iris.cm-labs.com:10080/img/video.mjpeg
Now, this is more of a request for comment than a proper submission. I'd
like others who might know more about these things to tell us if there
are any consequences of this change on other paths (like reading from
files with no audio instead of streams with no audio) that we can't test
out ourselves. We don't want our fix to break it for others. If so,
please suggest a fix that would work for both cases...
Nevertheless, this is encouraging, we're getting progress and we'll be
able to use this in our project.
Thanks in advance,
J-S
--
______________________________________________________
Jean-Sebastien Guay [email protected]
http://www.cm-labs.com/
http://whitestar02.webhop.org/
#ifndef HEADER_GUARD_OSGFFMPEG_FFMPEG_CLOCKS_H
#define HEADER_GUARD_OSGFFMPEG_FFMPEG_CLOCKS_H
#include <osg/Timer>
#include <OpenThreads/Mutex>
#include <OpenThreads/ScopedLock>
#include "FFmpegHeaders.hpp"
namespace osgFFmpeg {
class FFmpegClocks
{
public:
FFmpegClocks();
void reset(double start_time);
void rewindAudio();
void rewindVideo();
void audioSetBufferEndPts(double pts);
void audioAdjustBufferEndPts(double increment);
void audioSetDelay(double delay);
void audioDisable();
bool audioDisabled() const { return m_audio_disabled; }
double videoSynchClock(const AVFrame * frame, double time_base, double pts);
double videoRefreshSchedule(double pts);
double getStartTime() const;
private:
double getAudioTime() const;
typedef osg::Timer Timer;
typedef OpenThreads::Mutex Mutex;
typedef OpenThreads::ScopedLock<Mutex> ScopedLock;
mutable Mutex m_mutex;
double m_video_clock;
double m_start_time;
double m_last_frame_delay;
double m_last_frame_pts;
double m_last_actual_delay;
double m_frame_time;
double m_audio_buffer_end_pts;
double m_audio_delay;
Timer m_audio_timer;
bool m_audio_disabled;
bool m_rewind;
};
} // namespace osgFFmpeg
#endif // HEADER_GUARD_OSGFFMPEG_FFMPEG_CLOCKS_H
#include "FFmpegDecoderVideo.hpp"
#include <osg/Notify>
#include <osg/Timer>
#include <stdexcept>
#include <string.h>
namespace osgFFmpeg {
FFmpegDecoderVideo::FFmpegDecoderVideo(PacketQueue & packets, FFmpegClocks &
clocks) :
m_packets(packets),
m_clocks(clocks),
m_stream(0),
m_context(0),
m_codec(0),
m_packet_data(0),
m_bytes_remaining(0),
m_packet_pts(AV_NOPTS_VALUE),
m_writeBuffer(0),
m_user_data(0),
m_publish_func(0),
m_exit(false)
#ifdef USE_SWSCALE
,m_swscale_ctx(0)
#endif
{
}
FFmpegDecoderVideo::~FFmpegDecoderVideo()
{
osg::notify(osg::INFO)<<"Destructing FFmpegDecoderVideo..."<<std::endl;
if (isRunning())
{
m_exit = true;
#if 0
while(isRunning()) { OpenThreads::YieldCurrentThread(); }
#else
join();
#endif
}
#ifdef USE_SWSCALE
if (m_swscale_ctx)
{
sws_freeContext(m_swscale_ctx);
m_swscale_ctx = 0;
}
#endif
osg::notify(osg::INFO)<<"Destructed FFmpegDecoderVideo"<<std::endl;
}
void FFmpegDecoderVideo::open(AVStream * const stream)
{
m_stream = stream;
m_context = stream->codec;
// Trust the video size given at this point
// (avcodec_open seems to sometimes return a 0x0 size)
m_width = m_context->width;
m_height = m_context->height;
findAspectRatio();
// Find out whether we support Alpha channel
m_alpha_channel = (m_context->pix_fmt == PIX_FMT_YUVA420P);
// Find out the framerate
m_frame_rate = av_q2d(stream->r_frame_rate);
// Find the decoder for the video stream
m_codec = avcodec_find_decoder(m_context->codec_id);
if (m_codec == 0)
throw std::runtime_error("avcodec_find_decoder() failed");
// Inform the codec that we can handle truncated bitstreams
//if (p_codec->capabilities & CODEC_CAP_TRUNCATED)
// m_context->flags |= CODEC_FLAG_TRUNCATED;
// Open codec
if (avcodec_open(m_context, m_codec) < 0)
throw std::runtime_error("avcodec_open() failed");
// Allocate video frame
m_frame.reset(avcodec_alloc_frame());
// Allocate converted RGB frame
m_frame_rgba.reset(avcodec_alloc_frame());
m_buffer_rgba[0].resize(avpicture_get_size(PIX_FMT_RGB32, width(),
height()));
m_buffer_rgba[1].resize(m_buffer_rgba[0].size());
// Assign appropriate parts of the buffer to image planes in m_frame_rgba
avpicture_fill((AVPicture *) (m_frame_rgba).get(), &(m_buffer_rgba[0])[0],
PIX_FMT_RGB32, width(), height());
// Override get_buffer()/release_buffer() from codec context in order to
retrieve the PTS of each frame.
m_context->opaque = this;
m_context->get_buffer = getBuffer;
m_context->release_buffer = releaseBuffer;
}
void FFmpegDecoderVideo::close(bool waitForThreadToExit)
{
m_exit = true;
if (isRunning() && waitForThreadToExit)
{
while(isRunning()) { OpenThreads::Thread::YieldCurrentThread(); }
}
}
void FFmpegDecoderVideo::run()
{
try
{
decodeLoop();
}
catch (const std::exception & error)
{
osg::notify(osg::WARN) << "FFmpegDecoderVideo::run : " << error.what()
<< std::endl;
}
catch (...)
{
osg::notify(osg::WARN) << "FFmpegDecoderVideo::run : unhandled
exception" << std::endl;
}
}
void FFmpegDecoderVideo::decodeLoop()
{
FFmpegPacket packet;
double pts;
while (! m_exit)
{
// Work on the current packet until we have decoded all of it
while (m_bytes_remaining > 0)
{
// Save global PTS to be stored in m_frame via getBuffer()
m_packet_pts = packet.packet.pts;
// Decode video frame
int frame_finished = 0;
const int bytes_decoded = avcodec_decode_video(m_context,
m_frame.get(), &frame_finished, m_packet_data, m_bytes_remaining);
if (bytes_decoded < 0)
throw std::runtime_error("avcodec_decode_video failed()");
m_bytes_remaining -= bytes_decoded;
m_packet_data += bytes_decoded;
// Find out the frame pts
if (packet.packet.dts == AV_NOPTS_VALUE &&
m_frame->opaque != 0 &&
*reinterpret_cast<const int64_t*>(m_frame->opaque) !=
AV_NOPTS_VALUE)
{
pts = *reinterpret_cast<const int64_t*>(m_frame->opaque);
}
else if (packet.packet.dts != AV_NOPTS_VALUE)
{
pts = packet.packet.dts;
}
else
{
pts = 0;
}
pts *= av_q2d(m_stream->time_base);
// Publish the frame if we have decoded a complete frame
if (frame_finished)
{
const double synched_pts =
m_clocks.videoSynchClock(m_frame.get(), av_q2d(m_stream->time_base), pts);
const double frame_delay =
m_clocks.videoRefreshSchedule(synched_pts);
publishFrame(frame_delay, m_clocks.audioDisabled());
}
}
// Get the next packet
pts = 0;
if (packet.valid())
packet.clear();
bool is_empty = true;
packet = m_packets.timedPop(is_empty, 10);
if (! is_empty)
{
if (packet.type == FFmpegPacket::PACKET_DATA)
{
m_bytes_remaining = packet.packet.size;
m_packet_data = packet.packet.data;
}
else if (packet.type == FFmpegPacket::PACKET_FLUSH)
{
avcodec_flush_buffers(m_context);
m_clocks.rewindVideo();
}
}
}
}
void FFmpegDecoderVideo::findAspectRatio()
{
float ratio = 0.0f;
if (m_context->sample_aspect_ratio.num != 0)
ratio = float(av_q2d(m_context->sample_aspect_ratio));
if (ratio <= 0.0f)
ratio = 1.0f;
m_pixel_aspect_ratio = ratio;
}
int FFmpegDecoderVideo::convert(AVPicture *dst, int dst_pix_fmt, AVPicture *src,
int src_pix_fmt, int src_width, int src_height)
{
osg::Timer_t startTick = osg::Timer::instance()->tick();
#ifdef USE_SWSCALE
if (m_swscale_ctx==0)
{
m_swscale_ctx = sws_getContext(src_width, src_height, (PixelFormat)
src_pix_fmt,
src_width, src_height, (PixelFormat)
dst_pix_fmt,
/*SWS_BILINEAR*/ SWS_BICUBIC, NULL, NULL,
NULL);
}
osg::notify(osg::INFO)<<"Using sws_scale ";
int result = sws_scale(m_swscale_ctx,
(src->data), (src->linesize), 0, src_height,
(dst->data), (dst->linesize));
#else
osg::notify(osg::INFO)<<"Using img_convert ";
int result = img_convert(dst, dst_pix_fmt, src,
src_pix_fmt, src_width, src_height);
#endif
osg::Timer_t endTick = osg::Timer::instance()->tick();
osg::notify(osg::INFO)<<" time =
"<<osg::Timer::instance()->delta_m(startTick,endTick)<<"ms"<<std::endl;
return result;
}
void FFmpegDecoderVideo::publishFrame(const double delay, bool audio_disabled)
{
// If no publishing function, just ignore the frame
if (m_publish_func == 0)
return;
// If the display delay is too small, we better skip the frame.
if (!audio_disabled && delay < -0.010)
return;
AVPicture * const src = (AVPicture *) m_frame.get();
AVPicture * const dst = (AVPicture *) m_frame_rgba.get();
// Assign appropriate parts of the buffer to image planes in m_frame_rgba
avpicture_fill((AVPicture *) (m_frame_rgba).get(),
&(m_buffer_rgba[m_writeBuffer])[0], PIX_FMT_RGB32, width(), height());
// Convert YUVA420p (i.e. YUV420p plus alpha channel) using our own routine
if (m_context->pix_fmt == PIX_FMT_YUVA420P)
yuva420pToRgba(dst, src, width(), height());
else
convert(dst, PIX_FMT_RGB32, src, m_context->pix_fmt, width(), height());
// Wait 'delay' seconds before publishing the picture.
int i_delay = static_cast<int>(delay * 1000000 + 0.5);
while (i_delay > 1000)
{
// Avoid infinite/very long loops
if (m_exit)
return;
const int micro_delay = (std::min)(1000000, i_delay);
OpenThreads::Thread::microSleep(micro_delay);
i_delay -= micro_delay;
}
m_writeBuffer = 1-m_writeBuffer;
m_publish_func(* this, m_user_data);
}
void FFmpegDecoderVideo::yuva420pToRgba(AVPicture * const dst, AVPicture *
const src, int width, int height)
{
convert(dst, PIX_FMT_RGB32, src, m_context->pix_fmt, width, height);
const size_t bpp = 4;
uint8_t * a_dst = dst->data[0] + 3;
for (int h = 0; h < height; ++h) {
const uint8_t * a_src = src->data[3] + h * src->linesize[3];
for (int w = 0; w < width; ++w) {
*a_dst = *a_src;
a_dst += bpp;
a_src += 1;
}
}
}
int FFmpegDecoderVideo::getBuffer(AVCodecContext * const context, AVFrame *
const picture)
{
const FFmpegDecoderVideo * const this_ = reinterpret_cast<const
FFmpegDecoderVideo*>(context->opaque);
const int result = avcodec_default_get_buffer(context, picture);
int64_t * p_pts = reinterpret_cast<int64_t*>( av_malloc(sizeof(int64_t)) );
*p_pts = this_->m_packet_pts;
picture->opaque = p_pts;
return result;
}
void FFmpegDecoderVideo::releaseBuffer(AVCodecContext * const context, AVFrame
* const picture)
{
if (picture != 0)
av_freep(&picture->opaque);
avcodec_default_release_buffer(context, picture);
}
} // namespace osgFFmpeg
#ifndef HEADER_GUARD_OSGFFMPEG_FFMPEG_DECODER_VIDEO_H
#define HEADER_GUARD_OSGFFMPEG_FFMPEG_DECODER_VIDEO_H
#include "FFmpegHeaders.hpp"
#include "BoundedMessageQueue.hpp"
#include "FFmpegClocks.hpp"
#include "FFmpegPacket.hpp"
#include <OpenThreads/Thread>
#include <vector>
namespace osgFFmpeg {
class FramePtr
{
public:
typedef AVFrame T;
explicit FramePtr() : _ptr(0) {}
explicit FramePtr(T* ptr) : _ptr(ptr) {}
~FramePtr()
{
cleanup();
}
T* get() { return _ptr; }
T * operator-> () const // never throws
{
return _ptr;
}
void reset(T* ptr)
{
if (ptr==_ptr) return;
cleanup();
_ptr = ptr;
}
void cleanup()
{
if (_ptr) av_free(_ptr);
_ptr = 0;
}
protected:
T* _ptr;
};
class FFmpegDecoderVideo : public OpenThreads::Thread
{
public:
typedef BoundedMessageQueue<FFmpegPacket> PacketQueue;
typedef void (* PublishFunc) (const FFmpegDecoderVideo & decoder, void *
user_data);
FFmpegDecoderVideo(PacketQueue & packets, FFmpegClocks & clocks);
~FFmpegDecoderVideo();
void open(AVStream * stream);
void close(bool waitForThreadToExit);
virtual void run();
void setUserData(void * user_data);
void setPublishCallback(PublishFunc function);
int width() const;
int height() const;
float pixelAspectRatio() const;
bool alphaChannel() const;
double frameRate() const;
const uint8_t * image() const;
private:
typedef std::vector<uint8_t> Buffer;
void decodeLoop();
void findAspectRatio();
void publishFrame(double delay, bool audio_disabled);
double synchronizeVideo(double pts);
void yuva420pToRgba(AVPicture *dst, AVPicture *src, int width, int height);
int convert(AVPicture *dst, int dst_pix_fmt, AVPicture *src,
int src_pix_fmt, int src_width, int src_height);
static int getBuffer(AVCodecContext * context, AVFrame * picture);
static void releaseBuffer(AVCodecContext * context, AVFrame * picture);
PacketQueue & m_packets;
FFmpegClocks & m_clocks;
AVStream * m_stream;
AVCodecContext * m_context;
AVCodec * m_codec;
const uint8_t * m_packet_data;
int m_bytes_remaining;
int64_t m_packet_pts;
FramePtr m_frame;
FramePtr m_frame_rgba;
Buffer m_buffer_rgba[2];
int m_writeBuffer;
void * m_user_data;
PublishFunc m_publish_func;
double m_frame_rate;
float m_pixel_aspect_ratio;
int m_width;
int m_height;
size_t m_next_frame_index;
bool m_alpha_channel;
volatile bool m_exit;
#ifdef USE_SWSCALE
struct SwsContext * m_swscale_ctx;
#endif
};
inline void FFmpegDecoderVideo::setUserData(void * const user_data)
{
m_user_data = user_data;
}
inline void FFmpegDecoderVideo::setPublishCallback(const PublishFunc function)
{
m_publish_func = function;
}
inline int FFmpegDecoderVideo::width() const
{
return m_width;
}
inline int FFmpegDecoderVideo::height() const
{
return m_height;
}
inline float FFmpegDecoderVideo::pixelAspectRatio() const
{
return m_pixel_aspect_ratio;
}
inline bool FFmpegDecoderVideo::alphaChannel() const
{
return m_alpha_channel;
}
inline double FFmpegDecoderVideo::frameRate() const
{
return m_frame_rate;
}
inline const uint8_t * FFmpegDecoderVideo::image() const
{
return &((m_buffer_rgba[1-m_writeBuffer])[0]);
}
} // namespace osgFFmpeg
#endif // HEADER_GUARD_OSGFFMPEG_FFMPEG_DECODER_VIDEO_H
_______________________________________________
osg-submissions mailing list
[email protected]
http://lists.openscenegraph.org/listinfo.cgi/osg-submissions-openscenegraph.org
