hi,
attached patch provides decklink consumer rework in order to simplify
and fix current bugs (#3311153, #3309546, #3308341). it also makes
(IMHO) it more straightforward and easer to investigate...
i did a tests against videos i have and found no noticeable regressions.
--
________________________________________
Maksym Veremeyenko
>From ce42925614bccfe1cc4c5d36521ddd84862ccf1b Mon Sep 17 00:00:00 2001
From: Maksym Veremeyenko <ve...@m1stereo.tv>
Date: Wed, 8 Jun 2011 13:42:02 +0300
Subject: [PATCH] migrate to timestamped audio packet
---
src/modules/decklink/consumer_decklink.cpp | 364 +++++++---------------------
1 files changed, 83 insertions(+), 281 deletions(-)
diff --git a/src/modules/decklink/consumer_decklink.cpp b/src/modules/decklink/consumer_decklink.cpp
index a5fcfad..eea528e 100644
--- a/src/modules/decklink/consumer_decklink.cpp
+++ b/src/modules/decklink/consumer_decklink.cpp
@@ -28,81 +28,26 @@
static const unsigned PREROLL_MINIMUM = 3;
-typedef struct
-{
- int16_t *buffer;
- int size;
- int used;
- pthread_mutex_t mutex;
-} *sample_fifo;
-
-static sample_fifo sample_fifo_init()
-{
- sample_fifo fifo = (sample_fifo) calloc( 1, sizeof( *fifo ) );
- pthread_mutex_init( &fifo->mutex, NULL );
- return fifo;
-}
-
-static void sample_fifo_append( sample_fifo fifo, int16_t *samples, int count )
-{
- pthread_mutex_lock( &fifo->mutex );
- if ( ( fifo->size - fifo->used ) < count )
- {
- fifo->size += count * 5;
- fifo->buffer = (int16_t*) realloc( fifo->buffer, fifo->size * sizeof( int16_t ) );
- }
- memcpy( fifo->buffer + fifo->used, samples, count * sizeof( int16_t ) );
- fifo->used += count;
- pthread_mutex_unlock( &fifo->mutex );
-}
-
-static void sample_fifo_remove( sample_fifo fifo, int count )
-{
- pthread_mutex_lock( &fifo->mutex );
- if ( count > fifo->used )
- count = fifo->used;
- fifo->used -= count;
- memmove( fifo->buffer, fifo->buffer + count, fifo->used * sizeof( int16_t ) );
- pthread_mutex_unlock( &fifo->mutex );
-}
-
-static void sample_fifo_close( sample_fifo fifo )
-{
- free( fifo->buffer );
- pthread_mutex_destroy( &fifo->mutex );
- free( fifo );
-}
-
-
class DeckLinkConsumer
: public IDeckLinkVideoOutputCallback
- , public IDeckLinkAudioOutputCallback
{
private:
mlt_consumer_s m_consumer;
IDeckLink* m_deckLink;
IDeckLinkOutput* m_deckLinkOutput;
IDeckLinkDisplayMode* m_displayMode;
- pthread_mutex_t m_mutex;
- pthread_cond_t m_condition;
int m_width;
int m_height;
BMDTimeValue m_duration;
BMDTimeScale m_timescale;
double m_fps;
uint64_t m_count;
- sample_fifo m_fifo;
- unsigned m_preroll;
- bool m_isPrerolling;
- unsigned m_prerollCounter;
int m_channels;
- uint32_t m_maxAudioBuffer;
- mlt_deque m_videoFrameQ;
- mlt_frame m_frame;
unsigned m_dropped;
bool m_isAudio;
int m_isKeyer;
IDeckLinkKeyer* m_deckLinkKeyer;
+ bool m_terminate_on_pause;
IDeckLinkDisplayMode* getDisplayMode()
{
@@ -134,8 +79,6 @@ private:
public:
mlt_consumer getConsumer()
{ return &m_consumer; }
- uint64_t isBuffering() const
- { return m_prerollCounter < m_preroll; }
~DeckLinkConsumer()
{
@@ -145,12 +88,6 @@ public:
m_deckLinkOutput->Release();
if ( m_deckLink )
m_deckLink->Release();
- if ( m_videoFrameQ )
- {
- mlt_deque_close( m_videoFrameQ );
- pthread_mutex_destroy( &m_mutex );
- pthread_cond_destroy( &m_condition );
- }
}
bool open( unsigned card = 0 )
@@ -207,29 +144,23 @@ public:
// Provide this class as a delegate to the audio and video output interfaces
m_deckLinkOutput->SetScheduledFrameCompletionCallback( this );
- m_deckLinkOutput->SetAudioCallback( this );
-
- pthread_mutex_init( &m_mutex, NULL );
- pthread_cond_init( &m_condition, NULL );
- m_maxAudioBuffer = bmdAudioSampleRate48kHz;
- m_videoFrameQ = mlt_deque_init();
return true;
}
bool start( unsigned preroll )
{
+ unsigned i;
mlt_properties properties = MLT_CONSUMER_PROPERTIES( getConsumer() );
// Initialize members
m_count = 0;
- m_frame = 0;
m_dropped = 0;
- m_isPrerolling = true;
- m_prerollCounter = 0;
- m_preroll = preroll < PREROLL_MINIMUM ? PREROLL_MINIMUM : preroll;
+ preroll = preroll < PREROLL_MINIMUM ? PREROLL_MINIMUM : preroll;
m_channels = mlt_properties_get_int( properties, "channels" );
m_isAudio = !mlt_properties_get_int( properties, "audio_off" );
+ m_terminate_on_pause = mlt_properties_get_int( properties, "terminate_on_pause" );
+
m_displayMode = getDisplayMode();
if ( !m_displayMode )
@@ -268,40 +199,34 @@ public:
return true;
}
if ( S_OK != m_deckLinkOutput->EnableAudioOutput( bmdAudioSampleRate48kHz, bmdAudioSampleType16bitInteger,
- m_channels, bmdAudioOutputStreamContinuous ) )
+ m_channels, bmdAudioOutputStreamTimestamped ) )
{
mlt_log_error( getConsumer(), "Failed to enable audio output\n" );
stop();
return false;
}
- m_fifo = sample_fifo_init();
- m_deckLinkOutput->BeginAudioPreroll();
-
+
+ // preroll frames
+ for( i = 0; i < preroll; i++ )
+ ScheduleNextFrame( true );
+
+ // start scheduled playback
+ m_deckLinkOutput->StartScheduledPlayback( 0, m_timescale, 1.0 );
+
+ // Set the running state
+ mlt_properties_set_int( properties, "running", 1 );
+
return true;
}
- void wakeup()
- {
- pthread_mutex_lock( &m_mutex );
- pthread_cond_broadcast( &m_condition );
- pthread_mutex_unlock( &m_mutex );
- }
-
- void wait()
- {
- struct timeval tv;
- struct timespec ts;
-
- gettimeofday( &tv, NULL );
- ts.tv_sec = tv.tv_sec + 1;
- ts.tv_nsec = tv.tv_usec * 1000;
- pthread_mutex_lock( &m_mutex );
- pthread_cond_timedwait( &m_condition, &m_mutex, &ts );
- pthread_mutex_unlock( &m_mutex );
- }
-
- void stop()
+ bool stop()
{
+ mlt_properties properties = MLT_CONSUMER_PROPERTIES( getConsumer() );
+
+ // set running state is 0
+ mlt_properties_set_int( properties, "running", 0 );
+ mlt_consumer_stopped( getConsumer() );
+
// Stop the audio and video output streams immediately
if ( m_deckLinkOutput )
{
@@ -309,13 +234,8 @@ public:
m_deckLinkOutput->DisableAudioOutput();
m_deckLinkOutput->DisableVideoOutput();
}
- while ( mlt_deque_count( m_videoFrameQ ) )
- {
- IDeckLinkMutableVideoFrame* frame = (IDeckLinkMutableVideoFrame*) mlt_deque_pop_back( m_videoFrameQ );
- frame->Release();
- }
- if ( m_fifo ) sample_fifo_close( m_fifo );
- mlt_frame_close( m_frame );
+
+ return true;
}
void renderAudio( mlt_frame frame )
@@ -327,47 +247,24 @@ public:
if ( !mlt_frame_get_audio( frame, (void**) &pcm, &format, &frequency, &m_channels, &samples ) )
{
- int count = samples;
-
- if ( !m_isPrerolling )
- {
- uint32_t audioCount = 0;
- uint32_t videoCount = 0;
-
- // Check for resync
- m_deckLinkOutput->GetBufferedAudioSampleFrameCount( &audioCount );
- m_deckLinkOutput->GetBufferedVideoFrameCount( &videoCount );
+ uint32_t written = 0;
- // Underflow typically occurs during non-normal speed playback.
- if ( audioCount < 1 || videoCount < 1 )
- {
- // Upon switching to normal playback, buffer some frames faster than realtime.
- mlt_log_info( getConsumer(), "buffer underrun: audio buf %u video buf %u frames\n", audioCount, videoCount );
- m_prerollCounter = 0;
- }
+ m_deckLinkOutput->ScheduleAudioSamples( pcm, samples, m_count * frequency / m_fps, frequency, &written );
- // While rebuffering
- if ( videoCount == 0 && isBuffering() )
- {
- // Only append audio to reach the ideal level and not overbuffer.
- int ideal = ( m_preroll - 1 ) * bmdAudioSampleRate48kHz / m_fps;
- int actual = m_fifo->used / m_channels + audioCount;
- int diff = ideal / 2 - actual;
- count = diff < 0 ? 0 : diff < count ? diff : count;
- }
- }
- if ( count > 0 )
- sample_fifo_append( m_fifo, pcm, count * m_channels );
+ if ( written != samples )
+ mlt_log_verbose( getConsumer(), "renderAudio: samples=%d, written=%d\n", samples, written );
}
}
- bool createFrame()
+ bool createFrame( IDeckLinkMutableVideoFrame** decklinkFrame )
{
BMDPixelFormat format = m_isKeyer? bmdFormat8BitARGB : bmdFormat8BitYUV;
IDeckLinkMutableVideoFrame* frame = 0;
uint8_t *buffer = 0;
int stride = m_width * ( m_isKeyer? 4 : 2 );
+ *decklinkFrame = NULL;
+
// Generate a DeckLink video frame
if ( S_OK != m_deckLinkOutput->CreateVideoFrame( m_width, m_height,
stride, format, bmdFrameFlagDefault, &frame ) )
@@ -390,27 +287,29 @@ public:
*buffer++ = 16;
}
}
- mlt_log_debug( getConsumer(), "created video frame\n" );
- mlt_deque_push_back( m_videoFrameQ, frame );
+
+ *decklinkFrame = frame;
return true;
}
- void renderVideo()
+ void renderVideo( mlt_frame frame )
{
mlt_image_format format = m_isKeyer? mlt_image_rgb24a : mlt_image_yuv422;
uint8_t* image = 0;
- if ( !mlt_frame_get_image( m_frame, &image, &format, &m_width, &m_height, 0 ) )
+ if ( !mlt_frame_get_image( frame, &image, &format, &m_width, &m_height, 0 ) )
{
- IDeckLinkMutableVideoFrame* decklinkFrame = (IDeckLinkMutableVideoFrame*) mlt_deque_pop_back( m_videoFrameQ );
+ IDeckLinkMutableVideoFrame* decklinkFrame;
uint8_t* buffer = 0;
int stride = m_width * ( m_isKeyer? 4 : 2 );
- decklinkFrame->GetBytes( (void**) &buffer );
+ if ( createFrame( &decklinkFrame ) )
+ decklinkFrame->GetBytes( (void**) &buffer );
+
if ( buffer )
{
- int progressive = mlt_properties_get_int( MLT_FRAME_PROPERTIES( m_frame ), "progressive" );
+ int progressive = mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "progressive" );
if ( !m_isKeyer )
{
@@ -421,7 +320,7 @@ public:
else
swab( image, buffer, stride * m_height );
}
- else if ( !mlt_properties_get_int( MLT_FRAME_PROPERTIES( m_frame ), "test_image" ) )
+ else if ( !mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "test_image" ) )
{
// Normal keyer output
int y = m_height + 1;
@@ -454,7 +353,8 @@ public:
}
m_deckLinkOutput->ScheduleVideoFrame( decklinkFrame, m_count * m_duration, m_duration, m_timescale );
}
- mlt_deque_push_front( m_videoFrameQ, decklinkFrame );
+ if ( decklinkFrame )
+ decklinkFrame->Release();
}
}
@@ -466,45 +366,11 @@ public:
double speed = mlt_properties_get_double( MLT_FRAME_PROPERTIES(frame), "_speed" );
if ( m_isAudio && speed == 1.0 )
renderAudio( frame );
-
- // Create video frames while pre-rolling
- if ( m_isPrerolling )
- {
- if ( !createFrame() )
- {
- mlt_log_error( getConsumer(), "failed to create video frame\n" );
- return S_FALSE;
- }
- }
-
- if ( mlt_properties_get_int( MLT_FRAME_PROPERTIES( frame ), "rendered") )
- {
- // Close the previous frame and use the new one
- mlt_frame_close( m_frame );
- m_frame = frame;
- }
- else
- {
- if ( !m_frame )
- m_frame = frame;
- // Reuse the last frame
- mlt_log_verbose( getConsumer(), "dropped video frame %u\n", ++m_dropped );
- }
// Get the video
- renderVideo();
+ renderVideo( frame );
++m_count;
- // Check for end of pre-roll
- if ( ++m_prerollCounter > m_preroll && m_isPrerolling )
- {
- // Start audio and video output
- if ( m_isAudio )
- m_deckLinkOutput->EndAudioPreroll();
- m_deckLinkOutput->StartScheduledPlayback( 0, m_timescale, 1.0 );
- m_isPrerolling = false;
- }
-
return result;
}
@@ -523,8 +389,18 @@ public:
virtual HRESULT STDMETHODCALLTYPE ScheduledFrameCompleted( IDeckLinkVideoFrame* completedFrame, BMDOutputFrameCompletionResult completed )
{
// When a video frame has been released by the API, schedule another video frame to be output
- wakeup();
-
+
+ // ignore handler if frame was flushed
+ if(bmdOutputFrameFlushed == completed)
+ return S_OK;
+
+ // schedule next frame
+ ScheduleNextFrame(false);
+
+ // enqueu more frames if underrun
+ if(bmdOutputFrameDisplayedLate == completed)
+ ScheduleNextFrame(false);
+
return S_OK;
}
@@ -533,75 +409,37 @@ public:
return mlt_consumer_is_stopped( getConsumer() ) ? S_FALSE : S_OK;
}
- virtual HRESULT STDMETHODCALLTYPE RenderAudioSamples( bool preroll )
+
+ void ScheduleNextFrame(bool preroll)
{
- // Provide more audio samples to the DeckLink API
- HRESULT result = S_OK;
+ // get the consumer
+ mlt_consumer consumer = getConsumer();
+
+ // Get the properties
+ mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
- uint32_t count = m_fifo->used / m_channels;
- uint32_t buffered = count;
+ // Frame and size
+ mlt_frame frame = NULL;
- if ( count
- // Stay under preferred buffer level
- && ( S_OK == m_deckLinkOutput->GetBufferedAudioSampleFrameCount( &buffered ) )
- && buffered < m_maxAudioBuffer )
+ if( mlt_properties_get_int( properties, "running" ) || preroll )
{
- uint32_t written = 0;
-
- buffered = m_maxAudioBuffer - buffered;
- count = buffered > count ? count : buffered;
- result = m_deckLinkOutput->ScheduleAudioSamples( m_fifo->buffer, count, 0, 0, &written );
- if ( written )
- sample_fifo_remove( m_fifo, written * m_channels );
- }
+ frame = mlt_consumer_rt_frame( consumer );
+ if ( frame != NULL )
+ {
+ render( frame );
- return result;
- }
-};
+ mlt_events_fire( properties, "consumer-frame-show", frame, NULL );
-/** The main thread.
- */
+ // terminate on pause
+ if (m_terminate_on_pause &&
+ mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0)
+ stop();
-static void *run( void *arg )
-{
- // Map the argument to the object
- DeckLinkConsumer* decklink = (DeckLinkConsumer*) arg;
- mlt_consumer consumer = decklink->getConsumer();
-
- // Get the properties
- mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
-
- // Convenience functionality
- int terminate_on_pause = mlt_properties_get_int( properties, "terminate_on_pause" );
- int terminated = 0;
-
- // Frame and size
- mlt_frame frame = NULL;
-
- // Loop while running
- while ( !terminated && mlt_properties_get_int( properties, "running" ) )
- {
- // Get the frame
- if ( ( frame = mlt_consumer_rt_frame( consumer ) ) )
- {
- // Check for termination
- if ( terminate_on_pause )
- terminated = mlt_properties_get_double( MLT_FRAME_PROPERTIES( frame ), "_speed" ) == 0.0;
-
- decklink->render( frame );
- if ( !decklink->isBuffering() )
- decklink->wait();
- mlt_events_fire( properties, "consumer-frame-show", frame, NULL );
+ mlt_frame_close( frame );
+ }
}
}
-
- // Indicate that the consumer is stopped
- decklink->stop();
- mlt_properties_set_int( properties, "running", 0 );
- mlt_consumer_stopped( consumer );
-
- return NULL;
-}
+};
/** Start the consumer.
*/
@@ -611,25 +449,7 @@ static int start( mlt_consumer consumer )
// Get the properties
mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
DeckLinkConsumer* decklink = (DeckLinkConsumer*) consumer->child;
- int result = decklink->start( mlt_properties_get_int( properties, "preroll" ) ) ? 0 : 1;
-
- // Check that we're not already running
- if ( !result && !mlt_properties_get_int( properties, "running" ) )
- {
- // Allocate a thread
- pthread_t *pthread = (pthread_t*) calloc( 1, sizeof( pthread_t ) );
-
- // Assign the thread to properties
- mlt_properties_set_data( properties, "pthread", pthread, sizeof( pthread_t ), free, NULL );
-
- // Set the running state
- mlt_properties_set_int( properties, "running", 1 );
- mlt_properties_set_int( properties, "joined", 0 );
-
- // Create the thread
- pthread_create( pthread, NULL, run, consumer->child );
- }
- return result;
+ return decklink->start( mlt_properties_get_int( properties, "preroll" ) ) ? 0 : 1;
}
/** Stop the consumer.
@@ -638,26 +458,8 @@ static int start( mlt_consumer consumer )
static int stop( mlt_consumer consumer )
{
// Get the properties
- mlt_properties properties = MLT_CONSUMER_PROPERTIES( consumer );
-
- // Check that we're running
- if ( !mlt_properties_get_int( properties, "joined" ) )
- {
- // Get the thread
- pthread_t *pthread = (pthread_t*) mlt_properties_get_data( properties, "pthread", NULL );
-
- if ( pthread )
- {
- // Stop the thread
- mlt_properties_set_int( properties, "running", 0 );
- mlt_properties_set_int( properties, "joined", 1 );
-
- // Wait for termination
- pthread_join( *pthread, NULL );
- }
- }
-
- return 0;
+ DeckLinkConsumer* decklink = (DeckLinkConsumer*) consumer->child;
+ return decklink->stop();
}
/** Determine if the consumer is stopped.
--
1.7.4.4
------------------------------------------------------------------------------
EditLive Enterprise is the world's most technically advanced content
authoring tool. Experience the power of Track Changes, Inline Image
Editing and ensure content is compliant with Accessibility Checking.
http://p.sf.net/sfu/ephox-dev2dev
_______________________________________________
Mlt-devel mailing list
Mlt-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/mlt-devel
