On Wed, Jul 4, 2012 at 3:12 PM, Anton Khirnov <[email protected]> wrote:
> > Hi, > could you provide more complete code samples. Like what does your > get_audio_frame() look like (did you do any modifications to it?) or how > exactly are you calling it. > > -- > Anton Khirnov > Sure, no problem. Full code download here: *http://corp.productiveconcepts.com/qt-libav-recording-test-20120705.tar.gz* *Untar, cd into qtmultimedia/audio_inp_test, qmake and make, then run ./recorder. (yes, you must have the Qt SDK installed (>= ver 4.6.2, not tested with >= 5.0), and you must use qmake to generate the Makefile.) * If you just want to look at the code in context, I've attached the *very very messy* code (recorder.cpp) to this email. It's basically a hacked-up version output-example2 - scroll down to line 151 where get_audio_frame() starts. To answer your question, get_audio_frame() is called from write_audio_frame() (from the original output-example2 code) as "get_audio_frame(samples, audio_input_frame_size, c->channels);" And, yes - I made *extensive* modifications to get_audio_frame() to try to get it tied together with Qt's audio code. Since it's short, here's the get_audio_frame() code, below. *A quick walkthru:* * Basically, first it grabs a packet from the "buffer" (captured by Qt code in Recorder::sampleIO()) and plays back that audio packet over the speakers - this way, I know the capture code is working and the audio data is valid. * Then it pushes the audio data onto a "writeBuffer" (just a QByteArray) - since the frame_size*2 value is often smaller than amount of data in an AudioPacket, I must buffer the data for writing. * Next, if the writeBuffer isn't empty, it copies frame_size*2 bytes from the writeBuffer into the int16_t* samples pointer given to get_audio_frame() and removes the same number of bytes from the head of the writeBuffer. ** Bottom line problem description: Audio out while capturing plays correctly, but playing back the recorded "output.avi" is static taps at approx .25-.5 sec intervals. * static void get_audio_frame(int16_t *samples, int frame_size, int nb_channels) { if(!m_audioPackets.isEmpty()) { AudioPacket first = m_audioPackets.first(); int msecLatency = first.timestamp.msecsTo(QTime::currentTime()); qDebug() << "get_audio_frame(): playback: consumed packet, msecLatency:"<<msecLatency<<", buffer size:"<<m_audioPackets.size()<<", packet size:"<<first.data.size(); m_audioPackets.takeFirst(); outputDevice->write(first.data); writeBuffer.append(first.data); } if(!writeBuffer.isEmpty()) { int sz = writeBuffer.size(); char * data = writeBuffer.data(); // frame_size*2 becauase 2 bytes = 1 int16_t int dCounter = qMin(frame_size*2, sz); memcpy(samples, data, dCounter); writeBuffer.remove(0, dCounter); qDebug() << "get_audio_frame(): frame_size:"<<frame_size<<", nb_channels:"<<nb_channels<<", dCounter:"<<dCounter<<", sz:"<<sz; } else { qDebug() << "get_audio_frame(): ** BUFFER UNDERRUN **"; } } -- Josiah Bryan 765-215-0511 [email protected]
#include "recorder.h" #include <QDebug> #include <QApplication> //#include "../audio/qaudiodeviceinfo.h" //#include <QAudioDeviceInfo> #include "audio/qaudiodeviceinfo.h" #include "audio/qaudiodevicefactory_p.h" ///// from output-example2.c #include <stdlib.h> #include <stdio.h> #include <string.h> #include <math.h> #ifndef UINT64_C #define INT64_C(c) (c ## LL) #define UINT64_C(c) (c ## ULL) #endif // Defn from http://gcc.gnu.org/ml/gcc-bugs/2002-10/msg00259.html #ifndef INT64_C # define INT64_C(c) c ## LL #endif extern "C" { #include "libavformat/avformat.h" #include "libavutil/avutil.h" #include "libswscale/swscale.h" } #define AV_NOPTS_VALUE 0x8000000000000000LL #undef exit /* 5 seconds stream duration */ #define STREAM_DURATION 5.0 #define STREAM_FRAME_RATE 25 /* 25 images/s */ #define STREAM_NB_FRAMES ((int)(STREAM_DURATION * STREAM_FRAME_RATE)) #define STREAM_PIX_FMT PIX_FMT_YUV420P /* default pix_fmt */ static int sws_flags = SWS_BICUBIC; //////////// /**************************************************************/ /* audio output */ float t, tincr, tincr2; int16_t *samples; uint8_t *audio_outbuf; int audio_outbuf_size; int audio_input_frame_size; QBuffer buffer; QByteArray byteArray; QList<AudioPacket> m_audioPackets; QIODevice *outputDevice; QByteArray writeBuffer; const char *filename; AVOutputFormat *fmt; AVFormatContext *oc; AVStream *audio_st, *video_st; double audio_pts, video_pts; int i; /* * add an audio output stream */ static AVStream *add_audio_stream(AVFormatContext *oc, enum CodecID codec_id) { AVCodecContext *c; AVStream *st; st = av_new_stream(oc, 1); if (!st) { fprintf(stderr, "Could not alloc stream\n"); exit(1); } c = st->codec; c->codec_id = codec_id; c->codec_type = CODEC_TYPE_AUDIO; /* put sample parameters */ c->sample_fmt = SAMPLE_FMT_S16; c->bit_rate = 64000; c->sample_rate = 44100; c->channels = 1; // some formats want stream headers to be separate if(oc->oformat->flags & AVFMT_GLOBALHEADER) c->flags |= CODEC_FLAG_GLOBAL_HEADER; return st; } static void open_audio(AVFormatContext *oc, AVStream *st) { AVCodecContext *c; AVCodec *codec; c = st->codec; /* find the audio encoder */ codec = avcodec_find_encoder(c->codec_id); if (!codec) { fprintf(stderr, "codec not found\n"); exit(1); } /* open it */ if (avcodec_open(c, codec) < 0) { fprintf(stderr, "could not open codec\n"); exit(1); } /* init signal generator */ t = 0; tincr = 2 * M_PI * 110.0 / c->sample_rate; /* increment frequency by 110 Hz per second */ tincr2 = 2 * M_PI * 110.0 / c->sample_rate / c->sample_rate; audio_outbuf_size = 10000; audio_outbuf = (uint8_t*)av_malloc(audio_outbuf_size); /* ugly hack for PCM codecs (will be removed ASAP with new PCM support to compute the input frame size in samples */ if (c->frame_size <= 1) { audio_input_frame_size = audio_outbuf_size / c->channels; switch(st->codec->codec_id) { case CODEC_ID_PCM_S16LE: case CODEC_ID_PCM_S16BE: case CODEC_ID_PCM_U16LE: case CODEC_ID_PCM_U16BE: audio_input_frame_size >>= 1; break; default: break; } } else { audio_input_frame_size = c->frame_size; } samples = (int16_t*)av_malloc(audio_input_frame_size * 2 * c->channels); } /* prepare a 16 bit dummy audio frame of 'frame_size' samples and 'nb_channels' channels */ static void get_audio_frame(int16_t *samples, int frame_size, int nb_channels) { if(!m_audioPackets.isEmpty()) { AudioPacket first = m_audioPackets.first(); int msecLatency = first.timestamp.msecsTo(QTime::currentTime()); qDebug() << "get_audio_frame(): playback: consumed packet, msecLatency:"<<msecLatency/*<<", m_msecDelay:"<<m_msecDelay*/<<", buffer size:"<<m_audioPackets.size()<<", packet size:"<<first.data.size(); m_audioPackets.takeFirst(); outputDevice->write(first.data); writeBuffer.append(first.data); } if(!writeBuffer.isEmpty()) { int sz = writeBuffer.size(); char * data = writeBuffer.data(); // frame_size*2 becauase 2 bytes = 1 int16_t int dCounter = qMin(frame_size*2, sz); memcpy(samples, data, dCounter); writeBuffer.remove(0, dCounter); qDebug() << "get_audio_frame(): frame_size:"<<frame_size<<", nb_channels:"<<nb_channels<<", dCounter:"<<dCounter<<", sz:"<<sz; } else { qDebug() << "get_audio_frame(): ** BUFFER UNDERRUN **"; } } /* int j, i, v; int16_t *q; q = samples; for(j=0;j<frame_size;j++) { v = (int)(sin(t) * 10000); //v = 0; if(dCounter < sz - 2) { memcpy(&v, data, 2); *data ++; dCounter += 2; } for(i = 0; i < nb_channels; i++) *q++ = v; t += tincr; tincr += tincr2; }*/ // byteArray.remove(0, dCounter); // buffer.seek(dCounter); //printf("get_audio_frame(): frame_size:%d, nb_channels:%d, dCounter:%d, sz:%d, m_audioPackets.size(): %d\n", frame_size, nb_channels, dCounter, sz, m_audioPackets.size()); // exit(-1); static void write_audio_frame(AVFormatContext *oc, AVStream *st) { AVCodecContext *c; AVPacket pkt; av_init_packet(&pkt); c = st->codec; get_audio_frame(samples, audio_input_frame_size, c->channels); pkt.size= avcodec_encode_audio(c, audio_outbuf, audio_outbuf_size, samples); if (c->coded_frame && c->coded_frame->pts != AV_NOPTS_VALUE) pkt.pts= av_rescale_q(c->coded_frame->pts, c->time_base, st->time_base); pkt.flags |= PKT_FLAG_KEY; pkt.stream_index= st->index; pkt.data= audio_outbuf; /* write the compressed frame in the media file */ if (av_interleaved_write_frame(oc, &pkt) != 0) { fprintf(stderr, "Error while writing audio frame\n"); exit(1); } } static void close_audio(AVFormatContext *oc, AVStream *st) { avcodec_close(st->codec); av_free(samples); av_free(audio_outbuf); } /**************************************************************/ /* video output */ AVFrame *picture, *tmp_picture; uint8_t *video_outbuf; int frame_count, video_outbuf_size; /* add a video output stream */ static AVStream *add_video_stream(AVFormatContext *oc, enum CodecID codec_id) { AVCodecContext *c; AVStream *st; st = av_new_stream(oc, 0); if (!st) { fprintf(stderr, "Could not alloc stream\n"); exit(1); } c = st->codec; c->codec_id = codec_id; c->codec_type = CODEC_TYPE_VIDEO; /* put sample parameters */ c->bit_rate = 400000; /* resolution must be a multiple of two */ c->width = 352; c->height = 288; /* time base: this is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented. for fixed-fps content, timebase should be 1/framerate and timestamp increments should be identically 1. */ c->time_base.den = STREAM_FRAME_RATE; c->time_base.num = 1; c->gop_size = 12; /* emit one intra frame every twelve frames at most */ c->pix_fmt = STREAM_PIX_FMT; if (c->codec_id == CODEC_ID_MPEG2VIDEO) { /* just for testing, we also add B frames */ c->max_b_frames = 2; } if (c->codec_id == CODEC_ID_MPEG1VIDEO){ /* Needed to avoid using macroblocks in which some coeffs overflow. This does not happen with normal video, it just happens here as the motion of the chroma plane does not match the luma plane. */ c->mb_decision=2; } // some formats want stream headers to be separate if(oc->oformat->flags & AVFMT_GLOBALHEADER) c->flags |= CODEC_FLAG_GLOBAL_HEADER; return st; } static AVFrame *alloc_picture(enum PixelFormat pix_fmt, int width, int height) { AVFrame *picture; uint8_t *picture_buf; int size; picture = avcodec_alloc_frame(); if (!picture) return NULL; size = avpicture_get_size(pix_fmt, width, height); picture_buf = (uint8_t*)av_malloc(size); if (!picture_buf) { av_free(picture); return NULL; } avpicture_fill((AVPicture *)picture, picture_buf, pix_fmt, width, height); return picture; } static void open_video(AVFormatContext *oc, AVStream *st) { AVCodec *codec; AVCodecContext *c; c = st->codec; /* find the video encoder */ codec = avcodec_find_encoder(c->codec_id); if (!codec) { fprintf(stderr, "codec not found\n"); exit(1); } /* open the codec */ if (avcodec_open(c, codec) < 0) { fprintf(stderr, "could not open codec\n"); exit(1); } video_outbuf = NULL; if (!(oc->oformat->flags & AVFMT_RAWPICTURE)) { /* allocate output buffer */ /* XXX: API change will be done */ /* buffers passed into lav* can be allocated any way you prefer, as long as they're aligned enough for the architecture, and they're freed appropriately (such as using av_free for buffers allocated with av_malloc) */ video_outbuf_size = 200000; video_outbuf = (uint8_t*)av_malloc(video_outbuf_size); } /* allocate the encoded raw picture */ picture = alloc_picture(c->pix_fmt, c->width, c->height); if (!picture) { fprintf(stderr, "Could not allocate picture\n"); exit(1); } /* if the output format is not YUV420P, then a temporary YUV420P picture is needed too. It is then converted to the required output format */ tmp_picture = NULL; if (c->pix_fmt != PIX_FMT_YUV420P) { tmp_picture = alloc_picture(PIX_FMT_YUV420P, c->width, c->height); if (!tmp_picture) { fprintf(stderr, "Could not allocate temporary picture\n"); exit(1); } } } /* prepare a dummy image */ static void fill_yuv_image(AVFrame *pict, int frame_index, int width, int height) { int x, y, i; i = frame_index; /* Y */ for(y=0;y<height;y++) { for(x=0;x<width;x++) { pict->data[0][y * pict->linesize[0] + x] = x + y + i * 3; } } /* Cb and Cr */ for(y=0;y<height/2;y++) { for(x=0;x<width/2;x++) { pict->data[1][y * pict->linesize[1] + x] = 128 + y + i * 2; pict->data[2][y * pict->linesize[2] + x] = 64 + x + i * 5; } } } static void write_video_frame(AVFormatContext *oc, AVStream *st) { int out_size, ret; AVCodecContext *c; static struct SwsContext *img_convert_ctx; c = st->codec; if (frame_count >= STREAM_NB_FRAMES) { /* no more frame to compress. The codec has a latency of a few frames if using B frames, so we get the last frames by passing the same picture again */ } else { if (c->pix_fmt != PIX_FMT_YUV420P) { /* as we only generate a YUV420P picture, we must convert it to the codec pixel format if needed */ if (img_convert_ctx == NULL) { img_convert_ctx = sws_getContext(c->width, c->height, PIX_FMT_YUV420P, c->width, c->height, c->pix_fmt, sws_flags, NULL, NULL, NULL); if (img_convert_ctx == NULL) { fprintf(stderr, "Cannot initialize the conversion context\n"); exit(1); } } fill_yuv_image(tmp_picture, frame_count, c->width, c->height); sws_scale(img_convert_ctx, tmp_picture->data, tmp_picture->linesize, 0, c->height, picture->data, picture->linesize); } else { fill_yuv_image(picture, frame_count, c->width, c->height); } } if (oc->oformat->flags & AVFMT_RAWPICTURE) { /* raw video case. The API will change slightly in the near futur for that */ AVPacket pkt; av_init_packet(&pkt); pkt.flags |= PKT_FLAG_KEY; pkt.stream_index= st->index; pkt.data= (uint8_t *)picture; pkt.size= sizeof(AVPicture); ret = av_interleaved_write_frame(oc, &pkt); } else { /* encode the image */ out_size = avcodec_encode_video(c, video_outbuf, video_outbuf_size, picture); /* if zero size, it means the image was buffered */ if (out_size > 0) { AVPacket pkt; av_init_packet(&pkt); if (c->coded_frame->pts != AV_NOPTS_VALUE) pkt.pts= av_rescale_q(c->coded_frame->pts, c->time_base, st->time_base); if(c->coded_frame->key_frame) pkt.flags |= PKT_FLAG_KEY; pkt.stream_index= st->index; pkt.data= video_outbuf; pkt.size= out_size; /* write the compressed frame in the media file */ ret = av_interleaved_write_frame(oc, &pkt); } else { ret = 0; } } if (ret != 0) { fprintf(stderr, "Error while writing video frame\n"); exit(1); } frame_count++; } static void close_video(AVFormatContext *oc, AVStream *st) { avcodec_close(st->codec); av_free(picture->data[0]); av_free(picture); if (tmp_picture) { av_free(tmp_picture->data[0]); av_free(tmp_picture); } av_free(video_outbuf); } //////////////////////////// Recorder::Recorder() { //QList<QAudioDeviceInfo> list = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); QList<QAudioDeviceInfo> list = QAudioDeviceFactory::deviceList(QAudio::AudioInput); qDebug() << "Listing "<<list.size()<<" inputs..."; foreach(const QAudioDeviceInfo &deviceInfo, list) { qDebug() << "Device name: " << deviceInfo.deviceName(); QStringList codecs = deviceInfo.supportedCodecs(); qDebug() << " Supported Codecs: "<<codecs; } qDebug() << "Done."; //outputFile.setFileName("/test.raw"); //outputFile.open( QIODevice::WriteOnly | QIODevice::Truncate ); QAudioFormat format; format.setFrequency(44100); format.setChannels(1); format.setSampleSize(16); format.setCodec("audio/pcm"); format.setByteOrder(QAudioFormat::LittleEndian); format.setSampleType(QAudioFormat::UnSignedInt); qDebug() << "Going to create input..."; audio = new QAudioInput(QAudioDeviceInfo::defaultInputDevice(), format, this); //, format, this); qDebug() << "Created input..."; //QTimer::singleShot(10000, this, SLOT(stopRecording())); //audio->start(&outputFile); connect(&m_recordTimer, SIGNAL(timeout()), this, SLOT(sampleIO())); m_recordTimer.setInterval(60); m_recordTimer.start(); buffer.setBuffer(&byteArray); buffer.open(QBuffer::ReadWrite); //connect(&buffer, SIGNAL(bytesWritten()), this, SLOT(sampleIO())); audio->start(&buffer); //io = audio->start(); qDebug() << "Recording started."; // Records audio for 3000ms connect(&m_playbackTimer, SIGNAL(timeout()), this, SLOT(playback())); m_playbackTimer.setInterval(1000/50); m_playbackTimer.start(); // qDebug() << "Going to create output..."; audioOut = new QAudioOutput(QAudioDeviceInfo::defaultOutputDevice(), format, this); //, format, this); qDebug() << "Created output..."; connect(audioOut, SIGNAL(stateChanged(QAudio::State)), this, SLOT(outputStateChanged(QAudio::State))); outputDevice = audioOut->start(); //exit(-1); /// more output-example2.c code /* initialize libavcodec, and register all codecs and formats */ av_register_all(); int argc = 2; if (argc != 2) { printf("usage: <app name> output_file\n" "API example program to output a media file with libavformat.\n" "The output format is automatically guessed according to the file extension.\n" "Raw images can also be output by using '%%d' in the filename\n" "\n"); exit(1); } filename = "output.avi"; /* auto detect the output format from the name. default is mpeg. */ fmt = (AVOutputFormat*)guess_format(NULL, filename, NULL); if (!fmt) { printf("Could not deduce output format from file extension: using MPEG.\n"); fmt = (AVOutputFormat*)guess_format("mpeg", NULL, NULL); } if (!fmt) { fprintf(stderr, "Could not find suitable output format\n"); exit(1); } /* allocate the output media context */ oc = avformat_alloc_context(); if (!oc) { fprintf(stderr, "Memory error\n"); exit(1); } oc->oformat = fmt; snprintf(oc->filename, sizeof(oc->filename), "%s", filename); /* add the audio and video streams using the default format codecs and initialize the codecs */ video_st = NULL; audio_st = NULL; if (fmt->video_codec != CODEC_ID_NONE) { video_st = add_video_stream(oc, fmt->video_codec); } if (fmt->audio_codec != CODEC_ID_NONE) { audio_st = add_audio_stream(oc, fmt->audio_codec); } /* set the output parameters (must be done even if no parameters). */ if (av_set_parameters(oc, NULL) < 0) { fprintf(stderr, "Invalid output format parameters\n"); exit(1); } dump_format(oc, 0, filename, 1); /* now that all the parameters are set, we can open the audio and video codecs and allocate the necessary encode buffers */ if (video_st) open_video(oc, video_st); if (audio_st) open_audio(oc, audio_st); /* open the output file, if needed */ if (!(fmt->flags & AVFMT_NOFILE)) { if (url_fopen(&oc->pb, filename, URL_WRONLY) < 0) { fprintf(stderr, "Could not open '%s'\n", filename); exit(1); } } /* write the stream header, if any */ av_write_header(oc); } Recorder::~Recorder() { delete audio; audio = 0; } void Recorder::sampleIO() { /* char buf[4096]; //int read = io->read((char*)&buf,sizeof(buf)); //fwrite(buf, 8, read, stdout); int ba = io->bytesAvailable(); //QByteArray b = io->read(4096); //qDebug() << "SampleIO: ba:"<<ba<<", b:"<<b; int read = io->read( (char*)&buf, 4096); qDebug() << "**************** Bytes Read: "<<read; */ //int ba = buffer.bytesAvailable(); //QByteArray b = io->read(4096); //qDebug() << "SampleIO: data: "<<byteArray.size(); //byteArray.chop(byteArray.size()); int sz = byteArray.size(); int sum = 0; char * data = byteArray.data(); for(int i=0;i<sz;i++) { sum += (int)data[i]; } int avg = sz == 0 ? 0 : sum / sz; AudioPacket p; p.data = byteArray; p.timestamp = QTime::currentTime(); m_audioPackets.append(p); byteArray.clear(); buffer.seek(0); qDebug() << "SampleIO: bytes: "<<sz<<", avg:"<<avg; /* char ch; buffer.seek(0); buffer.getChar(&ch); // ch == 'Q' buffer.getChar(&ch); // ch == 't' buffer.getChar(&ch); // ch == ' ' buffer.getChar(&ch); // ch == 'r' */ } void Recorder::playback() { qDebug() << "'playback' slot: audio_pts: "<<audio_pts; //QApplication::processEvents(); /* compute current audio and video time */ if (audio_st) audio_pts = (double)audio_st->pts.val * audio_st->time_base.num / audio_st->time_base.den; else audio_pts = 0.0; if (video_st) video_pts = (double)video_st->pts.val * video_st->time_base.num / video_st->time_base.den; else video_pts = 0.0; if ((!audio_st || audio_pts >= STREAM_DURATION) && (!video_st || video_pts >= STREAM_DURATION)) { /* write the trailer, if any. the trailer must be written * before you close the CodecContexts open when you wrote the * header; otherwise write_trailer may try to use memory that * was freed on av_codec_close() */ av_write_trailer(oc); /* close each codec */ if (video_st) close_video(oc, video_st); if (audio_st) close_audio(oc, audio_st); /* free the streams */ for(i = 0; i < oc->nb_streams; i++) { av_freep(&oc->streams[i]->codec); av_freep(&oc->streams[i]); } if (!(fmt->flags & AVFMT_NOFILE)) { /* close the output file */ url_fclose(oc->pb); } /* free the stream */ av_free(oc); qDebug() << "Encoding done, exiting"; exit(0); } /* write interleaved audio and video frames */ if (!video_st || (video_st && audio_st && audio_pts < video_pts)) { write_audio_frame(oc, audio_st); } else { write_video_frame(oc, video_st); } //QApplication::processEvents(); // if(m_audioPackets.isEmpty()) // { // qDebug() << "playback: m_audioPackets is empty"; // return; // } // // m_msecDelay = 1000; // // bool done = false; // while(!done) // { // AudioPacket first = m_audioPackets.first(); // int msecLatency = first.timestamp.msecsTo(QTime::currentTime()); // // if(msecLatency < m_msecDelay) // { // qDebug() << "playback: holding packet, msecLatency:"<<msecLatency<<", m_msecDelay:"<<m_msecDelay<<", buffer size:"<<m_audioPackets.size(); // done = true; // continue; // } // // qDebug() << "playback: consumed packet, msecLatency:"<<msecLatency<<", m_msecDelay:"<<m_msecDelay<<", buffer size:"<<m_audioPackets.size(); // m_audioPackets.takeFirst(); // // outputDevice->write(first.data); // } } void Recorder::outputStateChanged(QAudio::State state) { qDebug() << "outputStateChanged: "<<state; } void Recorder::stopRecording() { audio->stop(); audioOut->stop(); //outputFile.close(); exit(-1); }
_______________________________________________ libav-api mailing list [email protected] https://lists.libav.org/mailman/listinfo/libav-api
