[FFmpeg-devel] [PATCH 2/2 v2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

From: Zhao, Halley halley.z...@intel.com

- do not support multi-thread decoding, it is unnecessary for hw
- create a decode thread to interface with yami decoding, decouple
  frame in and out
- the output frame type (raw data | drm handle | dmabuf) are specified
  in avctx-coder during init
- yami frame is assigned to AVFrame-buf[0], yami_recycle_frame() is
  registered to AVBufferRef. then it is recycle when AVFrame/AVBufferRef
  is unref'ed.
---
 libavcodec/libyami.cpp | 383 +
 1 file changed, 383 insertions(+)
 create mode 100644 libavcodec/libyami.cpp

diff --git a/libavcodec/libyami.cpp b/libavcodec/libyami.cpp
new file mode 100644
index 000..0d612e3
--- /dev/null
+++ b/libavcodec/libyami.cpp
@@ -0,0 +1,383 @@
+/*
+ * libyami.cpp -- h264 decoder uses libyami
+ *
+ *  Copyright (C) 2014 Intel Corporation
+ *Author: Zhao Halleyhalley.z...@intel.com
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include pthread.h
+#include unistd.h
+#include assert.h
+#include deque
+extern C {
+#include avcodec.h
+#include libavutil/imgutils.h
+#include internal.h
+}
+#include VideoDecoderHost.h
+
+using namespace YamiMediaCodec;
+#ifndef VA_FOURCC_I420
+#define VA_FOURCC_I420 VA_FOURCC('I','4','2','0')
+#endif
+#define PRINT_DECODE_THREAD(format, ...)  av_log(avctx, AV_LOG_VERBOSE, ## 
decode thread ## line:%4d  format, __LINE__, ##__VA_ARGS__)
+
+typedef enum {
+DECODE_THREAD_NOT_INIT = 0,
+DECODE_THREAD_RUNING,
+DECODE_THREAD_GOT_EOS,
+DECODE_THREAD_EXIT,
+} DecodeThreadStatus;
+
+struct YamiContext {
+AVCodecContext *avctx;
+pthread_mutex_t mutex_; // mutex for decoder-getOutput() and YamiContext 
itself update (decode_status, etc)
+
+IVideoDecoder *decoder;
+VideoDataMemoryType output_type;
+const VideoFormatInfo *format_info;
+pthread_t decode_thread_id;
+std::dequeVideoDecodeBuffer* *in_queue;
+pthread_mutex_t in_mutex; // mutex for in_queue
+pthread_cond_t in_cond;   // decode thread condition wait
+DecodeThreadStatus decode_status;
+
+// debug use
+int decode_count;
+int decode_count_yami;
+int render_count;
+};
+
+static av_cold int yami_init(AVCodecContext *avctx)
+{
+YamiContext *s = (YamiContext*)avctx-priv_data;
+Decode_Status status;
+
+av_log(avctx, AV_LOG_VERBOSE, yami_init\n);
+s-decoder = createVideoDecoder(video/h264);
+if (!s-decoder) {
+av_log(avctx, AV_LOG_ERROR, fail to create libyami h264 decoder\n);
+return -1;
+}
+
+NativeDisplay native_display;
+native_display.type = NATIVE_DISPLAY_DRM;
+native_display.handle = 0;
+s-decoder -setNativeDisplay(native_display);
+
+VideoConfigBuffer config_buffer;
+memset(config_buffer,0,sizeof(VideoConfigBuffer));
+if (avctx-extradata  avctx-extradata_size  avctx-extradata[0] == 1) 
{
+config_buffer.data = avctx-extradata;
+config_buffer.size = avctx-extradata_size;
+}
+config_buffer.profile = VAProfileNone;
+status = s-decoder-start(config_buffer);
+if (status != DECODE_SUCCESS) {
+av_log(avctx, AV_LOG_ERROR, yami h264 decoder fail to start\n);
+return -1;
+}
+
+switch (avctx-coder_type) {
+case 0:
+s-output_type = VIDEO_DATA_MEMORY_TYPE_RAW_POINTER;
+break;
+case 1:
+s-output_type = VIDEO_DATA_MEMORY_TYPE_DRM_NAME;
+break;
+case 2:
+s-output_type = VIDEO_DATA_MEMORY_TYPE_DMA_BUF;
+break;
+default:
+av_log(avctx, AV_LOG_ERROR, unknown output frame type: %d, 
avctx-coder_type);
+break;
+}
+
+s-in_queue = new std::dequeVideoDecodeBuffer*;
+pthread_mutex_init(s-mutex_, NULL);
+pthread_mutex_init(s-in_mutex, NULL);
+pthread_cond_init(s-in_cond, NULL);
+s-decode_status = DECODE_THREAD_NOT_INIT;
+s-decode_count = 0;
+s-decode_count_yami = 0;
+s-render_count = 0;
+
+return 0;
+}
+
+static void* decodeThread(void *arg)
+{
+AVCodecContext *avctx = (AVCodecContext*)arg;
+YamiContext *s = (YamiContext*)avctx-priv_data;
+
+while (1) {
+VideoDecodeBuffer *in_buffer = NULL;
+// deque one input buffer
+PRINT_DECODE_THREAD(decode thread runs one cycle

[FFmpeg-devel] [PATCH 0/2 v2] Add h264 decoder (hw acceleration) through libyami

[Zhao, Halley]

From: Zhao, Halley halley.z...@intel.com

libyami is a core codec library to support hw acceleration basing on vaapi:
https://github.com/01org/libyami

this patch add h264dec basing on libyami, similar to the solution basing on 
libstagefright.
moreover, it supports dma_buf and other frame mode, enables texture video 
easily.
here is an example player for texture video rendering uses ffmpeg/yami:
https://github.com/01org/player-ffmpeg-yami

Zhao, Halley (2):
  update configure etc to use libyami for h264 decode
  add libyami.cpp for h264 decoding by libyami

 configure  |  78 +++---
 libavcodec/Makefile|   1 +
 libavcodec/allcodecs.c |   1 +
 libavcodec/libyami.cpp | 383 +
 4 files changed, 442 insertions(+), 21 deletions(-)
 create mode 100644 libavcodec/libyami.cpp

-- 
1.8.3.2

___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

[FFmpeg-devel] [PATCH 1/2 v2] update configure etc to use libyami for h264 decode

[Zhao, Halley]

From: Zhao, Halley halley.z...@intel.com

add possible C++ based library dependency
add multiple packages dependency for one component
add h264dec basing on libyami
---
 configure  | 78 --
 libavcodec/Makefile|  1 +
 libavcodec/allcodecs.c |  1 +
 3 files changed, 59 insertions(+), 21 deletions(-)

diff --git a/configure b/configure
index a0dbafb..8131935 100755
--- a/configure
+++ b/configure
@@ -239,6 +239,7 @@ External library support:
   --enable-libspeexenable Speex de/encoding via libspeex [no]
   --enable-libssh  enable SFTP protocol via libssh [no]
   --enable-libstagefright-h264  enable H.264 decoding via libstagefright [no]
+  --enable-libyami-h264enable H.264 decoding via libyami [no]
   --enable-libtheora   enable Theora encoding via libtheora [no]
   --enable-libtwolame  enable MP2 encoding via libtwolame [no]
   --enable-libutvideo  enable Ut Video encoding and decoding via 
libutvideo [no]
@@ -1023,11 +1024,12 @@ int main(void){ return (int) foo; }
 EOF
 }
 
-check_func_headers(){
-log check_func_headers $@
-headers=$1
-funcs=$2
-shift 2
+check_func_headers_(){
+log check_func_headers_ $@
+compiler=$1
+headers=$2
+funcs=$3
+shift 3
 {
 for hdr in $headers; do
 print_include $hdr
@@ -1036,7 +1038,10 @@ check_func_headers(){
 echo long check_$func(void) { return (long) $func; }
 done
 echo int main(void) { return 0; }
-} | check_ld cc $@  enable $funcs  enable_safe $headers
+} | check_ld $compiler $@  enable $funcs  enable_safe $headers
+}
+check_func_headers() {
+   check_func_headers_ cc $@
 }
 
 check_class_headers_cpp(){
@@ -1095,20 +1100,26 @@ check_lib_cpp(){
 check_class_headers_cpp $headers $classes $@  add_extralibs $@
 }
 
-check_pkg_config(){
-log check_pkg_config $@
-pkgandversion=$1
-pkg=${1%% *}
-headers=$2
-funcs=$3
-shift 3
-check_cmd $pkg_config --exists --print-errors $pkgandversion || return
-pkg_cflags=$($pkg_config --cflags $pkg_config_flags $pkg)
-pkg_libs=$($pkg_config --libs $pkg_config_flags $pkg)
-check_func_headers $headers $funcs $pkg_cflags $pkg_libs $@ 
+check_pkg_config_(){
+log check_pkg_config_ $@
+compiler=$1
+## one component may depend on multiple pkgs
+allpkgs=$2
+pkg=${2%% *}
+headers=$3
+funcs=$4
+shift 4
+check_cmd $pkg_config --exists --print-errors $allpkgs || return
+pkg_cflags=$($pkg_config --cflags $pkg_config_flags $allpkgs)
+pkg_libs=$($pkg_config --libs $pkg_config_flags $allpkgs)
+log compiler: $compiler ; headers: $headers ; funcs: $funcs
+check_func_headers_ $compiler $headers $funcs $pkg_cflags $pkg_libs 
$@ 
 set_safe ${pkg}_cflags $pkg_cflags   
 set_safe ${pkg}_libs   $pkg_libs
 }
+check_pkg_config() {
+   check_pkg_config_ cc $@
+}
 
 check_exec(){
 check_ld cc $@  { enabled cross_compile || $TMPE  $logfile 21; }
@@ -1208,15 +1219,37 @@ require_cpp(){
 check_lib_cpp $headers $classes $@ || die ERROR: $name not found
 }
 
-use_pkg_config(){
-pkg=$1
-check_pkg_config $@ || return 1
+use_pkg_config_(){
+log use_pkg_config_: $@
+pkg=$2
+check_pkg_config_ $@ || return 1
 add_cflags$(get_safe ${pkg}_cflags)
 add_extralibs $(get_safe ${pkg}_libs)
 }
 
+use_pkg_config(){
+use_pkg_config_ cc $@
+}
+
+require_pkg_config_(){
+use_pkg_config_ $@ || die ERROR: $pkg not found using 
pkg-config$pkg_config_fail_message
+}
+
+cxx_extralibs_added=no
+add_cxx_extralibs(){
+if [ $cxx_extralibs_added = no ]; then
+add_extralibs -lstdc++ -Wl,--no-as-needed -ldl
+cxx_extralibs_added=yes
+fi
+}
+
 require_pkg_config(){
-use_pkg_config $@ || die ERROR: $pkg not found using 
pkg-config$pkg_config_fail_message
+   require_pkg_config_ cc $@
+}
+
+require_pkg_config_cxx(){
+   require_pkg_config_ cxx $@
+   add_cxx_extralibs
 }
 
 require_libfreetype(){
@@ -1381,6 +1414,7 @@ EXTERNAL_LIBRARY_LIST=
 libspeex
 libssh
 libstagefright_h264
+libyami_h264
 libtheora
 libtwolame
 libutvideo
@@ -2379,6 +2413,7 @@ libspeex_decoder_deps=libspeex
 libspeex_encoder_deps=libspeex
 libspeex_encoder_select=audio_frame_queue
 libstagefright_h264_decoder_deps=libstagefright_h264
+libyami_h264_decoder_deps=libyami_h264
 libtheora_encoder_deps=libtheora
 libtwolame_encoder_deps=libtwolame
 libvo_aacenc_encoder_deps=libvo_aacenc
@@ -4909,6 +4944,7 @@ enabled libspeex   require_pkg_config speex 
speex/speex.h speex_decode
 enabled libstagefright_h264  require_cpp libstagefright_h264 
binder/ProcessState.h media/stagefright/MetaData.h
 media/stagefright/MediaBufferGroup.h media/stagefright/MediaDebug.h 
media/stagefright/MediaDefs.h
 media/stagefright/OMXClient.h media/stagefright/OMXCodec.h 
android::OMXClient -lstagefright -lmedia -lutils -lbinder

Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

Thanks for your valued comments, I followed them in updated patch set.

 +config_buffer.profile = VAProfileNone;
 +status = s-decoder-start(config_buffer);
 +if (status != DECODE_SUCCESS) {
 +av_log(avctx, AV_LOG_ERROR, yami h264 decoder fail to start\n);
 +return -1;
 +}

 Does this check the codec profile and the hw support for it etc.?

Yes, it does.

 +if (DECODE_FORMAT_CHANGE == status) {
 +s-format_info = s-decoder-getFormatInfo();
 +PRINT_DECODE_THREAD(decode format change 
 %dx%d\n,s-format_info-width,s-format_info-height);
 +// resend the buffer
 +status = s-decoder-decode(in_buffer);
 +PRINT_DECODE_THREAD(decode() status=%d\n,status);
 +avctx-width = s-format_info-width;
 +avctx-height = s-format_info-height;
 +avctx-pix_fmt = AV_PIX_FMT_YUV420P;

 It writes to the global AVCodecContext without any form of synchronization?
So far, libyami supports avcC formatted codec data, not byte streamed SPS/PPS 
in decoder-start() yet.
After byte streamed sps/pps data is supported in decoder-start(), I will move 
format change check to yami_init.
Do you think it will fix the issue?

 +if (!s-decoder) // XXX, use shared pointer for s
 +return;
 +pthread_mutex_lock(s-mutex_);
 +s-decoder-renderDone(frame);
 +pthread_mutex_unlock(s-mutex_);
 +av_log(avctx, AV_LOG_DEBUG, recycle previous frame: %p\n, 
 +frame); }

 This fails if you keep a frame longer than the decoder. (Which worked with 
 _all_ other decoders so far.)
How about keep a reference of decoder in the video frame?

 +while (s-decode_status  DECODE_THREAD_GOT_EOS) { // we need 
 + enque eos buffer more than once

 Checking decode_status without a lock?
Yes, it seems not an issue.
We just try not to enque an empty buffer (for EOS) many times

 +pthread_mutex_lock(s-in_mutex);
 +if (s-in_queue-size()4) {
 +s-in_queue-push_back(in_buffer);
 +av_log(avctx, AV_LOG_VERBOSE, wakeup decode thread ...\n);
 +pthread_cond_signal(s-in_cond);
...
 +av_log(avctx, AV_LOG_DEBUG, s-in_queue-size()=%ld, 
 s-decode_count=%d, s-decode_count_yami=%d, too many buffer are under 
 decoding, wait ...\n,
 +s-in_queue-size(), s-decode_count, s-decode_count_yami);
 +usleep(1);

 No. Use condition variables.
I updated the patch to enque the input buffer unconditionally.

 Does it even need a decode thread?
Two threads help GPU runs in parallel

 +do {
 +if (!s-format_info) {
 +usleep(1);

 Nonsense again...
It waits during start until stream resolution/format are got, seems fine.

 +yami_frame-fourcc = VA_FOURCC_I420;
What about other pixel formats?
We support other formats (NV12/YUY2/RGBX etc), using GPU for color conversion 
automatically.

 +if (s-decode_status == DECODE_THREAD_GOT_EOS) {
 +usleep(1);
.
After got EOS, we try to wait until an available output frame, seems fine.

 +frame = (AVFrame*)data;
This assignment was already done above.
Thanks, I removed it in updated patch

 +frame-data[1] = (uint8_t*)yami_frame-pitch[0];
 Why not set linesize?
Thanks, I updated it in new patch

 +((AVFrame*)data)-extended_data = ((AVFrame*)data)-data;
 If you used the proper avframe functions, you wouldn't have to do this.
I don't know ffmpeg well, Which function do you mean to? 

 Also, for this frame no pixel format is set.
Add it, thanks.

 +frame-buf[0] = av_buffer_create((uint8_t*)yami_frame, 
 + sizeof(VideoFrameRawData), yami_recycle_frame, avctx, 0);
 Breaks refcounting of the YUV420P frame?
Sorry, not catch your meaning. Could you explain more?

___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

-Original Message-
From: ffmpeg-devel-boun...@ffmpeg.org [mailto:ffmpeg-devel-boun...@ffmpeg.org] 
On Behalf Of wm4
Sent: Friday, January 09, 2015 9:45 PM
To: ffmpeg-devel@ffmpeg.org
Subject: Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by 
libyami

On Fri,  9 Jan 2015 16:15:13 +0800
Zhao, Halley halley.z...@intel.com wrote:

 From: Zhao, Halley halley.z...@intel.com
 
 - do not support multi-thread decoding, it is unnecessary for hw
 - create a decode thread to interface with yami decoding, decouple
   frame in and out
 - the output frame type (raw data | drm handle | dmabuf) are specified
   in avctx-coder during init
 - yami frame is assigned to AVFrame-buf[0], yami_recycle_frame() is
   registered to AVBufferRef. then it is recycle when AVFrame/AVBufferRef
   is unref'ed.
 ---

What's the point of using this library, if ffmpeg already has most of the hw 
decoding infrastructure itself?

  libavcodec/libyami.cpp | 386 
 +
  1 file changed, 386 insertions

Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

Thanks your explanation.
1. I know our team/Intel plan to maintain this lib for the following years. I'm 
a developer, I can't promise more.
2. as to libxyz, it is some way like investment: Nobody is sure of the future, 
but make decision basing on investment and profit.
  A small yami wrapper enables rich codec on Intel platform, I think it deserve 
to try. Moreover, you can disable it easily if you don't like it.



-Original Message-
From: ffmpeg-devel-boun...@ffmpeg.org [mailto:ffmpeg-devel-boun...@ffmpeg.org] 
On Behalf Of Ronald S. Bultje
Sent: Monday, January 12, 2015 10:35 PM
To: FFmpeg development discussions and patches
Subject: Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by 
libyami

Hi,

On Mon, Jan 12, 2015 at 9:04 AM, wm4 nfx...@googlemail.com wrote:

 On Mon, 12 Jan 2015 08:24:35 -0500
 Ronald S. Bultje rsbul...@gmail.com wrote:

  Hi,
 
  On Mon, Jan 12, 2015 at 12:59 AM, Zhao, Halley 
  halley.z...@intel.com
  wrote:
 
   I understand you concern.
   The wrapper doesn't help much to ffmpeg itself, however, it 
   benefits
 much
   to the apps uses ffmpeg, to pick up hw capability for codec.
 
 
  So specifically, what are the features that you get with yaml that 
  ffmpeg doesn't already provide (that is: yaml is itself just a 
  wrapper for other stuff; what does it wrap that ffmpeg doesn't have 
  already?). I see h264enc/vp8dec/vp8enc/vp9dec/jpegdec/jpegenc vaapi. 
  Anything else? Why
 not
  just implement these in ffmpeg directly?

 From what I can see:

 - libyami doesn't require the relatively terrible boilerplate for
   hwaccel (creating the hw decoder yourself, maintaining a surface
   pool, etc.), making this lib very attractive for those who want
   something simple, and who possibly don't even want to depend on 
 ffmpeg
 - uses some new APIs (not supported by all drivers yet), which work
   around vaapi suckage, such as requiring a shared context. Namely, the
   dma_buf stuff.

 I wonder how well the latter actually works...


So I don't know if anyone remembers me ranting about libxyz suckage, but yami 
sounds a lot like xyz. So: what are the chances that five years down the line, 
this lib ends up being unmaintained, having security issues, portability issues 
etc., and us basically just being stuck with yet another xyz that doesn't 
really solve our issue so we have to re-do it internally anyway a few years 
down the line, wishing we had done so a few years earlier?

There's examples of good xyzs, e.g. x264 is a most wonderful software project; 
there's also examples of (imo) fails, such as librtmp. What guarantees do we 
have that yami is a good xyz?

Ronald
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

 Maintaining decoders is the point of this project.
Yes, I agree the core of ffmpeg is codec;
But, from the user, ffmpeg is usually treated as a light-weight media 
framework. Being a media framework, it is good to leverage hw codec in many 
cases.

 Besides, there are some more things missing from the patch.
What's the missing? I'm glad to hear it and to fix it.

 The libstagefright decoding wrapper is apparently a piece of shit. Who even 
 uses it?
I think libstagefright wrapper is good by giving more options to application, 
you dislike it, but other may do.
The bad of it is on buffer sharing; we solve it by dma_buf.

 I don't see any nice things.
The example player demonstrates how the AVFrame interface will be improved with 
dma_buf.
I do think dma_buf is a good buffer sharing mechanism to promote hw codec.
https://github.com/01org/player-ffmpeg-yami 

-Original Message-
From: ffmpeg-devel-boun...@ffmpeg.org [mailto:ffmpeg-devel-boun...@ffmpeg.org] 
On Behalf Of wm4
Sent: Saturday, January 10, 2015 6:30 AM
To: ffmpeg-devel@ffmpeg.org
Subject: Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by 
libyami

On Fri, 9 Jan 2015 17:17:00 -0500
compn te...@mi.rr.com wrote:

 On Fri, 9 Jan 2015 14:44:32 +0100
 wm4 nfx...@googlemail.com wrote:
 
  What's the point of using this library, if ffmpeg already has most 
  of the hw decoding infrastructure itself?
 
 probably so we dont have to maintain every single little hw decoding 
 and encoding feature , and libyami will do this for us.

Maintaining decoders is the point of this project.

Besides, there are some more things missing from the patch.

 same thing libstagefright does for us. so we dont have to test our 
 ffmpeg hw decoders against every single piece of hardware.

The libstagefright decoding wrapper is apparently a piece of shit. Who even 
uses it?

 the future plans for libyami are pretty nice, 
 https://www.mail-archive.com/libva@lists.freedesktop.org/msg02826.html

I don't see any nice things.
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

 There's already vaapi support in ffmpeg though. While I'm not opposed
 to improving the hw decoding situation in general. I think it'd be
 better to improve the ffmpeg code and API itself, instead of adding a
 wrapper for an API that behaves completely different.
I don't know current status of vaapi support in ffmpeg, the following codec 
are supported by libyami but may not in ffmpeg-vaapi yet:
h264enc/vp8dec/vp8enc/vp9dec/jpegdec/jpegenc

to be honest, libyami is core codec library for vaapi (especially on Intel
platform). The purpose is to not implement each codec for each media frameworks.
Libyami is used in chromeos, the same interface works on android as well; as 
talked
in 0.2.0 release, it can support gstreamer with least effort basing on dma_buf.


 At least export of metadata like profile, color matrix, and so on
 (there are a bunch of these, parsed from the h264 bitstream).
 
 There's also the question what happens if the hardware can't decode the
 given video.
Thanks, we will follow it next.

 
 Shouldn't this use vaapi surfaces (VASurfaceID)? I'm pretty sure this
 dma_buf stuff is not supported by all backends.
VASurfaceID is platform dependent opaque handle, depends on vaapi context to be 
used;
decoder and render are tightly coupled.

while dma_buf is Linux kernel buffer sharing mechanism, it can be better
inter-operated among camera/video/egl,opengl/opencl etc. in theory, the only 
dependency
is Linux kernel. dma_buf is not limited to Intel platform as well.
Vaapi API and Intel i965 driver starts supporting dma_buf half a year ago, I 
believe it is
the right direction to go.
By the way, 4 rendering-modes are supported in my example player; dma_buf is 
one of them.


 -Original Message-
 From: ffmpeg-devel-boun...@ffmpeg.org [mailto:ffmpeg-devel-
 boun...@ffmpeg.org] On Behalf Of wm4
 Sent: Monday, January 12, 2015 9:35 AM
 To: ffmpeg-devel@ffmpeg.org
 Subject: Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264
 decoding by libyami
 
 On Mon, 12 Jan 2015 00:46:26 +
 Zhao, Halley halley.z...@intel.com wrote:
 
   Maintaining decoders is the point of this project.
  Yes, I agree the core of ffmpeg is codec; But, from the user, ffmpeg
  is usually treated as a light-weight media framework. Being a media
 framework, it is good to leverage hw codec in many cases.
 
 There's already vaapi support in ffmpeg though. While I'm not opposed
 to improving the hw decoding situation in general. I think it'd be
 better to improve the ffmpeg code and API itself, instead of adding a
 wrapper for an API that behaves completely different.
 
   Besides, there are some more things missing from the patch.
  What's the missing? I'm glad to hear it and to fix it.
 
 At least export of metadata like profile, color matrix, and so on
 (there are a bunch of these, parsed from the h264 bitstream).
 
 There's also the question what happens if the hardware can't decode the
 given video.
 
   The libstagefright decoding wrapper is apparently a piece of shit.
 Who even uses it?
  I think libstagefright wrapper is good by giving more options to
 application, you dislike it, but other may do.
  The bad of it is on buffer sharing; we solve it by dma_buf.
 
 Shouldn't this use vaapi surfaces (VASurfaceID)? I'm pretty sure this
 dma_buf stuff is not supported by all backends.
 
   I don't see any nice things.
  The example player demonstrates how the AVFrame interface will be
 improved with dma_buf.
  I do think dma_buf is a good buffer sharing mechanism to promote hw
 codec.
  https://github.com/01org/player-ffmpeg-yami
 
 ___
 ffmpeg-devel mailing list
 ffmpeg-devel@ffmpeg.org
 http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

I understand you concern.
The wrapper doesn't help much to ffmpeg itself, however, it benefits much to 
the apps uses ffmpeg, to pick up hw capability for codec.


-Original Message-
From: ffmpeg-devel-boun...@ffmpeg.org [mailto:ffmpeg-devel-boun...@ffmpeg.org] 
On Behalf Of Ronald S. Bultje
Sent: Saturday, January 10, 2015 9:49 AM
To: FFmpeg development discussions and patches
Subject: Re: [FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by 
libyami

Hi,

On Fri, Jan 9, 2015 at 5:30 PM, wm4 nfx...@googlemail.com wrote:

 On Fri, 9 Jan 2015 17:17:00 -0500
 compn te...@mi.rr.com wrote:

  On Fri, 9 Jan 2015 14:44:32 +0100
  wm4 nfx...@googlemail.com wrote:
 
   What's the point of using this library, if ffmpeg already has most 
   of the hw decoding infrastructure itself?
 
  probably so we dont have to maintain every single little hw decoding 
  and encoding feature , and libyami will do this for us.

 Maintaining decoders is the point of this project.


Yeah, I also share this same concern. What's next, a gstreamer wrapper?

Ronald
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel
___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

[FFmpeg-devel] [PATCH 0/2] Add h264 decoder (hw acceleration) through libyami

[Zhao, Halley]

From: Zhao, Halley halley.z...@intel.com

libyami is a core codec library to support hw acceleration basing on vaapi:
https://github.com/01org/libyami

this patch add h264dec basing on libyami, similar to the solution basing on 
libstagefright.
moreover, it supports dma_buf and other frame mode, enables texture video 
easily.
here is an example player for texture video rendering uses ffmpeg/yami:
https://github.com/01org/player-ffmpeg-yami


Zhao, Halley (2):
  update configure etc to use libyami for h264 decode
  add libyami.cpp for h264 decoding by libyami

 configure  |  24 ++-
 libavcodec/Makefile|   1 +
 libavcodec/allcodecs.c |   1 +
 libavcodec/libyami.cpp | 386 +
 4 files changed, 407 insertions(+), 5 deletions(-)
 mode change 100644 = 100755 libavcodec/Makefile
 create mode 100644 libavcodec/libyami.cpp

-- 
1.8.3.2

___
ffmpeg-devel mailing list
ffmpeg-devel@ffmpeg.org
http://ffmpeg.org/mailman/listinfo/ffmpeg-devel

[FFmpeg-devel] [PATCH 2/2] add libyami.cpp for h264 decoding by libyami

[Zhao, Halley]

From: Zhao, Halley halley.z...@intel.com

- do not support multi-thread decoding, it is unnecessary for hw
- create a decode thread to interface with yami decoding, decouple
  frame in and out
- the output frame type (raw data | drm handle | dmabuf) are specified
  in avctx-coder during init
- yami frame is assigned to AVFrame-buf[0], yami_recycle_frame() is
  registered to AVBufferRef. then it is recycle when AVFrame/AVBufferRef
  is unref'ed.
---
 libavcodec/libyami.cpp | 386 +
 1 file changed, 386 insertions(+)
 create mode 100644 libavcodec/libyami.cpp

diff --git a/libavcodec/libyami.cpp b/libavcodec/libyami.cpp
new file mode 100644
index 000..e944cde
--- /dev/null
+++ b/libavcodec/libyami.cpp
@@ -0,0 +1,386 @@
+/*
+ * libyami.cpp -- h264 decoder uses libyami
+ *
+ *  Copyright (C) 2014 Intel Corporation
+ *Author: Zhao Halleyhalley.z...@intel.com
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include pthread.h
+#include unistd.h
+#include assert.h
+#include deque
+extern C {
+#include avcodec.h
+#include libavutil/imgutils.h
+#include internal.h
+}
+#include VideoDecoderHost.h
+
+using namespace YamiMediaCodec;
+#ifndef VA_FOURCC_I420
+#define VA_FOURCC_I420 VA_FOURCC('I','4','2','0')
+#endif
+#define PRINT_DECODE_THREAD(format, ...)  av_log(avctx, AV_LOG_VERBOSE, ## 
decode thread ## line:%4d  format, __LINE__, ##__VA_ARGS__)
+
+typedef enum {
+DECODE_THREAD_NOT_INIT = 0,
+DECODE_THREAD_RUNING,
+DECODE_THREAD_GOT_EOS,
+DECODE_THREAD_EXIT,
+} DecodeThreadStatus;
+
+struct YamiContext {
+AVCodecContext *avctx;
+pthread_mutex_t mutex_; // mutex for decoder-getOutput() and YamiContext 
itself update (decode_status, etc)
+
+IVideoDecoder *decoder;
+VideoDataMemoryType output_type;
+const VideoFormatInfo *format_info;
+pthread_t decode_thread_id;
+std::dequeVideoDecodeBuffer* *in_queue;
+pthread_mutex_t in_mutex; // mutex for in_queue
+pthread_cond_t in_cond;   // decode thread condition wait
+DecodeThreadStatus decode_status;
+
+// debug use
+int decode_count;
+int decode_count_yami;
+int render_count;
+};
+
+static av_cold int yami_init(AVCodecContext *avctx)
+{
+YamiContext *s = (YamiContext*)avctx-priv_data;
+Decode_Status status;
+
+av_log(avctx, AV_LOG_VERBOSE, yami_init\n);
+s-decoder = createVideoDecoder(video/h264);
+if (!s-decoder) {
+av_log(avctx, AV_LOG_ERROR, fail to create libyami h264 decoder\n);
+return -1;
+}
+
+NativeDisplay native_display;
+native_display.type = NATIVE_DISPLAY_DRM;
+native_display.handle = 0;
+s-decoder -setNativeDisplay(native_display);
+
+VideoConfigBuffer config_buffer;
+memset(config_buffer,0,sizeof(VideoConfigBuffer));
+if (avctx-extradata  avctx-extradata_size  avctx-extradata[0] == 1) 
{
+config_buffer.data = avctx-extradata;
+config_buffer.size = avctx-extradata_size;
+}
+config_buffer.profile = VAProfileNone;
+status = s-decoder-start(config_buffer);
+if (status != DECODE_SUCCESS) {
+av_log(avctx, AV_LOG_ERROR, yami h264 decoder fail to start\n);
+return -1;
+}
+
+switch (avctx-coder_type) {
+case 0:
+s-output_type = VIDEO_DATA_MEMORY_TYPE_RAW_POINTER;
+break;
+case 1:
+s-output_type = VIDEO_DATA_MEMORY_TYPE_DRM_NAME;
+break;
+case 2:
+s-output_type = VIDEO_DATA_MEMORY_TYPE_DMA_BUF;
+break;
+default:
+av_log(avctx, AV_LOG_ERROR, unknown output frame type: %d, 
avctx-coder_type);
+break;
+}
+
+s-in_queue = new std::dequeVideoDecodeBuffer*;
+pthread_mutex_init(s-mutex_, NULL);
+pthread_mutex_init(s-in_mutex, NULL);
+pthread_cond_init(s-in_cond, NULL);
+s-decode_status = DECODE_THREAD_NOT_INIT;
+s-decode_count = 0;
+s-decode_count_yami = 0;
+s-render_count = 0;
+
+return 0;
+}
+
+static void* decodeThread(void *arg)
+{
+AVCodecContext *avctx = (AVCodecContext*)arg;
+YamiContext *s = (YamiContext*)avctx-priv_data;
+
+while (1) {
+VideoDecodeBuffer *in_buffer = NULL;
+// deque one input buffer
+PRINT_DECODE_THREAD(decode thread runs one cycle

[FFmpeg-devel] [PATCH 1/2] update configure etc to use libyami for h264 decode

[Zhao, Halley]

From: Zhao, Halley halley.z...@intel.com

---
 configure  | 24 +++-
 libavcodec/Makefile|  1 +
 libavcodec/allcodecs.c |  1 +
 3 files changed, 21 insertions(+), 5 deletions(-)
 mode change 100644 = 100755 libavcodec/Makefile

diff --git a/configure b/configure
index d4a86c0..9c54ffc 100755
--- a/configure
+++ b/configure
@@ -236,6 +236,7 @@ External library support:
   --enable-libspeexenable Speex de/encoding via libspeex [no]
   --enable-libssh  enable SFTP protocol via libssh [no]
   --enable-libstagefright-h264  enable H.264 decoding via libstagefright [no]
+  --enable-libyami-h264enable H.264 decoding via libyami [no]
   --enable-libtheora   enable Theora encoding via libtheora [no]
   --enable-libtwolame  enable MP2 encoding via libtwolame [no]
   --enable-libutvideo  enable Ut Video encoding and decoding via 
libutvideo [no]
@@ -1031,7 +1032,8 @@ check_func_headers(){
 echo long check_$func(void) { return (long) $func; }
 done
 echo int main(void) { return 0; }
-} | check_ld cc $@  enable $funcs  enable_safe $headers
+## both C++/C header check goes here, use 'cxx'
+} | check_ld cxx $@  enable $funcs  enable_safe $headers
 }
 
 check_class_headers_cpp(){
@@ -1092,14 +1094,15 @@ check_lib_cpp(){
 
 check_pkg_config(){
 log check_pkg_config $@
-pkgandversion=$1
+## one component may depend on multiple pkgs
+allpkgs=$1
 pkg=${1%% *}
 headers=$2
 funcs=$3
 shift 3
-check_cmd $pkg_config --exists --print-errors $pkgandversion || return
-pkg_cflags=$($pkg_config --cflags $pkg_config_flags $pkg)
-pkg_libs=$($pkg_config --libs $pkg_config_flags $pkg)
+check_cmd $pkg_config --exists --print-errors $allpkgs || return
+pkg_cflags=$($pkg_config --cflags $pkg_config_flags $allpkgs)
+pkg_libs=$($pkg_config --libs $pkg_config_flags $allpkgs)
 check_func_headers $headers $funcs $pkg_cflags $pkg_libs $@ 
 set_safe ${pkg}_cflags $pkg_cflags   
 set_safe ${pkg}_libs   $pkg_libs
@@ -1210,6 +1213,14 @@ require_pkg_config(){
 add_extralibs $(get_safe ${pkg}_libs)
 }
 
+cxx_extralibs_added=no
+add_cxx_extralibs(){
+if [ $cxx_extralibs_added = no]; then
+add_extralibs -lstdc++ -Wl,--no-as-needed -ldl
+cxx_extralibs_added=yes
+fi
+}
+
 require_libfreetype(){
 log require_libfreetype $@
 pkg=freetype2
@@ -1371,6 +1382,7 @@ EXTERNAL_LIBRARY_LIST=
 libspeex
 libssh
 libstagefright_h264
+libyami_h264
 libtheora
 libtwolame
 libutvideo
@@ -2368,6 +2380,7 @@ libspeex_decoder_deps=libspeex
 libspeex_encoder_deps=libspeex
 libspeex_encoder_select=audio_frame_queue
 libstagefright_h264_decoder_deps=libstagefright_h264
+libyami_h264_decoder_deps=libyami_h264
 libtheora_encoder_deps=libtheora
 libtwolame_encoder_deps=libtwolame
 libvo_aacenc_encoder_deps=libvo_aacenc
@@ -4884,6 +4897,7 @@ enabled libspeex   require_pkg_config speex 
speex/speex.h speex_decode
 enabled libstagefright_h264  require_cpp libstagefright_h264 
binder/ProcessState.h media/stagefright/MetaData.h
 media/stagefright/MediaBufferGroup.h media/stagefright/MediaDebug.h 
media/stagefright/MediaDefs.h
 media/stagefright/OMXClient.h media/stagefright/OMXCodec.h 
android::OMXClient -lstagefright -lmedia -lutils -lbinder -lgnustl_static
+enabled libyami_h264   require_pkg_config libyami_decoder libva 
VideoDecoderHost.h createVideoDecoder  add_cxx_extralibs()
 enabled libtheora  require libtheora theora/theoraenc.h th_info_init 
-ltheoraenc -ltheoradec -logg
 enabled libtwolame require libtwolame twolame.h twolame_init 
-ltwolame 
  { check_lib twolame.h 
twolame_encode_buffer_float32_interleaved -ltwolame ||
diff --git a/libavcodec/Makefile b/libavcodec/Makefile
old mode 100644
new mode 100755
index fa0f53d..d8697ff
--- a/libavcodec/Makefile
+++ b/libavcodec/Makefile
@@ -747,6 +747,7 @@ OBJS-$(CONFIG_LIBSHINE_ENCODER)   += libshine.o
 OBJS-$(CONFIG_LIBSPEEX_DECODER)   += libspeexdec.o
 OBJS-$(CONFIG_LIBSPEEX_ENCODER)   += libspeexenc.o
 OBJS-$(CONFIG_LIBSTAGEFRIGHT_H264_DECODER)+= libstagefright.o
+OBJS-$(CONFIG_LIBYAMI_H264_DECODER)   += libyami.o
 OBJS-$(CONFIG_LIBTHEORA_ENCODER)  += libtheoraenc.o
 OBJS-$(CONFIG_LIBTWOLAME_ENCODER) += libtwolame.o
 OBJS-$(CONFIG_LIBUTVIDEO_DECODER) += libutvideodec.o
diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
index 0d39d33..beccc74 100644
--- a/libavcodec/allcodecs.c
+++ b/libavcodec/allcodecs.c
@@ -97,6 +97,7 @@ void avcodec_register_all(void)
 REGISTER_HWACCEL(WMV3_VDPAU,wmv3_vdpau);
 
 /* video codecs */
+REGISTER_DECODER(LIBYAMI_H264,  libyami_h264);
 REGISTER_ENCODER(A64MULTI,  a64multi);
 REGISTER_ENCODER(A64MULTI5, a64multi5);
 REGISTER_DECODER(AASC,  aasc);
-- 
1.8.3.2