On 20/02/17 06:42, wm4 wrote:
> On Sun, 19 Feb 2017 18:46:41 +0000
> Mark Thompson <[email protected]> wrote:
>
>> Supports all surface formats that the OpenCL hwcontext does, with
>> mapping in both direction.
>> ---
>> configure | 6 +
>> libavutil/hwcontext_opencl.c | 442
>> +++++++++++++++++++++++++++++++++++++++++++
>> 2 files changed, 448 insertions(+)
>>
>> diff --git a/configure b/configure
>> index 5b78131ab..a6933288b 100755
>> --- a/configure
>> +++ b/configure
>> @@ -1706,6 +1706,7 @@ HAVE_LIST="
>> $TYPES_LIST
>> dos_paths
>> dxva2_lib
>> + intel_opencl_vaapi
>> libc_msvcrt
>> MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS
>> sdl
>> @@ -4822,6 +4823,11 @@ enabled vaapi &&
>> enabled vaapi &&
>> check_lib vaapi_x11 "va/va.h va/va_x11.h" vaGetDisplay -lva -lva-x11
>> -lX11
>>
>> +if enabled opencl && enabled vaapi ; then
>> + check_type "CL/cl_intel.h" "clCreateImageFromFdINTEL_fn" &&
>> + enable intel_opencl_vaapi
>> +fi
>> +
>> enabled vdpau &&
>> check_cpp_condition vdpau/vdpau.h "defined
>> VDP_DECODER_PROFILE_MPEG4_PART2_ASP" ||
>> disable vdpau
>> diff --git a/libavutil/hwcontext_opencl.c b/libavutil/hwcontext_opencl.c
>> index 0b108b7fc..7d14052b8 100644
>> --- a/libavutil/hwcontext_opencl.c
>> +++ b/libavutil/hwcontext_opencl.c
>> @@ -29,6 +29,14 @@
>> #include "mem.h"
>> #include "pixdesc.h"
>>
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> +#include <unistd.h>
>> +#include <va/va.h>
>> +#include <va/va_drmcommon.h>
>> +#include <CL/cl_intel.h>
>> +#include "hwcontext_vaapi.h"
>> +#endif
>> +
>> // The maximum number of planes in an image. This must be structly
>> // less than AV_NUM_DATA_POINTERS because we place the whole-frame
>> // reference in a buffer entry after the final plane. For now, four
>> @@ -43,6 +51,12 @@ typedef struct OpenCLDeviceContext {
>> cl_command_queue command_queue;
>>
>> // Platform/device-specific functions.
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> + int vaapi_enabled;
>> + clGetMemObjectFdIntel_fn clGetMemObjectFdIntel;
>> + clCreateImageFromFdINTEL_fn clCreateImageFromFdINTEL;
>> + clCreateBufferFromFdINTEL_fn clCreateBufferFromFdINTEL;
>> +#endif
>> } OpenCLDeviceContext;
>>
>> static void opencl_error_callback(const char *errinfo,
>> @@ -279,6 +293,40 @@ static int opencl_device_init(AVHWDeviceContext *hwdev)
>> }
>> }
>>
>> +#define CL_FUNC(name, desc, failure) do { \
>> + ctx->name = clGetExtensionFunctionAddressForPlatform( \
>> + hwctx->platform_id, #name); \
>> + if (!ctx->name) { \
>> + av_log(hwdev, AV_LOG_VERBOSE, \
>> + desc " function not found (%s).\n", #name); \
>> + goto failure; \
>> + } else { \
>> + av_log(hwdev, AV_LOG_VERBOSE, \
>> + desc " function found (%s).\n", #name); \
>> + } \
>> + } while (0)
>> +
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> + {
>> + CL_FUNC(clGetMemObjectFdIntel,
>> + "Intel OpenCL to DRM mapping", no_vaapi);
>> + CL_FUNC(clCreateBufferFromFdINTEL,
>> + "Intel DRM to OpenCL buffer mapping", no_vaapi);
>> + CL_FUNC(clCreateImageFromFdINTEL,
>> + "Intel DRM to OpenCL image mapping", no_vaapi);
>> +
>> + ctx->vaapi_enabled = 1;
>> + if (0) {
>> + no_vaapi:
>> + av_log(hwdev, AV_LOG_ERROR, "VAAPI to OpenCL mapping "
>> + "not usable.\n");
>> + ctx->vaapi_enabled = 0;
>> + }
>> + }
>> +#endif
>> +
>> +#undef CL_FUNC
>> +
>> return 0;
>> }
>>
>> @@ -288,6 +336,38 @@ static int opencl_device_create(AVHWDeviceContext *ctx,
>> const char *device,
>> return opencl_device_create_internal(ctx, device, opts, flags, NULL);
>> }
>>
>> +static int opencl_device_derive(AVHWDeviceContext *ctx,
>> + AVHWDeviceContext *src_ctx,
>> + int flags)
>> +{
>> + int err;
>> + AVDictionary *opts = NULL;
>> + switch (src_ctx->type) {
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> + case AV_HWDEVICE_TYPE_VAAPI:
>> + // There is no specific derivation function here because surface
>> + // sharing just works via DRM fds and no special initialisation
>> + // is required in advance. However, we do still need to find
>> + // the Beignet ICD, so just create a reference to it by name.
>> + err = av_dict_set(&opts, "platform_vendor", "Intel", 0);
>> + if (err >= 0)
>> + err = av_dict_set(&opts, "platform_version", "beignet", 0);
>> + if (err >= 0)
>> + err = av_dict_set(&opts, "device_index", "0", 0);
>> + if (err >= 0)
>> + err = opencl_device_create(ctx, NULL, opts, 0);
>> + if (err >= 0)
>> + err = opencl_device_init(ctx);
>> + break;
>> +#endif
>> + default:
>> + err = AVERROR(ENOSYS);
>> + break;
>> + }
>> + av_dict_free(&opts);
>> + return err;
>> +}
>> +
>> static void opencl_device_uninit(AVHWDeviceContext *hwdev)
>> {
>> OpenCLDeviceContext *ctx = hwdev->internal->priv;
>> @@ -930,11 +1010,368 @@ fail:
>> return err;
>> }
>>
>> +
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> +
>> +typedef struct VAAPItoOpenCLMapping {
>> + VAImage va_image;
>> + VABufferInfo va_buffer_info;
>> +
>> + int nb_planes;
>> + cl_mem plane[MAX_PLANES];
>> +} VAAPItoOpenCLMapping;
>> +
>> +static void opencl_unmap_from_vaapi(AVHWFramesContext *src_fc,
>> + HWMapDescriptor *hwmap)
>> +{
>> + VAAPItoOpenCLMapping *mapping = hwmap->priv;
>> + AVVAAPIDeviceContext *src_dev = src_fc->device_ctx->hwctx;
>> + VASurfaceID surface_id;
>> + VAStatus vas;
>> + cl_int cle;
>> + int i;
>> +
>> + surface_id = (VASurfaceID)(uintptr_t)hwmap->source->data[3];
>> + av_log(src_fc, AV_LOG_DEBUG, "Unmap VAAPI surface %#x from OpenCL.\n",
>> + surface_id);
>> +
>> + for (i = 0; i < mapping->nb_planes; i++) {
>> + cle = clReleaseMemObject(mapping->plane[i]);
>> + if (cle != CL_SUCCESS) {
>> + av_log(src_fc, AV_LOG_ERROR, "Failed to release CL "
>> + "buffer of plane %d of VA image %#x (derived "
>> + "from surface %#x): %d.\n", i,
>> + mapping->va_image.buf, surface_id, cle);
>> + }
>> + }
>> +
>> + vas = vaReleaseBufferHandle(src_dev->display,
>> + mapping->va_image.buf);
>> + if (vas != VA_STATUS_SUCCESS) {
>> + av_log(src_fc, AV_LOG_ERROR, "Failed to release buffer "
>> + "handle of image %#x (derived from surface %#x): "
>> + "%d (%s).\n", mapping->va_image.buf, surface_id,
>> + vas, vaErrorStr(vas));
>> + }
>> +
>> + vas = vaDestroyImage(src_dev->display,
>> + mapping->va_image.image_id);
>> + if (vas != VA_STATUS_SUCCESS) {
>> + av_log(src_fc, AV_LOG_ERROR, "Failed to destroy image "
>> + "derived from surface %#x: %d (%s).\n",
>> + surface_id, vas, vaErrorStr(vas));
>> + }
>> +
>> + av_free(mapping);
>> +}
>> +
>> +static int opencl_map_from_vaapi(AVHWFramesContext *dst_fc, AVFrame *dst,
>> + const AVFrame *src, int flags)
>> +{
>> + AVHWFramesContext *src_fc =
>> + (AVHWFramesContext*)src->hw_frames_ctx->data;
>> + AVVAAPIDeviceContext *src_dev = src_fc->device_ctx->hwctx;
>> + AVOpenCLDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
>> + OpenCLDeviceContext *ctx = dst_fc->device_ctx->internal->priv;
>> + VAAPItoOpenCLMapping *mapping = NULL;
>> + VASurfaceID surface_id;
>> + VAStatus vas;
>> + cl_int cle;
>> + int err, i;
>> +
>> + surface_id = (VASurfaceID)(uintptr_t)src->data[3];
>> + av_log(src_fc, AV_LOG_DEBUG, "Map VAAPI surface %#x to OpenCL.\n",
>> + surface_id);
>> +
>> + mapping = av_mallocz(sizeof(*mapping));
>> + if (!mapping) {
>> + err = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + vas = vaDeriveImage(src_dev->display, surface_id,
>> + &mapping->va_image);
>> + if (vas != VA_STATUS_SUCCESS) {
>> + av_log(src_fc, AV_LOG_ERROR, "Failed to derive image from "
>> + "surface %#x: %d (%s).\n",
>> + surface_id, vas, vaErrorStr(vas));
>> + err = AVERROR(EIO);
>> + goto fail;
>> + }
>> +
>> + mapping->va_buffer_info.mem_type =
>> + VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME;
>> +
>> + vas = vaAcquireBufferHandle(src_dev->display,
>> + mapping->va_image.buf,
>> + &mapping->va_buffer_info);
>> + if (vas != VA_STATUS_SUCCESS) {
>> + av_log(src_fc, AV_LOG_ERROR, "Failed to get buffer "
>> + "handle from image %#x (derived from surface %#x): "
>> + "%d (%s).\n", mapping->va_image.buf, surface_id,
>> + vas, vaErrorStr(vas));
>> + vaDestroyImage(src_dev->display, mapping->va_image.buf);
>> + err = AVERROR(EIO);
>> + goto fail_derived;
>> + }
>> +
>> + av_log(dst_fc, AV_LOG_DEBUG, "DRM PRIME fd is %ld.\n",
>> + mapping->va_buffer_info.handle);
>> +
>> + mapping->nb_planes = mapping->va_image.num_planes;
>> + for (i = 0; i < mapping->nb_planes; i++) {
>> + cl_import_image_info_intel image_info = {
>> + .fd = mapping->va_buffer_info.handle,
>> + .size = mapping->va_buffer_info.mem_size,
>> + .type = CL_MEM_OBJECT_IMAGE2D,
>> + .offset = mapping->va_image.offsets[i],
>> + .row_pitch = mapping->va_image.pitches[i],
>> + };
>> + cl_image_desc image_desc;
>> +
>> + err = opencl_get_plane_format(src_fc->sw_format, i,
>> + mapping->va_image.width,
>> + mapping->va_image.height,
>> + &image_info.fmt,
>> + &image_desc);
>> + if (err < 0) {
>> + av_log(dst_fc, AV_LOG_ERROR, "VA %#x (derived from "
>> + "surface %#x) has invalid parameters: %d.\n",
>> + mapping->va_image.buf, surface_id, err);
>> + goto fail_mapped;
>> + }
>> + image_info.width = image_desc.image_width;
>> + image_info.height = image_desc.image_height;
>> +
>> + mapping->plane[i] =
>> + ctx->clCreateImageFromFdINTEL(dst_dev->context,
>> + &image_info, &cle);
>> + if (!mapping->plane[i]) {
>> + av_log(dst_fc, AV_LOG_ERROR, "Failed to create CL image "
>> + "from plane %d of VA image %#x (derived from "
>> + "surface %#x): %d.\n", i,
>> + mapping->va_image.buf, surface_id, cle);
>> + err = AVERROR(EIO);
>> + goto fail_mapped;
>> + }
>> +
>> + dst->data[i] = (uint8_t*)mapping->plane[i];
>> + dst->linesize[i] = mapping->va_image.pitches[i];
>> + }
>> +
>> + err = ff_hwframe_map_create(src->hw_frames_ctx,
>> + dst, src, &opencl_unmap_from_vaapi,
>> + mapping);
>> + if (err < 0)
>> + goto fail_mapped;
>> +
>> + dst->width = src->width;
>> + dst->height = src->height;
>> +
>> + return 0;
>> +
>> +fail_mapped:
>> + for (i = 0; i < mapping->nb_planes; i++) {
>> + if (mapping->plane[i])
>> + clReleaseMemObject(mapping->plane[i]);
>> + }
>> + vaReleaseBufferHandle(src_dev->display, mapping->va_image.buf);
>> +fail_derived:
>> + vaDestroyImage(src_dev->display, mapping->va_image.image_id);
>> +fail:
>> + av_freep(&mapping);
>> + return err;
>> +}
>> +
>> +typedef struct OpenCLtoVAAPIMapping {
>> + int fd;
>> +
>> + VASurfaceID surface_id;
>> +} OpenCLtoVAAPIMapping;
>> +
>> +static void opencl_unmap_to_vaapi(AVHWFramesContext *dst_fc,
>> + HWMapDescriptor *hwmap)
>> +{
>> + OpenCLtoVAAPIMapping *mapping = hwmap->priv;
>> + AVVAAPIDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
>> +
>> + av_log(dst_fc, AV_LOG_VERBOSE, "Unmap fd %d / surface %#x.\n",
>> + mapping->fd, mapping->surface_id);
>> +
>> + vaDestroySurfaces(dst_dev->display, &mapping->surface_id, 1);
>> +
>> + close(mapping->fd);
>> +
>> + av_free(mapping);
>> +}
>> +
>> +static int opencl_map_to_vaapi(AVHWFramesContext *src_fc, AVFrame *dst,
>> + const AVFrame *src, int flags)
>> +{
>> + AVOpenCLDeviceContext *src_dev = src_fc->device_ctx->hwctx;
>> + OpenCLDeviceContext *ctx = src_fc->device_ctx->internal->priv;
>> + AVHWFramesContext *dst_fc =
>> + (AVHWFramesContext*)dst->hw_frames_ctx->data;
>> + AVVAAPIDeviceContext *dst_dev = dst_fc->device_ctx->hwctx;
>> + OpenCLtoVAAPIMapping *mapping = NULL;
>> + AVBufferRef *obj_ref = NULL;
>> + VAStatus vas;
>> + unsigned long va_fourcc;
>> + cl_int cle;
>> + cl_mem obj;
>> + unsigned long buffer_handle;
>> + size_t offset;
>> + int err, plane, nb_planes;
>> +
>> + VASurfaceAttribExternalBuffers buffer_desc;
>> + VASurfaceAttrib attrs[2] = {
>> + {
>> + .type = VASurfaceAttribMemoryType,
>> + .flags = VA_SURFACE_ATTRIB_SETTABLE,
>> + .value.type = VAGenericValueTypeInteger,
>> + .value.value.i = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME,
>> + },
>> + {
>> + .type = VASurfaceAttribExternalBufferDescriptor,
>> + .flags = VA_SURFACE_ATTRIB_SETTABLE,
>> + .value.type = VAGenericValueTypePointer,
>> + .value.value.p = &buffer_desc,
>> + }
>> + };
>> +
>> + // This must have been created by opencl_get_buffer() above, so we
>> + // should have a data pointer for each plane with an associated
>> + // buffer reference, and then a whole-object reference in the next
>> + // buffer slot after that.
>> + for (plane = 0; plane < MAX_PLANES; plane++) {
>> + if (!src->data[plane])
>> + break;
>> + if (!src->buf[plane])
>> + return AVERROR(EINVAL);
>> + }
>> + if (plane == MAX_PLANES)
>> + return AVERROR(EINVAL);
>> + obj_ref = src->buf[nb_planes = plane];
>> + if (!obj_ref)
>> + return AVERROR(EINVAL);
>> + obj = (cl_mem)obj_ref->data;
>> + if (!obj)
>> + return AVERROR(EINVAL);
>> +
>> + switch (src_fc->sw_format) {
>> + case AV_PIX_FMT_YUV420P: va_fourcc = VA_FOURCC_IYUV; break;
>> + case AV_PIX_FMT_NV12: va_fourcc = VA_FOURCC_NV12; break;
>> + case AV_PIX_FMT_P010: va_fourcc = VA_FOURCC_P010; break;
>> + case AV_PIX_FMT_RGBA: va_fourcc = VA_FOURCC_RGBA; break;
>> + case AV_PIX_FMT_BGRA: va_fourcc = VA_FOURCC_BGRA; break;
>> + case AV_PIX_FMT_ARGB: va_fourcc = VA_FOURCC_ARGB; break;
>> + case AV_PIX_FMT_ABGR: va_fourcc = VA_FOURCC_ABGR; break;
>
> Don't we (or should we) have a table for mapping those somewhere?
Yes. I guess it could be an ff_* function in hwcontext_vaapi.c? (I've been
trying to avoid that sort of dependency, but maybe it doesn't matter. The
miscellaneous functions can just go at the bottom of hwcontext_internal.h.)
>> + default:
>> + av_log(src_fc, AV_LOG_ERROR, "Unsupported format %d for "
>> + "OpenCL to VAAPI mapping.\n", src_fc->sw_format);
>> + goto fail;
>> + }
>> +
>> + mapping = av_mallocz(sizeof(*mapping));
>> + if (!mapping) {
>> + err = AVERROR(ENOMEM);
>> + goto fail;
>> + }
>> +
>> + cle = ctx->clGetMemObjectFdIntel(src_dev->context, obj,
>> + &mapping->fd);
>> + if (cle != CL_SUCCESS) {
>> + av_log(src_fc, AV_LOG_ERROR, "Failed to get PRIME fd from "
>> + "CL buffer: %d.\n", cle);
>> + err = AVERROR(EIO);
>> + goto fail;
>> + }
>> +
>> + av_log(dst_fc, AV_LOG_DEBUG, "DRM PRIME fd is %d.\n",
>> + mapping->fd);
>> +
>> + buffer_handle = mapping->fd;
>> + buffer_desc.pixel_format = va_fourcc;
>> + buffer_desc.width = src_fc->width;
>> + buffer_desc.height = src_fc->height;
>> + buffer_desc.buffers = &buffer_handle;
>> + buffer_desc.num_buffers = 1;
>> +
>> + offset = 0;
>> + for (plane = 0;; plane++) {
>> + cl_image_format image_format;
>> + cl_image_desc image_desc;
>> +
>> + err = opencl_get_plane_format(src_fc->sw_format, plane,
>> + src_fc->width, src_fc->height,
>> + &image_format, &image_desc);
>> + if (err == AVERROR(ENOENT))
>> + break;
>> + if (err < 0)
>> + goto fail_mapped;
>> +
>> + buffer_desc.pitches[plane] = image_desc.image_row_pitch;
>> + buffer_desc.offsets[plane] = offset;
>> +
>> + offset += (image_desc.image_row_pitch *
>> + image_desc.image_height);
>> + }
>> + if (plane != nb_planes) {
>> + err = AVERROR(EINVAL);
>> + goto fail_mapped;
>> + }
>> +
>> + buffer_desc.num_planes = nb_planes;
>> + buffer_desc.data_size = offset;
>> +
>> + vas = vaCreateSurfaces(dst_dev->display,
>> + VA_RT_FORMAT_YUV420,
>> + src->width, src->height,
>> + &mapping->surface_id, 1,
>> + attrs, FF_ARRAY_ELEMS(attrs));
>> + if(vas != VA_STATUS_SUCCESS) {
>> + av_log(ctx, AV_LOG_ERROR, "Failed to create surface from "
>> + "DRM buffer: %d (%s).\n", vas, vaErrorStr(vas));
>> + err = AVERROR(EIO);
>> + goto fail_mapped;
>> + }
>> +
>> + err = ff_hwframe_map_create(dst->hw_frames_ctx,
>> + dst, src, &opencl_unmap_to_vaapi,
>> + mapping);
>> + if (err < 0)
>> + goto fail_created;
>> +
>> + dst->width = src->width;
>> + dst->height = src->height;
>> + dst->data[3] = (uint8_t*)(uintptr_t)mapping->surface_id;
>> +
>> + av_log(dst_fc, AV_LOG_DEBUG, "Mapped DRM PRIME fd %d to "
>> + "surface %#x.\n", mapping->fd, mapping->surface_id);
>> +
>> + return 0;
>> +
>> +fail_created:
>> + vaDestroySurfaces(dst_dev->display, &mapping->surface_id, 1);
>> +fail_mapped:
>> + close(mapping->fd);
>> +fail:
>> + av_freep(&mapping);
>> + return err;
>> +}
>> +
>> +#endif
>> +
>> +
>> static int opencl_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
>> const AVFrame *src, int flags)
>> {
>> av_assert0(dst->format == AV_PIX_FMT_OPENCL);
>> switch (src->format) {
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> + case AV_PIX_FMT_VAAPI:
>> + return opencl_map_from_vaapi(hwfc, dst, src, flags);
>> +#endif
>> default:
>> return AVERROR(ENOSYS);
>> }
>> @@ -945,6 +1382,10 @@ static int opencl_map_from(AVHWFramesContext *hwfc,
>> AVFrame *dst,
>> {
>> av_assert0(src->format == AV_PIX_FMT_OPENCL);
>> switch (dst->format) {
>> +#if HAVE_INTEL_OPENCL_VAAPI
>> + case AV_PIX_FMT_VAAPI:
>> + return opencl_map_to_vaapi(hwfc, dst, src, flags);
>> +#endif
>> default:
>> if (hwfc->sw_format != dst->format)
>> return AVERROR(ENOSYS);
>> @@ -961,6 +1402,7 @@ const HWContextType ff_hwcontext_type_opencl = {
>> .frames_hwctx_size = sizeof(AVOpenCLFramesContext),
>>
>> .device_create = &opencl_device_create,
>> + .device_derive = &opencl_device_derive,
>> .device_init = &opencl_device_init,
>> .device_uninit = &opencl_device_uninit,
>>
>
> Otherwise I can't comment much.
I'm not sure whether to be glad or not about that... (This is probably the
craziest patch in the series.)
_______________________________________________
libav-devel mailing list
[email protected]
https://lists.libav.org/mailman/listinfo/libav-devel
