Author: maskray Date: Mon May 6 02:24:36 2019 New Revision: 360022 URL: http://llvm.org/viewvc/llvm-project?rev=360022&view=rev Log: [X86] Move files to correct directories after D60552
Added: cfe/trunk/lib/Headers/avx512bf16intrin.h cfe/trunk/lib/Headers/avx512vlbf16intrin.h cfe/trunk/test/CodeGen/avx512bf16-builtins.c cfe/trunk/test/CodeGen/avx512vlbf16-builtins.c Added: cfe/trunk/lib/Headers/avx512bf16intrin.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/avx512bf16intrin.h?rev=360022&view=auto ============================================================================== --- cfe/trunk/lib/Headers/avx512bf16intrin.h (added) +++ cfe/trunk/lib/Headers/avx512bf16intrin.h Mon May 6 02:24:36 2019 @@ -0,0 +1,212 @@ +/*===------------ avx512bf16intrin.h - AVX512_BF16 intrinsics --------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ +#ifndef __IMMINTRIN_H +#error "Never use <avx512bf16intrin.h> directly; include <immintrin.h> instead." +#endif + +#ifndef __AVX512BF16INTRIN_H +#define __AVX512BF16INTRIN_H + +typedef short __m512bh __attribute__((__vector_size__(64), __aligned__(64))); +typedef short __m256bh __attribute__((__vector_size__(32), __aligned__(32))); + +#define __DEFAULT_FN_ATTRS512 \ + __attribute__((__always_inline__, __nodebug__, __target__("avx512bf16"), \ + __min_vector_width__(512))) + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [16 x float]. +/// \param __B +/// A 512-bit vector of [16 x float]. +/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from +/// convertion of src2, and higher 256 bits come from conversion of src1. +static __inline__ __m512bh __DEFAULT_FN_ATTRS512 +_mm512_cvtne2ps_pbh(__m512 __A, __m512 __B) { + return (__m512bh)__builtin_ia32_cvtne2ps2bf16_512((__v16sf) __A, + (__v16sf) __B); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [16 x float]. +/// \param __B +/// A 512-bit vector of [16 x float]. +/// \param __W +/// A 512-bit vector of [32 x bfloat]. +/// \param __U +/// An immediate value containing an 32-bit value specifying which element +/// is choosed. 1 means __A or __B, 0 means __W. +/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from +/// convertion of src2, and higher 256 bits come from conversion of src1. +static __inline__ __m512bh __DEFAULT_FN_ATTRS512 +_mm512_mask_cvtne2ps_pbh(__m512bh __W, __mmask32 __U, __m512 __A, __m512 __B) { + return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U, + (__v32hi)_mm512_cvtne2ps_pbh(__A, __B), + (__v32hi)__W); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [16 x float]. +/// \param __B +/// A 512-bit vector of [16 x float]. +/// \param __U +/// An immediate value containing an 32-bit value specifying which element +/// is choosed. 1 means __A or __B, 0 means zero. +/// \returns A 512-bit vector of [32 x bfloat] whose lower 256 bits come from +/// convertion of src2, and higher 256 bits come from conversion of src1. +static __inline__ __m512bh __DEFAULT_FN_ATTRS512 +_mm512_maskz_cvtne2ps_pbh(__mmask32 __U, __m512 __A, __m512 __B) { + return (__m512bh)__builtin_ia32_selectw_512((__mmask32)__U, + (__v32hi)_mm512_cvtne2ps_pbh(__A, __B), + (__v32hi)_mm512_setzero_si512()); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [16 x float]. +/// \returns A 256-bit vector of [16 x bfloat] come from convertion of src +static __inline__ __m256bh __DEFAULT_FN_ATTRS512 +_mm512_cvtneps_pbh(__m512 __A) { + return (__m256bh)__builtin_ia32_cvtneps2bf16_512((__v16sf) __A); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [16 x float]. +/// \param __W +/// A 256-bit vector of [16 x bfloat]. +/// \param __U +/// An immediate value containing an 16-bit value specifying which element +/// is choosed. 1 means __A, 0 means __W. +/// \returns A 256-bit vector of [16 x bfloat] come from convertion of src +static __inline__ __m256bh __DEFAULT_FN_ATTRS512 +_mm512_mask_cvtneps_pbh(__m256bh __W, __mmask16 __U, __m512 __A) { + return (__m256bh)__builtin_ia32_selectw_256((__mmask16)__U, + (__v16hi)_mm512_cvtneps_pbh(__A), + (__v16hi)__W); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [16 x float]. +/// \param __U +/// An immediate value containing an 16-bit value specifying which element +/// is choosed. 1 means __A, 0 means zero. +/// \returns A 256-bit vector of [16 x bfloat] come from convertion of src +static __inline__ __m256bh __DEFAULT_FN_ATTRS512 +_mm512_maskz_cvtneps_pbh(__mmask16 __U, __m512 __A) { + return (__m256bh)__builtin_ia32_selectw_256((__mmask16)__U, + (__v16hi)_mm512_cvtneps_pbh(__A), + (__v16hi)_mm256_setzero_si256()); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [32 x bfloat]. +/// \param __B +/// A 512-bit vector of [32 x bfloat]. +/// \param __D +/// A 512-bit vector of [16 x float]. +/// \returns A 512-bit vector of [16 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m512 __DEFAULT_FN_ATTRS512 +_mm512_dpbf16_ps(__m512 __D, __m512bh __A, __m512bh __B) { + return (__m512)__builtin_ia32_dpbf16ps_512((__v16sf) __D, + (__v16si) __A, + (__v16si) __B); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [32 x bfloat]. +/// \param __B +/// A 512-bit vector of [32 x bfloat]. +/// \param __D +/// A 512-bit vector of [16 x float]. +/// \param __U +/// An immediate value containing an 16-bit value specifying which element +/// is choosed. 1 means __A and __B's dot product, 0 means __D. +/// \returns A 512-bit vector of [16 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m512 __DEFAULT_FN_ATTRS512 +_mm512_mask_dpbf16_ps(__m512 __D, __mmask16 __U, __m512bh __A, __m512bh __B) { + return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, + (__v16sf)_mm512_dpbf16_ps(__D, __A, __B), + (__v16sf)__D); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 512-bit vector of [32 x bfloat]. +/// \param __B +/// A 512-bit vector of [32 x bfloat]. +/// \param __D +/// A 512-bit vector of [16 x float]. +/// \param __U +/// An immediate value containing an 16-bit value specifying which element +/// is choosed. 1 means __A and __B's dot product, 0 means 0. +/// \returns A 512-bit vector of [16 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m512 __DEFAULT_FN_ATTRS512 +_mm512_maskz_dpbf16_ps(__mmask16 __U, __m512 __D, __m512bh __A, __m512bh __B) { + return (__m512)__builtin_ia32_selectps_512((__mmask16)__U, + (__v16sf)_mm512_dpbf16_ps(__D, __A, __B), + (__v16sf)_mm512_setzero_si512()); +} + +#undef __DEFAULT_FN_ATTRS512 + +#endif Added: cfe/trunk/lib/Headers/avx512vlbf16intrin.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Headers/avx512vlbf16intrin.h?rev=360022&view=auto ============================================================================== --- cfe/trunk/lib/Headers/avx512vlbf16intrin.h (added) +++ cfe/trunk/lib/Headers/avx512vlbf16intrin.h Mon May 6 02:24:36 2019 @@ -0,0 +1,406 @@ +/*===--------- avx512vlbf16intrin.h - AVX512_BF16 intrinsics ---------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===-----------------------------------------------------------------------=== + */ +#ifndef __IMMINTRIN_H +#error "Never use <avx512vlbf16intrin.h> directly; include <immintrin.h> instead." +#endif + +#ifndef __AVX512VLBF16INTRIN_H +#define __AVX512VLBF16INTRIN_H + +typedef short __m128bh __attribute__((__vector_size__(16), __aligned__(16))); + +#define __DEFAULT_FN_ATTRS128 \ + __attribute__((__always_inline__, __nodebug__, \ + __target__("avx512vl, avx512bf16"), __min_vector_width__(128))) +#define __DEFAULT_FN_ATTRS256 \ + __attribute__((__always_inline__, __nodebug__, \ + __target__("avx512vl, avx512bf16"), __min_vector_width__(256))) + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [4 x float]. +/// \param __B +/// A 128-bit vector of [4 x float]. +/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from +/// convertion of src2, and higher 64 bits come from conversion of src1. +static __inline__ __m128bh __DEFAULT_FN_ATTRS128 +_mm_cvtne2ps_pbh(__m128 __A, __m128 __B) { + return (__m128bh)__builtin_ia32_cvtne2ps2bf16_128((__v4sf) __A, + (__v4sf) __B); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [4 x float]. +/// \param __B +/// A 128-bit vector of [4 x float]. +/// \param __W +/// A 128-bit vector of [8 x bfloat]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A or __B, 0 means __W. +/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from +/// convertion of src2, and higher 64 bits come from conversion of src1. +static __inline__ __m128bh __DEFAULT_FN_ATTRS128 +_mm_mask_cvtne2ps_pbh(__m128bh __W, __mmask8 __U, __m128 __A, __m128 __B) { + return (__m128bh)__builtin_ia32_selectw_128((__mmask8)__U, + (__v8hi)_mm_cvtne2ps_pbh(__A, __B), + (__v8hi)__W); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [4 x float]. +/// \param __B +/// A 128-bit vector of [4 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A or __B, 0 means zero. +/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from +/// convertion of src2, and higher 64 bits come from conversion of src1. +static __inline__ __m128bh __DEFAULT_FN_ATTRS128 +_mm_maskz_cvtne2ps_pbh(__mmask8 __U, __m128 __A, __m128 __B) { + return (__m128bh)__builtin_ia32_selectw_128((__mmask8)__U, + (__v8hi)_mm_cvtne2ps_pbh(__A, __B), + (__v8hi)_mm_setzero_si128()); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [8 x float]. +/// \param __B +/// A 256-bit vector of [8 x float]. +/// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from +/// convertion of src2, and higher 128 bits come from conversion of src1. +static __inline__ __m256bh __DEFAULT_FN_ATTRS256 +_mm256_cvtne2ps_pbh(__m256 __A, __m256 __B) { + return (__m256bh)__builtin_ia32_cvtne2ps2bf16_256((__v8sf) __A, + (__v8sf) __B); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [8 x float]. +/// \param __B +/// A 256-bit vector of [8 x float]. +/// \param __W +/// A 256-bit vector of [16 x bfloat]. +/// \param __U +/// An immediate value containing an 16-bit value specifying which element +/// is choosed. 1 means __A or __B, 0 means __W. +/// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from +/// convertion of src2, and higher 128 bits come from conversion of src1. +static __inline__ __m256bh __DEFAULT_FN_ATTRS256 +_mm256_mask_cvtne2ps_pbh(__m256bh __W, __mmask16 __U, __m256 __A, __m256 __B) { + return (__m256bh)__builtin_ia32_selectw_256((__mmask16)__U, + (__v16hi)_mm256_cvtne2ps_pbh(__A, __B), + (__v16hi)__W); +} + +/// Convert Two Packed Single Data to One Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNE2PS2BF16 </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [8 x float]. +/// \param __B +/// A 256-bit vector of [8 x float]. +/// \param __U +/// An immediate value containing an 16-bit value specifying which element +/// is choosed. 1 means __A or __B, 0 means zero. +/// \returns A 256-bit vector of [16 x bfloat] whose lower 128 bits come from +/// convertion of src2, and higher 128 bits come from conversion of src1. +static __inline__ __m256bh __DEFAULT_FN_ATTRS256 +_mm256_maskz_cvtne2ps_pbh(__mmask16 __U, __m256 __A, __m256 __B) { + return (__m256bh)__builtin_ia32_selectw_256((__mmask16)__U, + (__v16hi)_mm256_cvtne2ps_pbh(__A, __B), + (__v16hi)_mm256_setzero_si256()); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [4 x float]. +/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from +/// convertion of src, and higher 64 bits are 0. +static __inline__ __m128bh __DEFAULT_FN_ATTRS128 +_mm_cvtneps_pbh(__m128 __A) { + return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A, + (__v8hi)_mm_undefined_si128(), + (__mmask8)-1); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [4 x float]. +/// \param __W +/// A 128-bit vector of [8 x bfloat]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A, 0 means __W. +/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from +/// convertion of src, and higher 64 bits are 0. +static __inline__ __m128bh __DEFAULT_FN_ATTRS128 +_mm_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m128 __A) { + return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A, + (__v8hi)__W, + (__mmask8)__U); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [4 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A, 0 means 0. +/// \returns A 128-bit vector of [8 x bfloat] whose lower 64 bits come from +/// convertion of src, and higher 64 bits are 0. +static __inline__ __m128bh __DEFAULT_FN_ATTRS128 +_mm_maskz_cvtneps_pbh(__mmask8 __U, __m128 __A) { + return (__m128bh)__builtin_ia32_cvtneps2bf16_128_mask((__v4sf) __A, + (__v8hi)_mm_setzero_si128(), + (__mmask8)__U); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [8 x float]. +/// \returns A 128-bit vector of [8 x bfloat] comes from convertion of src. +static __inline__ __m128bh __DEFAULT_FN_ATTRS256 +_mm256_cvtneps_pbh(__m256 __A) { + return (__m128bh)__builtin_ia32_cvtneps2bf16_256((__v8sf)__A); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [8 x float]. +/// \param __W +/// A 256-bit vector of [8 x bfloat]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A, 0 means __W. +/// \returns A 128-bit vector of [8 x bfloat] comes from convertion of src. +static __inline__ __m128bh __DEFAULT_FN_ATTRS256 +_mm256_mask_cvtneps_pbh(__m128bh __W, __mmask8 __U, __m256 __A) { + return (__m128bh)__builtin_ia32_selectw_128((__mmask8)__U, + (__v8hi)_mm256_cvtneps_pbh(__A), + (__v8hi)__W); +} + +/// Convert Packed Single Data to Packed BF16 Data. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VCVTNEPS2BF16 </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [8 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A, 0 means __W. +/// \returns A 128-bit vector of [8 x bfloat] comes from convertion of src. +static __inline__ __m128bh __DEFAULT_FN_ATTRS256 +_mm256_maskz_cvtneps_pbh(__mmask8 __U, __m256 __A) { + return (__m128bh)__builtin_ia32_selectw_128((__mmask8)__U, + (__v8hi)_mm256_cvtneps_pbh(__A), + (__v8hi)_mm_setzero_si128()); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [8 x bfloat]. +/// \param __B +/// A 128-bit vector of [8 x bfloat]. +/// \param __D +/// A 128-bit vector of [4 x float]. +/// \returns A 128-bit vector of [4 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m128 __DEFAULT_FN_ATTRS128 +_mm_dpbf16_ps(__m128 __D, __m128bh __A, __m128bh __B) { + return (__m128)__builtin_ia32_dpbf16ps_128((__v4sf)__D, + (__v4si)__A, + (__v4si)__B); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [8 x bfloat]. +/// \param __B +/// A 128-bit vector of [8 x bfloat]. +/// \param __D +/// A 128-bit vector of [4 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A and __B's dot product, 0 means __D. +/// \returns A 128-bit vector of [4 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m128 __DEFAULT_FN_ATTRS128 +_mm_mask_dpbf16_ps(__m128 __D, __mmask8 __U, __m128bh __A, __m128bh __B) { + return (__m128)__builtin_ia32_selectps_128((__mmask8)__U, + (__v4sf)_mm_dpbf16_ps(__D, __A, __B), + (__v4sf)__D); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 128-bit vector of [8 x bfloat]. +/// \param __B +/// A 128-bit vector of [8 x bfloat]. +/// \param __D +/// A 128-bit vector of [4 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A and __B's dot product, 0 means 0. +/// \returns A 128-bit vector of [4 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m128 __DEFAULT_FN_ATTRS128 +_mm_maskz_dpbf16_ps(__mmask8 __U, __m128 __D, __m128bh __A, __m128bh __B) { + return (__m128)__builtin_ia32_selectps_128((__mmask8)__U, + (__v4sf)_mm_dpbf16_ps(__D, __A, __B), + (__v4sf)_mm_setzero_si128()); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [16 x bfloat]. +/// \param __B +/// A 256-bit vector of [16 x bfloat]. +/// \param __D +/// A 256-bit vector of [8 x float]. +/// \returns A 256-bit vector of [8 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m256 __DEFAULT_FN_ATTRS256 +_mm256_dpbf16_ps(__m256 __D, __m256bh __A, __m256bh __B) { + return (__m256)__builtin_ia32_dpbf16ps_256((__v8sf)__D, + (__v8si)__A, + (__v8si)__B); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [16 x bfloat]. +/// \param __B +/// A 256-bit vector of [16 x bfloat]. +/// \param __D +/// A 256-bit vector of [8 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A and __B's dot product, 0 means __D. +/// \returns A 256-bit vector of [8 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m256 __DEFAULT_FN_ATTRS256 +_mm256_mask_dpbf16_ps(__m256 __D, __mmask8 __U, __m256bh __A, __m256bh __B) { + return (__m256)__builtin_ia32_selectps_256((__mmask8)__U, + (__v8sf)_mm256_dpbf16_ps(__D, __A, __B), + (__v8sf)__D); +} + +/// Dot Product of BF16 Pairs Accumulated into Packed Single Precision. +/// +/// \headerfile <x86intrin.h> +/// +/// This intrinsic corresponds to the <c> VDPBF16PS </c> instructions. +/// +/// \param __A +/// A 256-bit vector of [16 x bfloat]. +/// \param __B +/// A 256-bit vector of [16 x bfloat]. +/// \param __D +/// A 256-bit vector of [8 x float]. +/// \param __U +/// An immediate value containing an 8-bit value specifying which element +/// is choosed. 1 means __A and __B's dot product, 0 means 0. +/// \returns A 256-bit vector of [8 x float] comes from Dot Product of +/// __A, __B and __D +static __inline__ __m256 __DEFAULT_FN_ATTRS256 +_mm256_maskz_dpbf16_ps(__mmask8 __U, __m256 __D, __m256bh __A, __m256bh __B) { + return (__m256)__builtin_ia32_selectps_256((__mmask8)__U, + (__v8sf)_mm256_dpbf16_ps(__D, __A, __B), + (__v8sf)_mm256_setzero_si256()); +} +#undef __DEFAULT_FN_ATTRS128 +#undef __DEFAULT_FN_ATTRS256 + +#endif Added: cfe/trunk/test/CodeGen/avx512bf16-builtins.c URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CodeGen/avx512bf16-builtins.c?rev=360022&view=auto ============================================================================== --- cfe/trunk/test/CodeGen/avx512bf16-builtins.c (added) +++ cfe/trunk/test/CodeGen/avx512bf16-builtins.c Mon May 6 02:24:36 2019 @@ -0,0 +1,74 @@ +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin \ +// RUN: -target-feature +avx512bf16 -emit-llvm -o - -Wall -Werror \ +// RUN: | FileCheck %s + +#include <immintrin.h> + +__m512bh test_mm512_cvtne2ps2bf16(__m512 A, __m512 B) { + // CHECK-LABEL: @test_mm512_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512 + // CHECK: ret <32 x i16> %{{.*}} + return _mm512_cvtne2ps_pbh(A, B); +} + +__m512bh test_mm512_maskz_cvtne2ps2bf16(__m512 A, __m512 B, __mmask32 U) { + // CHECK-LABEL: @test_mm512_maskz_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512 + // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}} + // CHECK: ret <32 x i16> %{{.*}} + return _mm512_maskz_cvtne2ps_pbh(U, A, B); +} + +__m512bh test_mm512_mask_cvtne2ps2bf16(__m512bh C, __mmask32 U, __m512 A, __m512 B) { + // CHECK-LABEL: @test_mm512_mask_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512 + // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}} + // CHECK: ret <32 x i16> %{{.*}} + return _mm512_mask_cvtne2ps_pbh(C, U, A, B); +} + +__m256bh test_mm512_cvtneps2bf16(__m512 A) { + // CHECK-LABEL: @test_mm512_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512 + // CHECK: ret <16 x i16> %{{.*}} + return _mm512_cvtneps_pbh(A); +} + +__m256bh test_mm512_mask_cvtneps2bf16(__m256bh C, __mmask16 U, __m512 A) { + // CHECK-LABEL: @test_mm512_mask_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512 + // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}} + // CHECK: ret <16 x i16> %{{.*}} + return _mm512_mask_cvtneps_pbh(C, U, A); +} + +__m256bh test_mm512_maskz_cvtneps2bf16(__m512 A, __mmask16 U) { + // CHECK-LABEL: @test_mm512_maskz_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.512 + // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}} + // CHECK: ret <16 x i16> %{{.*}} + return _mm512_maskz_cvtneps_pbh(U, A); +} + +__m512 test_mm512_dpbf16_ps(__m512 D, __m512bh A, __m512bh B) { + // CHECK-LABEL: @test_mm512_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512 + // CHECK: ret <16 x float> %{{.*}} + return _mm512_dpbf16_ps(D, A, B); +} + +__m512 test_mm512_maskz_dpbf16_ps(__m512 D, __m512bh A, __m512bh B, __mmask16 U) { + // CHECK-LABEL: @test_mm512_maskz_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512 + // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} + // CHECK: ret <16 x float> %{{.*}} + return _mm512_maskz_dpbf16_ps(U, D, A, B); +} + +__m512 test_mm512_mask_dpbf16_ps(__m512 D, __m512bh A, __m512bh B, __mmask16 U) { + // CHECK-LABEL: @test_mm512_mask_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.512 + // CHECK: select <16 x i1> %{{.*}}, <16 x float> %{{.*}}, <16 x float> %{{.*}} + // CHECK: ret <16 x float> %{{.*}} + return _mm512_mask_dpbf16_ps(D, U, A, B); +} Added: cfe/trunk/test/CodeGen/avx512vlbf16-builtins.c URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CodeGen/avx512vlbf16-builtins.c?rev=360022&view=auto ============================================================================== --- cfe/trunk/test/CodeGen/avx512vlbf16-builtins.c (added) +++ cfe/trunk/test/CodeGen/avx512vlbf16-builtins.c Mon May 6 02:24:36 2019 @@ -0,0 +1,163 @@ +// RUN: %clang_cc1 -ffreestanding %s -triple=x86_64-apple-darwin \ +// RUN: -target-feature +avx512bf16 -target-feature \ +// RUN: +avx512vl -emit-llvm -o - -Wall -Werror | FileCheck %s + +#include <immintrin.h> + +__m128bh test_mm_cvtne2ps2bf16(__m128 A, __m128 B) { + // CHECK-LABEL: @test_mm_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.128 + // CHECK: ret <8 x i16> %{{.*}} + return _mm_cvtne2ps_pbh(A, B); +} + +__m128bh test_mm_maskz_cvtne2ps2bf16(__m128 A, __m128 B, __mmask8 U) { + // CHECK-LABEL: @test_mm_maskz_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.128 + // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}} + // CHECK: ret <8 x i16> %{{.*}} + return _mm_maskz_cvtne2ps_pbh(U, A, B); +} + +__m128bh test_mm_mask_cvtne2ps2bf16(__m128bh C, __mmask8 U, __m128 A, __m128 B) { + // CHECK-LABEL: @test_mm_mask_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.128 + // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}} + // CHECK: ret <8 x i16> %{{.*}} + return _mm_mask_cvtne2ps_pbh(C, U, A, B); +} + +__m256bh test_mm256_cvtne2ps2bf16(__m256 A, __m256 B) { + // CHECK-LABEL: @test_mm256_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.256 + // CHECK: ret <16 x i16> %{{.*}} + return _mm256_cvtne2ps_pbh(A, B); +} + +__m256bh test_mm256_maskz_cvtne2ps2bf16(__m256 A, __m256 B, __mmask16 U) { + // CHECK-LABEL: @test_mm256_maskz_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.256 + // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}} + // CHECK: ret <16 x i16> %{{.*}} + return _mm256_maskz_cvtne2ps_pbh(U, A, B); +} + +__m256bh test_mm256_mask_cvtne2ps2bf16(__m256bh C, __mmask16 U, __m256 A, __m256 B) { + // CHECK-LABEL: @test_mm256_mask_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.256 + // CHECK: select <16 x i1> %{{.*}}, <16 x i16> %{{.*}}, <16 x i16> %{{.*}} + // CHECK: ret <16 x i16> %{{.*}} + return _mm256_mask_cvtne2ps_pbh(C, U, A, B); +} + +__m512bh test_mm512_cvtne2ps2bf16(__m512 A, __m512 B) { + // CHECK-LABEL: @test_mm512_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512 + // CHECK: ret <32 x i16> %{{.*}} + return _mm512_cvtne2ps_pbh(A, B); +} + +__m512bh test_mm512_maskz_cvtne2ps2bf16(__m512 A, __m512 B, __mmask32 U) { + // CHECK-LABEL: @test_mm512_maskz_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512 + // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}} + // CHECK: ret <32 x i16> %{{.*}} + return _mm512_maskz_cvtne2ps_pbh(U, A, B); +} + +__m512bh test_mm512_mask_cvtne2ps2bf16(__m512bh C, __mmask32 U, __m512 A, __m512 B) { + // CHECK-LABEL: @test_mm512_mask_cvtne2ps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtne2ps2bf16.512 + // CHECK: select <32 x i1> %{{.*}}, <32 x i16> %{{.*}}, <32 x i16> %{{.*}} + // CHECK: ret <32 x i16> %{{.*}} + return _mm512_mask_cvtne2ps_pbh(C, U, A, B); +} + +__m128bh test_mm_cvtneps2bf16(__m128 A) { + // CHECK-LABEL: @test_mm_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.mask.cvtneps2bf16.128 + // CHECK: ret <8 x i16> %{{.*}} + return _mm_cvtneps_pbh(A); +} + +__m128bh test_mm_mask_cvtneps2bf16(__m128bh C, __mmask8 U, __m128 A) { + // CHECK-LABEL: @test_mm_mask_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.mask.cvtneps2bf16. + // CHECK: ret <8 x i16> %{{.*}} + return _mm_mask_cvtneps_pbh(C, U, A); +} + +__m128bh test_mm_maskz_cvtneps2bf16(__m128 A, __mmask8 U) { + // CHECK-LABEL: @test_mm_maskz_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.mask.cvtneps2bf16.128 + // CHECK: ret <8 x i16> %{{.*}} + return _mm_maskz_cvtneps_pbh(U, A); +} + +__m128bh test_mm256_cvtneps2bf16(__m256 A) { + // CHECK-LABEL: @test_mm256_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.256 + // CHECK: ret <8 x i16> %{{.*}} + return _mm256_cvtneps_pbh(A); +} + +__m128bh test_mm256_mask_cvtneps2bf16(__m128bh C, __mmask8 U, __m256 A) { + // CHECK-LABEL: @test_mm256_mask_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.256 + // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}} + // CHECK: ret <8 x i16> %{{.*}} + return _mm256_mask_cvtneps_pbh(C, U, A); +} + +__m128bh test_mm256_maskz_cvtneps2bf16(__m256 A, __mmask8 U) { + // CHECK-LABEL: @test_mm256_maskz_cvtneps2bf16 + // CHECK: @llvm.x86.avx512bf16.cvtneps2bf16.256 + // CHECK: select <8 x i1> %{{.*}}, <8 x i16> %{{.*}}, <8 x i16> %{{.*}} + // CHECK: ret <8 x i16> %{{.*}} + return _mm256_maskz_cvtneps_pbh(U, A); +} + +__m128 test_mm_dpbf16_ps(__m128 D, __m128bh A, __m128bh B) { + // CHECK-LABEL: @test_mm_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.128 + // CHECK: ret <4 x float> %{{.*}} + return _mm_dpbf16_ps(D, A, B); +} + +__m128 test_mm_maskz_dpbf16_ps(__m128 D, __m128bh A, __m128bh B, __mmask8 U) { + // CHECK-LABEL: @test_mm_maskz_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.128 + // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}} + // CHECK: ret <4 x float> %{{.*}} + return _mm_maskz_dpbf16_ps(U, D, A, B); +} + +__m128 test_mm_mask_dpbf16_ps(__m128 D, __m128bh A, __m128bh B, __mmask8 U) { + // CHECK-LABEL: @test_mm_mask_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.128 + // CHECK: select <4 x i1> %{{.*}}, <4 x float> %{{.*}}, <4 x float> %{{.*}} + // CHECK: ret <4 x float> %{{.*}} + return _mm_mask_dpbf16_ps(D, U, A, B); +} +__m256 test_mm256_dpbf16_ps(__m256 D, __m256bh A, __m256bh B) { + // CHECK-LABEL: @test_mm256_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.256 + // CHECK: ret <8 x float> %{{.*}} + return _mm256_dpbf16_ps(D, A, B); +} + +__m256 test_mm256_maskz_dpbf16_ps(__m256 D, __m256bh A, __m256bh B, __mmask8 U) { + // CHECK-LABEL: @test_mm256_maskz_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.256 + // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} + // CHECK: ret <8 x float> %{{.*}} + return _mm256_maskz_dpbf16_ps(U, D, A, B); +} + +__m256 test_mm256_mask_dpbf16_ps(__m256 D, __m256bh A, __m256bh B, __mmask8 U) { + // CHECK-LABEL: @test_mm256_mask_dpbf16_ps + // CHECK: @llvm.x86.avx512bf16.dpbf16ps.256 + // CHECK: select <8 x i1> %{{.*}}, <8 x float> %{{.*}}, <8 x float> %{{.*}} + // CHECK: ret <8 x float> %{{.*}} + return _mm256_mask_dpbf16_ps(D, U, A, B); +} _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits