https://github.com/phoebewang updated https://github.com/llvm/llvm-project/pull/114070
>From 587d0105e7724db0f35fc5c8179519fa6319e5c8 Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" <phoebe.w...@intel.com> Date: Tue, 29 Oct 2024 22:29:25 +0800 Subject: [PATCH 1/6] [X86][AMX] Support AMX-AVX512 --- clang/docs/ReleaseNotes.rst | 2 + clang/include/clang/Basic/BuiltinsX86_64.def | 13 + clang/include/clang/Driver/Options.td | 2 + clang/lib/Basic/Targets/X86.cpp | 6 + clang/lib/Basic/Targets/X86.h | 1 + clang/lib/Headers/CMakeLists.txt | 1 + clang/lib/Headers/amxavx512intrin.h | 381 ++++++++++++++++++ clang/lib/Headers/immintrin.h | 4 + clang/lib/Sema/SemaX86.cpp | 6 + clang/test/CodeGen/X86/amx_avx512_api.c | 52 +++ clang/test/CodeGen/X86/amxavx512-builtins.c | 41 ++ clang/test/CodeGen/attr-target-x86.c | 8 +- clang/test/Driver/x86-target-features.c | 7 + clang/test/Preprocessor/x86_target_features.c | 7 + llvm/include/llvm/IR/IntrinsicsX86.td | 50 +++ .../llvm/TargetParser/X86TargetParser.def | 1 + llvm/lib/Target/X86/X86.td | 4 + llvm/lib/Target/X86/X86ExpandPseudo.cpp | 64 ++- llvm/lib/Target/X86/X86ISelLowering.cpp | 76 ++++ llvm/lib/Target/X86/X86InstrAMX.td | 147 +++++++ llvm/lib/Target/X86/X86InstrPredicates.td | 1 + llvm/lib/Target/X86/X86LowerAMXType.cpp | 11 + llvm/lib/Target/X86/X86PreTileConfig.cpp | 19 +- llvm/lib/TargetParser/Host.cpp | 4 + llvm/lib/TargetParser/X86TargetParser.cpp | 2 + .../CodeGen/X86/amx-across-func-tilemovrow.ll | 171 ++++++++ .../test/CodeGen/X86/amx-avx512-intrinsics.ll | 116 ++++++ .../CodeGen/X86/amx-tile-avx512-internals.ll | 61 +++ llvm/test/MC/Disassembler/X86/amx-avx512.txt | 106 +++++ llvm/test/MC/X86/amx-avx512-att.s | 105 +++++ llvm/test/MC/X86/amx-avx512-intel.s | 105 +++++ 31 files changed, 1564 insertions(+), 10 deletions(-) create mode 100644 clang/lib/Headers/amxavx512intrin.h create mode 100644 clang/test/CodeGen/X86/amx_avx512_api.c create mode 100644 clang/test/CodeGen/X86/amxavx512-builtins.c create mode 100644 llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll create mode 100644 llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll create mode 100644 llvm/test/CodeGen/X86/amx-tile-avx512-internals.ll create mode 100644 llvm/test/MC/Disassembler/X86/amx-avx512.txt create mode 100644 llvm/test/MC/X86/amx-avx512-att.s create mode 100644 llvm/test/MC/X86/amx-avx512-intel.s diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index ce046a305c89b6..d45bd1240dd173 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -611,6 +611,8 @@ X86 Support * Supported MINMAX intrinsics of ``*_(mask(z)))_minmax(ne)_p[s|d|h|bh]`` and ``*_(mask(z)))_minmax_s[s|d|h]``. +- Support ISA of ``AMX-AVX512``. + - All intrinsics in adcintrin.h can now be used in constant expressions. - All intrinsics in adxintrin.h can now be used in constant expressions. diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def index 2c591edb2835cd..70644f3f6b6054 100644 --- a/clang/include/clang/Basic/BuiltinsX86_64.def +++ b/clang/include/clang/Basic/BuiltinsX86_64.def @@ -128,6 +128,12 @@ TARGET_BUILTIN(__builtin_ia32_tdpbf16ps_internal, "V256iUsUsUsV256iV256iV256i", TARGET_BUILTIN(__builtin_ia32_tdpfp16ps_internal, "V256iUsUsUsV256iV256iV256i", "n", "amx-fp16") TARGET_BUILTIN(__builtin_ia32_tcmmimfp16ps_internal, "V256iUsUsUsV256iV256iV256i", "n", "amx-complex") TARGET_BUILTIN(__builtin_ia32_tcmmrlfp16ps_internal, "V256iUsUsUsV256iV256iV256i", "n", "amx-complex") +TARGET_BUILTIN(__builtin_ia32_tcvtrowd2ps_internal, "V16fUsUsV256iUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16h_internal, "V32yUsUsV256iUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16l_internal, "V32yUsUsV256iUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phh_internal, "V32xUsUsV256iUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phl_internal, "V32xUsUsV256iUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tilemovrow_internal, "V16iUsUsV256iUi", "n", "amx-avx512") // AMX TARGET_BUILTIN(__builtin_ia32_tile_loadconfig, "vvC*", "n", "amx-tile") TARGET_BUILTIN(__builtin_ia32_tile_storeconfig, "vvC*", "n", "amx-tile") @@ -148,6 +154,13 @@ TARGET_BUILTIN(__builtin_ia32_ptwrite64, "vUOi", "n", "ptwrite") TARGET_BUILTIN(__builtin_ia32_tcmmimfp16ps, "vIUcIUcIUc", "n", "amx-complex") TARGET_BUILTIN(__builtin_ia32_tcmmrlfp16ps, "vIUcIUcIUc", "n", "amx-complex") +TARGET_BUILTIN(__builtin_ia32_tcvtrowd2ps, "V16fIUcUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16h, "V32yIUcUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16l, "V32yIUcUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phh, "V32xIUcUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phl, "V32xIUcUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tilemovrow, "V16iIUcUi", "n", "amx-avx512") + TARGET_BUILTIN(__builtin_ia32_prefetchi, "vvC*Ui", "nc", "prefetchi") TARGET_BUILTIN(__builtin_ia32_cmpccxadd32, "Siv*SiSiIi", "n", "cmpccxadd") TARGET_BUILTIN(__builtin_ia32_cmpccxadd64, "SLLiv*SLLiSLLiIi", "n", "cmpccxadd") diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 2ddb2f5312148e..fd200abebceb11 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -6277,6 +6277,8 @@ def mno_80387 : Flag<["-"], "mno-80387">, Alias<mno_x87>; def mno_fp_ret_in_387 : Flag<["-"], "mno-fp-ret-in-387">, Alias<mno_x87>; def mmmx : Flag<["-"], "mmmx">, Group<m_x86_Features_Group>; def mno_mmx : Flag<["-"], "mno-mmx">, Group<m_x86_Features_Group>; +def mamx_avx512 : Flag<["-"], "mamx-avx512">, Group<m_x86_Features_Group>; +def mno_amx_avx512 : Flag<["-"], "mno-amx-avx512">, Group<m_x86_Features_Group>; def mamx_bf16 : Flag<["-"], "mamx-bf16">, Group<m_x86_Features_Group>; def mno_amx_bf16 : Flag<["-"], "mno-amx-bf16">, Group<m_x86_Features_Group>; def mamx_complex : Flag<["-"], "mamx-complex">, Group<m_x86_Features_Group>; diff --git a/clang/lib/Basic/Targets/X86.cpp b/clang/lib/Basic/Targets/X86.cpp index 5448bd841959f4..52cab65cbd9451 100644 --- a/clang/lib/Basic/Targets/X86.cpp +++ b/clang/lib/Basic/Targets/X86.cpp @@ -418,6 +418,8 @@ bool X86TargetInfo::handleTargetFeatures(std::vector<std::string> &Features, HasAMXTILE = true; } else if (Feature == "+amx-complex") { HasAMXCOMPLEX = true; + } else if (Feature == "+amx-avx512") { + HasAMXAVX512 = true; } else if (Feature == "+cmpccxadd") { HasCMPCCXADD = true; } else if (Feature == "+raoint") { @@ -935,6 +937,8 @@ void X86TargetInfo::getTargetDefines(const LangOptions &Opts, Builder.defineMacro("__AMX_FP16__"); if (HasAMXCOMPLEX) Builder.defineMacro("__AMX_COMPLEX__"); + if (HasAMXAVX512) + Builder.defineMacro("__AMX_AVX512__"); if (HasCMPCCXADD) Builder.defineMacro("__CMPCCXADD__"); if (HasRAOINT) @@ -1060,6 +1064,7 @@ bool X86TargetInfo::isValidFeatureName(StringRef Name) const { return llvm::StringSwitch<bool>(Name) .Case("adx", true) .Case("aes", true) + .Case("amx-avx512", true) .Case("amx-bf16", true) .Case("amx-complex", true) .Case("amx-fp16", true) @@ -1177,6 +1182,7 @@ bool X86TargetInfo::hasFeature(StringRef Feature) const { return llvm::StringSwitch<bool>(Feature) .Case("adx", HasADX) .Case("aes", HasAES) + .Case("amx-avx512", HasAMXAVX512) .Case("amx-bf16", HasAMXBF16) .Case("amx-complex", HasAMXCOMPLEX) .Case("amx-fp16", HasAMXFP16) diff --git a/clang/lib/Basic/Targets/X86.h b/clang/lib/Basic/Targets/X86.h index a99ae62984c7d5..ce7458ae99ad64 100644 --- a/clang/lib/Basic/Targets/X86.h +++ b/clang/lib/Basic/Targets/X86.h @@ -156,6 +156,7 @@ class LLVM_LIBRARY_VISIBILITY X86TargetInfo : public TargetInfo { bool HasAMXINT8 = false; bool HasAMXBF16 = false; bool HasAMXCOMPLEX = false; + bool HasAMXAVX512 = false; bool HasSERIALIZE = false; bool HasTSXLDTRK = false; bool HasUSERMSR = false; diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt index ff392e7122a448..88e8f282bd7ec2 100644 --- a/clang/lib/Headers/CMakeLists.txt +++ b/clang/lib/Headers/CMakeLists.txt @@ -146,6 +146,7 @@ set(x86_files adcintrin.h adxintrin.h ammintrin.h + amxavx512intrin.h amxcomplexintrin.h amxfp16intrin.h amxintrin.h diff --git a/clang/lib/Headers/amxavx512intrin.h b/clang/lib/Headers/amxavx512intrin.h new file mode 100644 index 00000000000000..f819696f8086b7 --- /dev/null +++ b/clang/lib/Headers/amxavx512intrin.h @@ -0,0 +1,381 @@ +/*===--------------------- amxavx512intrin.h - AMXAVX512 --------------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===------------------------------------------------------------------------=== + */ +#ifndef __IMMINTRIN_H +#error "Never use <amxavx512intrin.h> directly; include <immintrin.h> instead." +#endif // __IMMINTRIN_H + +#ifndef __AMX_AVX512INTRIN_H +#define __AMX_AVX512INTRIN_H +#ifdef __x86_64__ + +#define __DEFAULT_FN_ATTRS_AVX512 \ + __attribute__((__always_inline__, __nodebug__, __target__("amx-avx512"))) + +/// Moves a row from a tile register to a zmm destination register, converting +/// the int32 source elements to fp32. The row of the tile is selected by an +/// 32b GPR. +/// +/// \headerfile <x86intrin.h> +/// +/// \code +/// __m512i _tile_cvtrowd2ps(__tile tsrc, unsigned int row); +/// \endcode +/// +/// \code{.operation} +/// VL := 512 +/// VL_bytes := VL >> 3 +/// row_index := row & 0xffff +/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes +/// FOR i := 0 TO (VL_bytes / 4) - 1 +/// IF i + row_chunk / 4 >= tsrc.colsb / 4 +/// dst.dword[i] := 0 +/// ELSE +/// dst.f32[i] := CONVERT_INT32_TO_FP32(tsrc.row[row_index].dword[row_chunk/4+i], RNE) +/// FI +/// ENDFOR +/// dst[MAX_VL-1:VL] := 0 +/// zero_tileconfig_start() +/// \endcode +/// +/// This intrinsic corresponds to the \c TCVTROWD2PS instruction. +/// +/// \param tsrc +/// The 1st source tile. Max size is 1024 Bytes. +/// \param row +/// The row of the source tile +#define _tile_cvtrowd2ps(tsrc, row) __builtin_ia32_tcvtrowd2ps(tsrc, row) + +/// Moves a row from a tile register to a zmm destination register, converting +/// the fp32 source elements to bf16. It places the resulting bf16 elements +/// in the high 16 bits within each dword. The row of the tile is selected +/// by an 32b GPR. +/// +/// \headerfile <x86intrin.h> +/// +/// \code +/// __m512i _tile_cvtrowps2pbf16h(__tile tsrc, unsigned int row); +/// \endcode +/// +/// \code{.operation} +/// VL := 512 +/// VL_bytes := VL >> 3 +/// row_index := row & 0xffff +/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes +/// FOR i := 0 TO (VL_bytes / 4) - 1 +/// IF i + row_chunk / 4 >= tsrc.colsb / 4 +/// dst.dword[i] := 0 +/// ELSE +/// dst.word[2*i+0] := 0 +/// dst.bf16[2*i+1] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE) +/// FI +/// ENDFOR +/// dst[MAX_VL-1:VL] := 0 +/// zero_tileconfig_start() +/// \endcode +/// +/// This intrinsic corresponds to the \c TCVTROWPS2PBF16H instruction. +/// +/// \param tsrc +/// The source tile. Max size is 1024 Bytes. +/// \param row +/// The the row of the source tile. +#define _tile_cvtrowps2pbf16h(tsrc, row) \ + __builtin_ia32_tcvtrowps2pbf16h(tsrc, row) + +/// Moves a row from a tile register to a zmm destination register, converting +/// the fp32 source elements to bf16. It places the resulting bf16 elements +/// in the low 16 bits within each dword. The row of the tile is selected +/// by an 32b GPR. +/// +/// \headerfile <x86intrin.h> +/// +/// \code +/// __m512i _tile_cvtrowps2pbf16l(__tile tsrc, unsigned int row); +/// \endcode +/// +/// \code{.operation} +/// VL := 512 +/// VL_bytes := VL >> 3 +/// row_index := row & 0xffff +/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes +/// FOR i := 0 TO (VL_bytes / 4) - 1 +/// IF i + row_chunk / 4 >= tsrc.colsb / 4 +/// dst.dword[i] := 0 +/// ELSE +/// dst.word[2*i+1] := 0 +/// dst.bf16[2*i+0] := CONVERT_FP32_TO_BF16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE) +/// FI +/// ENDFOR +/// dst[MAX_VL-1:VL] := 0 +/// zero_tileconfig_start() +/// \endcode +/// +/// This intrinsic corresponds to the \c TCVTROWPS2PBF16L instruction. +/// +/// \param tsrc +/// The source tile. Max size is 1024 Bytes. +/// \param row +/// The the row of the source tile. +#define _tile_cvtrowps2pbf16l(tsrc, row) \ + __builtin_ia32_tcvtrowps2pbf16l(tsrc, row) + +/// Moves a row from a tile register to a zmm destination register, converting +/// the fp32 source elements to fp16. It places the resulting fp16 elements +/// in the high 16 bits within each dword. The row of the tile is selected +/// by an 32b GPR. +/// +/// \headerfile <x86intrin.h> +/// +/// \code +/// __m512i _tile_cvtrowps2phh(__tile tsrc, unsigned int row); +/// \endcode +/// +/// \code{.operation} +/// VL := 512 +/// VL_bytes := VL >> 3 +/// row_index := row & 0xffff +/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes +/// FOR i := 0 TO (VL_bytes / 4) - 1 +/// IF i + row_chunk / 4 >= tsrc.colsb / 4 +/// dst.dword[i] := 0 +/// ELSE +/// dst.word[2*i+0] := 0 +/// dst.fp16[2*i+1] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE) +/// FI +/// ENDFOR +/// dst[MAX_VL-1:VL] := 0 +/// zero_tileconfig_start() +/// \endcode +/// +/// This intrinsic corresponds to the \c TCVTROWPS2PHH instruction. +/// +/// \param tsrc +/// The source tile. Max size is 1024 Bytes. +/// \param row +/// The the row of the source tile. +#define _tile_cvtrowps2phh(tsrc, row) __builtin_ia32_tcvtrowps2phh(tsrc, row) + +/// Moves a row from a tile register to a zmm destination register, converting +/// the fp32 source elements to fp16. It places the resulting fp16 elements +/// in the low 16 bits within each dword. The row of the tile is selected +/// by an 32b GPR. +/// +/// \headerfile <x86intrin.h> +/// +/// \code +/// __m512i _tile_cvtrowps2phl(__tile tsrc, unsigned int row); +/// \endcode +/// +/// \code{.operation} +/// VL := 512 +/// VL_bytes := VL >> 3 +/// row_index := row & 0xffff +/// row_chunk := ((row >> 16) & 0xffff) * VL_bytes +/// FOR i := 0 TO (VL_bytes / 4) - 1 +/// IF i + row_chunk / 4 >= tsrc.colsb / 4 +/// dst.dword[i] := 0 +/// ELSE +/// dst.word[2*i+1] := 0 +/// dst.fp16[2*i+0] := CONVERT_FP32_TO_FP16(tsrc.row[row_index].fp32[row_chunk/4+i], RNE) +/// FI +/// ENDFOR +/// dst[MAX_VL-1:VL] := 0 +/// zero_tileconfig_start() +/// \endcode +/// +/// This intrinsic corresponds to the \c TCVTROWPS2PHL instruction. +/// +/// \param tsrc +/// The source tile. Max size is 1024 Bytes. +/// \param row +/// The the row of the source tile. +#define _tile_cvtrowps2phl(tsrc, row) __builtin_ia32_tcvtrowps2phl(tsrc, row) + +/// Move one row of a tile data to a v16f32 data. +/// The row of the tile is selected by a 32b GPR. +/// +/// \headerfile <immintrin.h> +/// +/// \code +/// __m512 _tile_movrow(__tile a, unsigned b); +/// \endcode +/// +/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction. +/// +/// \param a +/// The 1st source tile. Max size is 1024 Bytes. +/// \param b +/// The 2nd source r32. Size is 4 Bytes. +/// \returns +/// The destination v16f32 data. Size is 64 Bytes. +/// +/// \code{.operation} +/// VL := 512 +/// VL_bytes := VL>>3 +/// row_index := b&0xffff +/// row_chunk := ((b>>16)&0xffff) * VL_bytes +/// FOR i := 0 TO (VL_bytes-1) +/// IF (row_chunk + i >= a.colsb) +/// dst.byte[i] := 0 +/// ELSE +/// dst.byte[i] := a.row[row_index].byte[row_chunk+i] +/// ENDFOR +/// \endcode +#define _tile_movrow(a, b) __builtin_ia32_tilemovrow(a, b) + +/// This is internal intrinsic. C/C++ user should avoid calling it directly. + +static __inline__ __m512 __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowd2ps_internal( + unsigned short m, unsigned short n, _tile1024i src, unsigned u) { + return __builtin_ia32_tcvtrowd2ps_internal(m, n, src, u); +} + +static __inline__ __m512bh __DEFAULT_FN_ATTRS_AVX512 +_tile_cvtrowps2pbf16h_internal(unsigned short m, unsigned short n, + _tile1024i src, unsigned u) { + return __builtin_ia32_tcvtrowps2pbf16h_internal(m, n, src, u); +} + +static __inline__ __m512bh __DEFAULT_FN_ATTRS_AVX512 +_tile_cvtrowps2pbf16l_internal(unsigned short m, unsigned short n, + _tile1024i src, unsigned u) { + return __builtin_ia32_tcvtrowps2pbf16l_internal(m, n, src, u); +} + +static __inline__ __m512h __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowps2phh_internal( + unsigned short m, unsigned short n, _tile1024i src, unsigned u) { + return __builtin_ia32_tcvtrowps2phh_internal(m, n, src, u); +} + +static __inline__ __m512h __DEFAULT_FN_ATTRS_AVX512 _tile_cvtrowps2phl_internal( + unsigned short m, unsigned short n, _tile1024i src, unsigned u) { + return __builtin_ia32_tcvtrowps2phl_internal(m, n, src, u); +} + +static __inline__ __m512i __DEFAULT_FN_ATTRS_AVX512 _tile_movrow_internal( + unsigned short m, unsigned short n, _tile1024i src, unsigned u) { + return (__m512i)__builtin_ia32_tilemovrow_internal(m, n, src, u); +} + +/// Move a row from a tile (src0) to a v16f32 dst, converting the int32 source +/// elements to fp32. No SIMD exceptions are generated. Rounding is done as if +/// MXCSR.RC=RNE. Embedded rounding is not supported. +/// The row and chunk elements of tile is fetched from 32bit src1. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TCVTROWD2PS </c> instruction. +/// +/// \param src0 +/// The 1st source tile. Max size is 1024 Bytes. +/// \param src1 +/// The 1st source r32. Size is 4 Bytes. +/// \returns +/// The destination v16f32 data. Size is 64 Bytes. +__DEFAULT_FN_ATTRS_AVX512 +static __m512 __tile_cvtrowd2ps(__tile1024i src0, unsigned src1) { + return _tile_cvtrowd2ps_internal(src0.row, src0.col, src0.tile, src1); +} + +/// Move a row from a tile (src0) to a v32bf16 dst, converting the fp32 source +/// elements to bf16 at high 16-bits of each dword. +/// The row and chunk elements of tile is fetched from 32bit src1. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TCVTROWPS2PBF16H </c> instruction. +/// +/// \param src0 +/// The 1st source tile. Max size is 1024 Bytes. +/// \param src1 +/// The 1st source r32. Size is 4 Bytes. +/// \returns +/// The destination v32bf16 data. Size is 64 Bytes. +__DEFAULT_FN_ATTRS_AVX512 +static __m512bh __tile_cvtrowps2pbf16h(__tile1024i src0, unsigned src1) { + return _tile_cvtrowps2pbf16h_internal(src0.row, src0.col, src0.tile, src1); +} + +/// Move a row from a tile (src0) to a v32bf16 dst, converting the fp32 source +/// elements to bf16 at low 16-bits of each dword. +/// The row and chunk elements of tile is fetched from 32bit src1. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TCVTROWPS2PBF16L </c> instruction. +/// +/// \param src0 +/// The 1st source tile. Max size is 1024 Bytes. +/// \param src1 +/// The 1st source r32. Size is 4 Bytes. +/// \returns +/// The destination v32bf16 data. Size is 64 Bytes. +__DEFAULT_FN_ATTRS_AVX512 +static __m512bh __tile_cvtrowps2pbf16l(__tile1024i src0, unsigned src1) { + return _tile_cvtrowps2pbf16l_internal(src0.row, src0.col, src0.tile, src1); +} + +/// Move a row from a tile (src0) to a v32fp16 dst, converting the fp32 source +/// elements to fp16 at high 16-bits of each dword. +/// The row and chunk elements of tile is fetched from 32bit src1. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TCVTROWPS2PHH </c> instruction. +/// +/// \param src0 +/// The 1st source tile. Max size is 1024 Bytes. +/// \param src1 +/// The 1st source r32. Size is 4 Bytes. +/// \returns +/// The destination v32fp16 data. Size is 64 Bytes. +__DEFAULT_FN_ATTRS_AVX512 +static __m512h __tile_cvtrowps2phh(__tile1024i src0, unsigned src1) { + return _tile_cvtrowps2phh_internal(src0.row, src0.col, src0.tile, src1); +} + +/// Move a row from a tile (src0) to a v32fp16 dst, converting the fp32 source +/// elements to fp16 at low 16-bits of each dword. +/// The row and chunk elements of tile is fetched from 32bit src1. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TCVTROWPS2PHL </c> instruction. +/// +/// \param src0 +/// The 1st source tile. Max size is 1024 Bytes. +/// \param src1 +/// The 1st source r32. Size is 4 Bytes. +/// \returns +/// The destination v32fp16 data. Size is 64 Bytes. +__DEFAULT_FN_ATTRS_AVX512 +static __m512h __tile_cvtrowps2phl(__tile1024i src0, unsigned src1) { + return _tile_cvtrowps2phl_internal(src0.row, src0.col, src0.tile, src1); +} + +/// Move one row of a tile data to a v16f32 data. +/// The row of the tile is selected by a 32b GPR. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction. +/// +/// \param src0 +/// The 1st source tile. Max size is 1024 Bytes. +/// \param src1 +/// The 1st source r32. Size is 4 Bytes. +/// \returns +/// The destination v16i32 data. Size is 64 Bytes. +__DEFAULT_FN_ATTRS_AVX512 +static __m512i __tile_movrow(__tile1024i src0, unsigned src1) { + return (__m512i)_tile_movrow_internal(src0.row, src0.col, src0.tile, src1); +} + +#endif // __x86_64__ +#endif // __AMX_AVX512INTRIN_H diff --git a/clang/lib/Headers/immintrin.h b/clang/lib/Headers/immintrin.h index 3fbabffa98df20..84e56238fcf2dc 100644 --- a/clang/lib/Headers/immintrin.h +++ b/clang/lib/Headers/immintrin.h @@ -638,6 +638,10 @@ _storebe_i64(void * __P, long long __D) { #include <amxcomplexintrin.h> #endif +#if !defined(__SCE__) || __has_feature(modules) || defined(__AMX_AVX512__) +#include <amxavx512intrin.h> +#endif + #if !defined(__SCE__) || __has_feature(modules) || \ defined(__AVX512VP2INTERSECT__) #include <avx512vp2intersectintrin.h> diff --git a/clang/lib/Sema/SemaX86.cpp b/clang/lib/Sema/SemaX86.cpp index 6a4d78f0ca9084..fba901473e6e18 100644 --- a/clang/lib/Sema/SemaX86.cpp +++ b/clang/lib/Sema/SemaX86.cpp @@ -631,6 +631,12 @@ bool SemaX86::CheckBuiltinTileArguments(unsigned BuiltinID, CallExpr *TheCall) { case X86::BI__builtin_ia32_tileloaddt164: case X86::BI__builtin_ia32_tilestored64: case X86::BI__builtin_ia32_tilezero: + case X86::BI__builtin_ia32_tcvtrowps2pbf16h: + case X86::BI__builtin_ia32_tcvtrowps2pbf16l: + case X86::BI__builtin_ia32_tcvtrowps2phh: + case X86::BI__builtin_ia32_tcvtrowps2phl: + case X86::BI__builtin_ia32_tcvtrowd2ps: + case X86::BI__builtin_ia32_tilemovrow: return CheckBuiltinTileArgumentsRange(TheCall, 0); case X86::BI__builtin_ia32_tdpbssd: case X86::BI__builtin_ia32_tdpbsud: diff --git a/clang/test/CodeGen/X86/amx_avx512_api.c b/clang/test/CodeGen/X86/amx_avx512_api.c new file mode 100644 index 00000000000000..aea790d61268d3 --- /dev/null +++ b/clang/test/CodeGen/X86/amx_avx512_api.c @@ -0,0 +1,52 @@ +// RUN: %clang_cc1 %s -flax-vector-conversions=none -ffreestanding -triple=x86_64-unknown-unknown \ +// RUN: -target-feature +amx-avx512 -target-feature +avx10.2-512 \ +// RUN: -emit-llvm -o - -Werror -pedantic | FileCheck %s --check-prefixes=CHECK + +#include <immintrin.h> + +char buf[1024]; +#define STRIDE 32 + +char buf2[1024]; + +__m512 test_tile_cvtrowd2ps(__tile1024i a, unsigned b) { + //CHECK-LABEL: @test_tile_cvtrowd2ps + //CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}}) + //CHECK-DAG: call <16 x float> @llvm.x86.tcvtrowd2ps.internal + return __tile_cvtrowd2ps(a, b); +} + +__m512bh test_tile_cvtrowps2pbf16h(__tile1024i a, unsigned b) { + //CHECK-LABEL: @test_tile_cvtrowps2pbf16h + //CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}}) + //CHECK-DAG: call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h.internal + return __tile_cvtrowps2pbf16h(a, b); +} + +__m512bh test_tile_cvtrowps2pbf16l(__tile1024i a, unsigned b) { + //CHECK-LABEL: @test_tile_cvtrowps2pbf16l + //CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}}) + //CHECK-DAG: call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l.internal + return __tile_cvtrowps2pbf16l(a, b); +} + +__m512h test_tile_cvtrowps2phh(__tile1024i a, unsigned b) { + //CHECK-LABEL: @test_tile_cvtrowps2phh + //CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}}) + //CHECK-DAG: call <32 x half> @llvm.x86.tcvtrowps2phh.internal + return __tile_cvtrowps2phh(a, b); +} + +__m512h test_tile_cvtrowps2phl(__tile1024i a, unsigned b) { + //CHECK-LABEL: @test_tile_cvtrowps2phl + //CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}}) + //CHECK-DAG: call <32 x half> @llvm.x86.tcvtrowps2phl.internal + return __tile_cvtrowps2phl(a, b); +} + +__m512i test_tile_movrow(__tile1024i a, unsigned b) { + //CHECK-LABEL: @test_tile_movrow + //CHECK-DAG: call x86_amx @llvm.x86.cast.vector.to.tile.v256i32(<256 x i32> {{%.*}}) + //CHECK-DAG: call <16 x i32> @llvm.x86.tilemovrow.internal + return __tile_movrow(a, b); +} diff --git a/clang/test/CodeGen/X86/amxavx512-builtins.c b/clang/test/CodeGen/X86/amxavx512-builtins.c new file mode 100644 index 00000000000000..172b5ae8f53081 --- /dev/null +++ b/clang/test/CodeGen/X86/amxavx512-builtins.c @@ -0,0 +1,41 @@ +// RUN: %clang_cc1 %s -ffreestanding -triple=x86_64-unknown-unknown -target-feature +amx-tile -target-feature +amx-avx512 \ +// RUN: -target-feature +avx10.2-512 -emit-llvm -o - -Wall -Werror -pedantic -Wno-gnu-statement-expression | FileCheck %s + +#include <immintrin.h> +#include <stddef.h> + +__m512 test_tile_cvtrowd2ps(unsigned int A) { + // CHECK-LABEL: @test_tile_cvtrowd2ps( + // CHECK: call <16 x float> @llvm.x86.tcvtrowd2ps(i8 1, i32 %{{.*}}) + return _tile_cvtrowd2ps(1, A); +} + +__m512bh test_tile_cvtrowps2pbf16h(unsigned int A) { + // CHECK-LABEL: @test_tile_cvtrowps2pbf16h( + // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h(i8 1, i32 %{{.*}}) + return _tile_cvtrowps2pbf16h(1, A); +} + +__m512bh test_tile_cvtrowps2pbf16l(unsigned int A) { + // CHECK-LABEL: @test_tile_cvtrowps2pbf16l( + // CHECK: call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l(i8 1, i32 %{{.*}}) + return _tile_cvtrowps2pbf16l(1, A); +} + +__m512h test_tile_cvtrowps2phh(unsigned int A) { + // CHECK-LABEL: @test_tile_cvtrowps2phh( + // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phh(i8 1, i32 %{{.*}}) + return _tile_cvtrowps2phh(1, A); +} + +__m512h test_tile_cvtrowps2phl(unsigned int A) { + // CHECK-LABEL: @test_tile_cvtrowps2phl( + // CHECK: call <32 x half> @llvm.x86.tcvtrowps2phl(i8 1, i32 %{{.*}}) + return _tile_cvtrowps2phl(1, A); +} + +__m512i test_tile_movrow(unsigned int A) { + // CHECK-LABEL: @test_tile_movrow + // CHECK: %1 = call <16 x i32> @llvm.x86.tilemovrow(i8 1, i32 %{{.*}}) + return _tile_movrow(1, A); +} diff --git a/clang/test/CodeGen/attr-target-x86.c b/clang/test/CodeGen/attr-target-x86.c index 593ccffbcda095..2033a8b4c335f9 100644 --- a/clang/test/CodeGen/attr-target-x86.c +++ b/clang/test/CodeGen/attr-target-x86.c @@ -59,10 +59,10 @@ void __attribute__((target("avx10.1-512"))) avx10_1_512(void) {} // CHECK: #0 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87" "tune-cpu"="i686" // CHECK: #1 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt" // CHECK-NOT: tune-cpu -// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686" +// CHECK: #2 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-aes,-amx-avx512,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-gfni,-kl,-pclmul,-sha,-sha512,-sm3,-sm4,-sse2,-sse3,-sse4.1,-sse4.2,-sse4a,-ssse3,-vaes,-vpclmulqdq,-widekl,-xop" "tune-cpu"="i686" // CHECK: #3 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+crc32,+cx8,+mmx,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87" "tune-cpu"="i686" -// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686" -// CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-vaes" +// CHECK: #4 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-amx-avx512,-avx,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-avx2,-avx512bf16,-avx512bitalg,-avx512bw,-avx512cd,-avx512dq,-avx512f,-avx512fp16,-avx512ifma,-avx512vbmi,-avx512vbmi2,-avx512vl,-avx512vnni,-avx512vp2intersect,-avx512vpopcntdq,-avxifma,-avxneconvert,-avxvnni,-avxvnniint16,-avxvnniint8,-f16c,-fma,-fma4,-sha512,-sm3,-sm4,-sse4.1,-sse4.2,-vaes,-vpclmulqdq,-xop" "tune-cpu"="i686" +// CHECK: #5 = {{.*}}"target-cpu"="ivybridge" "target-features"="+avx,+cmov,+crc32,+cx16,+cx8,+f16c,+fsgsbase,+fxsr,+mmx,+pclmul,+popcnt,+rdrnd,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave,+xsaveopt,-aes,-amx-avx512,-avx10.1-256,-avx10.1-512,-avx10.2-256,-avx10.2-512,-vaes" // CHECK-NOT: tune-cpu // CHECK: #6 = {{.*}}"target-cpu"="i686" "target-features"="+cmov,+cx8,+x87,-mmx" // CHECK: #7 = {{.*}}"target-cpu"="lakemont" "target-features"="+cx8,+mmx" @@ -76,5 +76,5 @@ void __attribute__((target("avx10.1-512"))) avx10_1_512(void) {} // CHECK: "target-cpu"="x86-64-v4" // CHECK-SAME: "target-features"="+avx,+avx2,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl,+bmi,+bmi2,+cmov,+crc32,+cx16,+cx8,+evex512,+f16c,+fma,+fxsr,+lzcnt,+mmx,+movbe,+popcnt,+sahf,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+x87,+xsave" -// CHECK: #12 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave,-avx10.1-512,-avx10.2-512,-evex512" +// CHECK: #12 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave,-amx-avx512,-avx10.1-512,-avx10.2-512,-evex512" // CHECK: #13 = {{.*}}"target-cpu"="i686" "target-features"="+aes,+avx,+avx10.1-256,+avx10.1-512,+avx2,+avx512bf16,+avx512bitalg,+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512fp16,+avx512ifma,+avx512vbmi,+avx512vbmi2,+avx512vl,+avx512vnni,+avx512vpopcntdq,+cmov,+crc32,+cx8,+evex512,+f16c,+fma,+mmx,+pclmul,+popcnt,+sse,+sse2,+sse3,+sse4.1,+sse4.2,+ssse3,+vaes,+vpclmulqdq,+x87,+xsave" diff --git a/clang/test/Driver/x86-target-features.c b/clang/test/Driver/x86-target-features.c index ddfbb29a48f8d5..c3a6159f70f601 100644 --- a/clang/test/Driver/x86-target-features.c +++ b/clang/test/Driver/x86-target-features.c @@ -304,6 +304,13 @@ // AMX-COMPLEX: "-target-feature" "+amx-complex" // NO-AMX-COMPLEX: "-target-feature" "-amx-complex" +// RUN: %clang -target x86_64-unknown-linux-gnu -mamx-avx512 %s \ +// RUN: -### -o %t.o 2>&1 | FileCheck -check-prefix=AMX-AVX512 %s +// RUN: %clang -target x86_64-unknown-linux-gnu -mno-amx-avx512 %s \ +// RUN: -### -o %t.o 2>&1 | FileCheck -check-prefix=NO-AMX-AVX512 %s +// AMX-AVX512: "-target-feature" "+amx-avx512" +// NO-AMX-AVX512: "-target-feature" "-amx-avx512" + // RUN: %clang --target=i386 -march=i386 -mhreset %s -### 2>&1 | FileCheck -check-prefix=HRESET %s // RUN: %clang --target=i386 -march=i386 -mno-hreset %s -### 2>&1 | FileCheck -check-prefix=NO-HRESET %s // HRESET: "-target-feature" "+hreset" diff --git a/clang/test/Preprocessor/x86_target_features.c b/clang/test/Preprocessor/x86_target_features.c index 8b4e6bdc09226a..c9ced62a91565b 100644 --- a/clang/test/Preprocessor/x86_target_features.c +++ b/clang/test/Preprocessor/x86_target_features.c @@ -546,6 +546,13 @@ // NO-AMX-COMPLEX-NOT: #define __AMX_COMPLEX__ 1 +// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mno-amx-avx512 -x c \ +// RUN: -E -dM -o - %s | FileCheck -check-prefix=NO-AMX-AVX512 %s +// RUN: %clang -target x86_64-unknown-linux-gnu -march=x86-64 -mamx-avx512 -mno-amx-tile \ +// RUN: -x c -E -dM -o - %s | FileCheck -check-prefix=NO-AMX-AVX512 %s + +// NO-AMX-AVX512-NOT: #define __AMX_AVX512__ 1 + // RUN: %clang -target i386-unknown-unknown -march=atom -mavxvnni -x c -E -dM -o - %s | FileCheck -match-full-lines --check-prefix=AVXVNNI %s // AVXVNNI: #define __AVX2__ 1 diff --git a/llvm/include/llvm/IR/IntrinsicsX86.td b/llvm/include/llvm/IR/IntrinsicsX86.td index 5262e3154ff721..844dd291fd089f 100644 --- a/llvm/include/llvm/IR/IntrinsicsX86.td +++ b/llvm/include/llvm/IR/IntrinsicsX86.td @@ -5917,6 +5917,26 @@ let TargetPrefix = "x86" in { [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; + // AMX-AVX512 + def int_x86_tcvtrowd2ps : ClangBuiltin<"__builtin_ia32_tcvtrowd2ps">, + Intrinsic<[llvm_v16f32_ty], [llvm_i8_ty, llvm_i32_ty], + [ImmArg<ArgIndex<0>>]>; + def int_x86_tcvtrowps2pbf16h : ClangBuiltin<"__builtin_ia32_tcvtrowps2pbf16h">, + Intrinsic<[llvm_v32bf16_ty], [llvm_i8_ty, llvm_i32_ty], + [ImmArg<ArgIndex<0>>]>; + def int_x86_tcvtrowps2pbf16l : ClangBuiltin<"__builtin_ia32_tcvtrowps2pbf16l">, + Intrinsic<[llvm_v32bf16_ty], [llvm_i8_ty, llvm_i32_ty], + [ImmArg<ArgIndex<0>>]>; + def int_x86_tcvtrowps2phh : ClangBuiltin<"__builtin_ia32_tcvtrowps2phh">, + Intrinsic<[llvm_v32f16_ty], [llvm_i8_ty, llvm_i32_ty], + [ImmArg<ArgIndex<0>>]>; + def int_x86_tcvtrowps2phl : ClangBuiltin<"__builtin_ia32_tcvtrowps2phl">, + Intrinsic<[llvm_v32f16_ty], [llvm_i8_ty, llvm_i32_ty], + [ImmArg<ArgIndex<0>>]>; + def int_x86_tilemovrow : ClangBuiltin<"__builtin_ia32_tilemovrow">, + Intrinsic<[llvm_v16i32_ty], [llvm_i8_ty, llvm_i32_ty], + [ImmArg<ArgIndex<0>>]>; + // AMX - internal intrinsics def int_x86_ldtilecfg_internal : ClangBuiltin<"__builtin_ia32_tile_loadconfig_internal">, @@ -5994,6 +6014,36 @@ let TargetPrefix = "x86" in { [llvm_i16_ty, llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_x86amx_ty, llvm_x86amx_ty], []>; + def int_x86_tcvtrowd2ps_internal : + ClangBuiltin<"__builtin_ia32_tcvtrowd2ps_internal">, + Intrinsic<[llvm_v16f32_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty], + []>; + def int_x86_tcvtrowps2pbf16h_internal : + ClangBuiltin<"__builtin_ia32_tcvtrowps2pbf16h_internal">, + Intrinsic<[llvm_v32bf16_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty], + []>; + def int_x86_tcvtrowps2pbf16l_internal : + ClangBuiltin<"__builtin_ia32_tcvtrowps2pbf16l_internal">, + Intrinsic<[llvm_v32bf16_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty], + []>; + def int_x86_tcvtrowps2phh_internal : + ClangBuiltin<"__builtin_ia32_tcvtrowps2phh_internal">, + Intrinsic<[llvm_v32f16_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty], + []>; + def int_x86_tcvtrowps2phl_internal : + ClangBuiltin<"__builtin_ia32_tcvtrowps2phl_internal">, + Intrinsic<[llvm_v32f16_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty], + []>; + def int_x86_tilemovrow_internal : + ClangBuiltin<"__builtin_ia32_tilemovrow_internal">, + Intrinsic<[llvm_v16i32_ty], + [llvm_i16_ty, llvm_i16_ty, llvm_x86amx_ty, llvm_i32_ty], + []>; } //===----------------------------------------------------------------------===// diff --git a/llvm/include/llvm/TargetParser/X86TargetParser.def b/llvm/include/llvm/TargetParser/X86TargetParser.def index e5bf196559ba63..134c33b6162181 100644 --- a/llvm/include/llvm/TargetParser/X86TargetParser.def +++ b/llvm/include/llvm/TargetParser/X86TargetParser.def @@ -262,6 +262,7 @@ X86_FEATURE_COMPAT(AVX10_1_512, "avx10.1-512", 37) X86_FEATURE_COMPAT(AVX10_2, "avx10.2-256", 0) X86_FEATURE_COMPAT(AVX10_2_512, "avx10.2-512", 0) X86_FEATURE (ZU, "zu") +X86_FEATURE (AMX_AVX512, "amx-avx512") // These features aren't really CPU features, but the frontend can set them. X86_FEATURE (RETPOLINE_EXTERNAL_THUNK, "retpoline-external-thunk") X86_FEATURE (RETPOLINE_INDIRECT_BRANCHES, "retpoline-indirect-branches") diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td index d57450d91ea2dd..c1e50ed93ff9fd 100644 --- a/llvm/lib/Target/X86/X86.td +++ b/llvm/lib/Target/X86/X86.td @@ -270,6 +270,10 @@ def FeatureAMXFP16 : SubtargetFeature<"amx-fp16", "HasAMXFP16", "true", def FeatureAMXCOMPLEX : SubtargetFeature<"amx-complex", "HasAMXCOMPLEX", "true", "Support AMX-COMPLEX instructions", [FeatureAMXTILE]>; +def FeatureAMXAVX512 : SubtargetFeature<"amx-avx512", + "HasAMXAVX512", "true", + "Support AMX-AVX512 instructions", + [FeatureAMXTILE, FeatureAVX512]>; def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true", "Support CMPCCXADD instructions">; def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true", diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index f4c67f115c9f3d..adcaf96cb38b31 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -559,12 +559,68 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, return true; } case X86::PTILELOADDV: - case X86::PTILELOADDT1V: { + case X86::PTILELOADDT1V: + case X86::PTCVTROWD2PSrreV: + case X86::PTCVTROWD2PSrriV: + case X86::PTCVTROWPS2PBF16HrreV: + case X86::PTCVTROWPS2PBF16HrriV: + case X86::PTCVTROWPS2PBF16LrreV: + case X86::PTCVTROWPS2PBF16LrriV: + case X86::PTCVTROWPS2PHHrreV: + case X86::PTCVTROWPS2PHHrriV: + case X86::PTCVTROWPS2PHLrreV: + case X86::PTCVTROWPS2PHLrriV: + case X86::PTILEMOVROWrreV: + case X86::PTILEMOVROWrriV: { for (unsigned i = 2; i > 0; --i) MI.removeOperand(i); - unsigned Opc = Opcode == X86::PTILELOADDV - ? GET_EGPR_IF_ENABLED(X86::TILELOADD) - : GET_EGPR_IF_ENABLED(X86::TILELOADDT1); + unsigned Opc; + switch (Opcode) { + case X86::PTILELOADDV: + Opc = GET_EGPR_IF_ENABLED(X86::TILELOADD); + break; + case X86::PTILELOADDT1V: + Opc = GET_EGPR_IF_ENABLED(X86::TILELOADDT1); + break; + case X86::PTCVTROWD2PSrreV: + Opc = X86::TCVTROWD2PSrre; + break; + case X86::PTCVTROWD2PSrriV: + Opc = X86::TCVTROWD2PSrri; + break; + case X86::PTCVTROWPS2PBF16HrreV: + Opc = X86::TCVTROWPS2PBF16Hrre; + break; + case X86::PTCVTROWPS2PBF16HrriV: + Opc = X86::TCVTROWPS2PBF16Hrri; + break; + case X86::PTCVTROWPS2PBF16LrreV: + Opc = X86::TCVTROWPS2PBF16Lrre; + break; + case X86::PTCVTROWPS2PBF16LrriV: + Opc = X86::TCVTROWPS2PBF16Lrri; + break; + case X86::PTCVTROWPS2PHHrreV: + Opc = X86::TCVTROWPS2PHHrre; + break; + case X86::PTCVTROWPS2PHHrriV: + Opc = X86::TCVTROWPS2PHHrri; + break; + case X86::PTCVTROWPS2PHLrreV: + Opc = X86::TCVTROWPS2PHLrre; + break; + case X86::PTCVTROWPS2PHLrriV: + Opc = X86::TCVTROWPS2PHLrri; + break; + case X86::PTILEMOVROWrreV: + Opc = X86::TILEMOVROWrre; + break; + case X86::PTILEMOVROWrriV: + Opc = X86::TILEMOVROWrri; + break; + default: + llvm_unreachable("Impossible Opcode!"); + } MI.setDesc(TII->get(Opc)); return true; } diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 7f4dc12a20837f..980292c0811758 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -37503,6 +37503,82 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI, MI.eraseFromParent(); // The pseudo is gone now. return BB; } + case X86::PTCVTROWPS2PBF16Hrri: + case X86::PTCVTROWPS2PBF16Lrri: + case X86::PTCVTROWPS2PHHrri: + case X86::PTCVTROWPS2PHLrri: + case X86::PTCVTROWD2PSrri: + case X86::PTILEMOVROWrri: { + const DebugLoc &DL = MI.getDebugLoc(); + unsigned Opc; + switch (MI.getOpcode()) { + default: + llvm_unreachable("Unexpected instruction!"); + case X86::PTCVTROWD2PSrri: + Opc = X86::TCVTROWD2PSrri; + break; + case X86::PTCVTROWPS2PBF16Hrri: + Opc = X86::TCVTROWPS2PBF16Hrri; + break; + case X86::PTCVTROWPS2PHHrri: + Opc = X86::TCVTROWPS2PHHrri; + break; + case X86::PTCVTROWPS2PBF16Lrri: + Opc = X86::TCVTROWPS2PBF16Lrri; + break; + case X86::PTCVTROWPS2PHLrri: + Opc = X86::TCVTROWPS2PHLrri; + break; + case X86::PTILEMOVROWrri: + Opc = X86::TILEMOVROWrri; + break; + } + MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII->get(Opc)); + MIB.add(MI.getOperand(0)); + MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef); + MIB.addImm(MI.getOperand(2).getImm()); + + MI.eraseFromParent(); // The pseudo is gone now. + return BB; + } + case X86::PTCVTROWPS2PBF16Hrre: + case X86::PTCVTROWPS2PBF16Lrre: + case X86::PTCVTROWPS2PHHrre: + case X86::PTCVTROWPS2PHLrre: + case X86::PTCVTROWD2PSrre: + case X86::PTILEMOVROWrre: { + const DebugLoc &DL = MI.getDebugLoc(); + unsigned Opc; + switch (MI.getOpcode()) { + default: + llvm_unreachable("Unexpected instruction!"); + case X86::PTCVTROWD2PSrre: + Opc = X86::TCVTROWD2PSrre; + break; + case X86::PTCVTROWPS2PBF16Hrre: + Opc = X86::TCVTROWPS2PBF16Hrre; + break; + case X86::PTCVTROWPS2PBF16Lrre: + Opc = X86::TCVTROWPS2PBF16Lrre; + break; + case X86::PTCVTROWPS2PHHrre: + Opc = X86::TCVTROWPS2PHHrre; + break; + case X86::PTCVTROWPS2PHLrre: + Opc = X86::TCVTROWPS2PHLrre; + break; + case X86::PTILEMOVROWrre: + Opc = X86::TILEMOVROWrre; + break; + } + MachineInstrBuilder MIB = BuildMI(*BB, MI, DL, TII->get(Opc)); + MIB.add(MI.getOperand(0)); + MIB.addReg(TMMImmToTMMReg(MI.getOperand(1).getImm()), RegState::Undef); + MIB.add(MI.getOperand(2)); + + MI.eraseFromParent(); // The pseudo is gone now. + return BB; + } } } diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td index 99deacc811a170..e2cf12956d934e 100644 --- a/llvm/lib/Target/X86/X86InstrAMX.td +++ b/llvm/lib/Target/X86/X86InstrAMX.td @@ -267,3 +267,150 @@ let Predicates = [HasAMXCOMPLEX, In64BitMode] in { } } // SchedRW = [WriteSystem] } + +multiclass m_tcvtrowd2ps { + let Predicates = [HasAMXAVX512, In64BitMode] in { + let SchedRW = [WriteSystem] in { + def rri : Ii8<0x7, MRMSrcReg, (outs VR512:$dst), + (ins TILE:$src1, i32u8imm:$src2), + "tcvtrowd2ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", + []>, TA,XS, EVEX, EVEX_V512; + def rre : I<0x4A, MRMSrcReg4VOp3, (outs VR512:$dst), + (ins TILE:$src1, GR32:$src2), + "tcvtrowd2ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", + []>, T8,XS, EVEX, VVVV, EVEX_V512; + } + } // HasAMXAVX512, In64BitMode +} + +defm TCVTROWD2PS : m_tcvtrowd2ps; + +let Predicates = [HasAMXAVX512, In64BitMode] in { + let SchedRW = [WriteSystem] in { + let usesCustomInserter = 1 in { + def PTCVTROWD2PSrri : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2), + [(set VR512:$dst, (int_x86_tcvtrowd2ps timm:$src1, imm:$src2))]>; + def PTCVTROWD2PSrre : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2), + [(set VR512:$dst, (int_x86_tcvtrowd2ps timm:$src1, GR32:$src2))]>; + } + + def PTCVTROWD2PSrriV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowd2ps_internal GR16:$src1, GR16:$src2, + TILE:$src3, imm:$src4))]>; + def PTCVTROWD2PSrreV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowd2ps_internal GR16:$src1, GR16:$src2, + TILE:$src3, GR32:$src4))]>; + def PTCVTROWPS2PBF16HrriV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2pbf16h_internal GR16:$src1, GR16:$src2, + TILE:$src3, imm:$src4))]>; + def PTCVTROWPS2PBF16HrreV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2pbf16h_internal GR16:$src1, GR16:$src2, + TILE:$src3, GR32:$src4))]>; + def PTCVTROWPS2PBF16LrriV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2pbf16l_internal GR16:$src1, GR16:$src2, + TILE:$src3, imm:$src4))]>; + def PTCVTROWPS2PBF16LrreV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2pbf16l_internal GR16:$src1, GR16:$src2, + TILE:$src3, GR32:$src4))]>; + def PTCVTROWPS2PHHrriV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2phh_internal GR16:$src1, GR16:$src2, + TILE:$src3, imm:$src4))]>; + def PTCVTROWPS2PHHrreV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2phh_internal GR16:$src1, GR16:$src2, + TILE:$src3, GR32:$src4))]>; + def PTCVTROWPS2PHLrriV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2phl_internal GR16:$src1, GR16:$src2, + TILE:$src3, imm:$src4))]>; + def PTCVTROWPS2PHLrreV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4), + [(set VR512: $dst, + (int_x86_tcvtrowps2phl_internal GR16:$src1, GR16:$src2, + TILE:$src3, GR32:$src4))]>; + } +} + +multiclass AMXAVX512_BASE<bits<8> Opcode1, bits<8> Opcode2, string Opstr, + Prefix P1, Prefix P2> { + let Predicates = [HasAMXAVX512, In64BitMode], SchedRW = [WriteSystem] in { + let OpPrefix = P1 in + def rre : I<Opcode1, MRMSrcReg4VOp3, (outs VR512:$dst), + (ins TILE:$src1, GR32:$src2), + !strconcat(Opstr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX, VVVV, EVEX_V512, T8; + let OpPrefix = P2 in + def rri : Ii8<Opcode2, MRMSrcReg, (outs VR512:$dst), + (ins TILE:$src1, i32u8imm:$src2), + !strconcat(Opstr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), + []>, EVEX, EVEX_V512, TA; + let usesCustomInserter = 1 in { + def "P"#NAME#"rre" : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2), + [(set VR512:$dst, + (!cast<Intrinsic>("int_x86_"#Opstr) timm:$src1, GR32:$src2))]>; + def "P"#NAME#"rri" : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2), + [(set VR512:$dst, + (!cast<Intrinsic>("int_x86_"#Opstr) timm:$src1, imm:$src2))]>; + } + } +} + +defm TCVTROWPS2PHH : AMXAVX512_BASE<0x6d, 0x07, "tcvtrowps2phh", PS, PS>; +defm TCVTROWPS2PHL : AMXAVX512_BASE<0x6d, 0x77, "tcvtrowps2phl", PD, XD>; +defm TCVTROWPS2PBF16H : AMXAVX512_BASE<0x6d, 0x07, "tcvtrowps2pbf16h", XD, XD>; +defm TCVTROWPS2PBF16L : AMXAVX512_BASE<0x6d, 0x77, "tcvtrowps2pbf16l", XS, XS>; + +multiclass m_tilemovrow { + let Predicates = [HasAMXAVX512, In64BitMode] in { + let SchedRW = [WriteSystem] in { + def rri : Ii8<0x7, MRMSrcReg, (outs VR512:$dst), + (ins TILE:$src1, u8imm:$src2), + "tilemovrow\t{$src2, $src1, $dst|$dst, $src1, $src2}", + []>, TA,PD, EVEX, EVEX_V512; + def rre : I<0x4A, MRMSrcReg4VOp3, (outs VR512:$dst), + (ins TILE:$src1, GR32:$src2), + "tilemovrow\t{$src2, $src1, $dst|$dst, $src1, $src2}", + []>, T8,PD, EVEX, VVVV, EVEX_V512; + } + } // HasAMXAVX512, In64BitMode +} + +defm TILEMOVROW : m_tilemovrow; + +let Predicates = [HasAMXAVX512, In64BitMode] in { + let SchedRW = [WriteSystem] in { + let usesCustomInserter = 1 in { + def PTILEMOVROWrri : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2), + [(set VR512:$dst, (int_x86_tilemovrow timm:$src1, imm:$src2))]>; + def PTILEMOVROWrre : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, GR32:$src2), + [(set VR512:$dst, (int_x86_tilemovrow timm:$src1, GR32:$src2))]>; + } + + def PTILEMOVROWrriV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, i32u8imm:$src4), + [(set VR512: $dst, + (int_x86_tilemovrow_internal GR16:$src1, GR16:$src2, + TILE:$src3, imm:$src4))]>; + def PTILEMOVROWrreV : PseudoI<(outs VR512:$dst), + (ins GR16:$src1, GR16:$src2, TILE:$src3, GR32:$src4), + [(set VR512: $dst, + (int_x86_tilemovrow_internal GR16:$src1, GR16:$src2, + TILE:$src3, GR32:$src4))]>; + } +} diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td index a815ddc9714f0c..97da03e5d3fdf2 100644 --- a/llvm/lib/Target/X86/X86InstrPredicates.td +++ b/llvm/lib/Target/X86/X86InstrPredicates.td @@ -182,6 +182,7 @@ def HasAMXTILE : Predicate<"Subtarget->hasAMXTILE()">; def HasAMXBF16 : Predicate<"Subtarget->hasAMXBF16()">; def HasAMXINT8 : Predicate<"Subtarget->hasAMXINT8()">; def HasAMXCOMPLEX : Predicate<"Subtarget->hasAMXCOMPLEX()">; +def HasAMXAVX512 : Predicate<"Subtarget->hasAMXAVX512()">; def HasUINTR : Predicate<"Subtarget->hasUINTR()">; def HasUSERMSR : Predicate<"Subtarget->hasUSERMSR()">; def HasCRC32 : Predicate<"Subtarget->hasCRC32()">; diff --git a/llvm/lib/Target/X86/X86LowerAMXType.cpp b/llvm/lib/Target/X86/X86LowerAMXType.cpp index 919e1eb3e38e9a..063f2e9a614f6f 100644 --- a/llvm/lib/Target/X86/X86LowerAMXType.cpp +++ b/llvm/lib/Target/X86/X86LowerAMXType.cpp @@ -185,6 +185,17 @@ static std::pair<Value *, Value *> getShape(IntrinsicInst *II, unsigned OpNo) { } break; } + case Intrinsic::x86_tcvtrowd2ps_internal: + case Intrinsic::x86_tcvtrowps2pbf16h_internal: + case Intrinsic::x86_tcvtrowps2pbf16l_internal: + case Intrinsic::x86_tcvtrowps2phh_internal: + case Intrinsic::x86_tcvtrowps2phl_internal: + case Intrinsic::x86_tilemovrow_internal: { + assert(OpNo == 2 && "Illegal Operand Number."); + Row = II->getArgOperand(0); + Col = II->getArgOperand(1); + break; + } } return std::make_pair(Row, Col); diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp index 1d1885a3dcd246..b9cc0d1fe22e2b 100644 --- a/llvm/lib/Target/X86/X86PreTileConfig.cpp +++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp @@ -118,6 +118,22 @@ class X86PreTileConfig : public MachineFunctionPass { bool isAMXInstruction(MachineInstr &MI) { if (MI.isPHI() || MI.isDebugInstr() || MI.getNumOperands() < 3) return false; + switch (MI.getOpcode()) { + case X86::PTILESTOREDV: + case X86::PTCVTROWD2PSrreV: + case X86::PTCVTROWD2PSrriV: + case X86::PTCVTROWPS2PBF16HrreV: + case X86::PTCVTROWPS2PBF16HrriV: + case X86::PTCVTROWPS2PBF16LrreV: + case X86::PTCVTROWPS2PBF16LrriV: + case X86::PTCVTROWPS2PHHrreV: + case X86::PTCVTROWPS2PHHrriV: + case X86::PTCVTROWPS2PHLrreV: + case X86::PTCVTROWPS2PHLrriV: + case X86::PTILEMOVROWrreV: + case X86::PTILEMOVROWrriV: + return true; + } MachineOperand &MO = MI.getOperand(0); // We can simply check if it is AMX instruction by its def. // But we should exclude old API which uses physical registers. @@ -126,8 +142,7 @@ class X86PreTileConfig : public MachineFunctionPass { collectShapeInfo(MI); return true; } - // PTILESTOREDV is the only exception that doesn't def a AMX register. - return MI.getOpcode() == X86::PTILESTOREDV; + return false; } /// Check if it is an edge from loop bottom to loop head. diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index 1f608f47ef79f4..aa1419b7451524 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -1878,6 +1878,10 @@ const StringMap<bool> sys::getHostCPUFeatures() { bool HasLeaf24 = MaxLevel >= 0x24 && !getX86CpuIDAndInfo(0x24, &EAX, &EBX, &ECX, &EDX); + bool HasLeaf1E = MaxLevel >= 0x1e && + !getX86CpuIDAndInfoEx(0x1e, 0x1, &EAX, &EBX, &ECX, &EDX); + Features["amx-avx512"] = HasLeaf1E && ((EAX >> 7) & 1) && HasAMXSave; + int AVX10Ver = HasLeaf24 && (EBX & 0xff); int Has512Len = HasLeaf24 && ((EBX >> 18) & 1); Features["avx10.1-256"] = HasAVX10 && AVX10Ver >= 1; diff --git a/llvm/lib/TargetParser/X86TargetParser.cpp b/llvm/lib/TargetParser/X86TargetParser.cpp index 09d4312918acfe..b923cb052e62e1 100644 --- a/llvm/lib/TargetParser/X86TargetParser.cpp +++ b/llvm/lib/TargetParser/X86TargetParser.cpp @@ -598,6 +598,8 @@ constexpr FeatureBitset ImpliedFeaturesAMX_BF16 = FeatureAMX_TILE; constexpr FeatureBitset ImpliedFeaturesAMX_FP16 = FeatureAMX_TILE; constexpr FeatureBitset ImpliedFeaturesAMX_INT8 = FeatureAMX_TILE; constexpr FeatureBitset ImpliedFeaturesAMX_COMPLEX = FeatureAMX_TILE; +constexpr FeatureBitset ImpliedFeaturesAMX_AVX512 = + FeatureAMX_TILE | FeatureAVX10_2_512; constexpr FeatureBitset ImpliedFeaturesHRESET = {}; constexpr FeatureBitset ImpliedFeaturesPREFETCHI = {}; diff --git a/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll b/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll new file mode 100644 index 00000000000000..71f8f231747fe7 --- /dev/null +++ b/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll @@ -0,0 +1,171 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx10.2-512 -mattr=+amx-avx512 -verify-machineinstrs | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx10.2-512 -mattr=+amx-avx512 -verify-machineinstrs -enable-ipra | FileCheck -check-prefix=IPRA %s +; RUN: llc < %s -O0 -mtriple=x86_64-unknown-unknown -mattr=+amx-int8 -mattr=+avx10.2-512 -mattr=+amx-avx512 -verify-machineinstrs | FileCheck -check-prefix=O0 %s + +@buf = dso_local global [3072 x i8] zeroinitializer, align 64 + +define internal void @foo() { +; CHECK-LABEL: foo: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: retq +; +; IPRA-LABEL: foo: +; IPRA: # %bb.0: # %entry +; IPRA-NEXT: retq +; +; O0-LABEL: foo: +; O0: # %bb.0: # %entry +; O0-NEXT: retq +entry: + ret void +} + +define dso_local <16 x i32> @test_api(i16 signext %0, i16 signext %1) nounwind { +; CHECK-LABEL: test_api: +; CHECK: # %bb.0: +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: pushq %r14 +; CHECK-NEXT: pushq %rbx +; CHECK-NEXT: subq $2112, %rsp # imm = 0x840 +; CHECK-NEXT: movl %esi, %ebx +; CHECK-NEXT: movl %edi, %ebp +; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: vmovups %zmm0, (%rsp) +; CHECK-NEXT: movb $1, (%rsp) +; CHECK-NEXT: movw $8, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movb $8, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movw %bx, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movb %bpl, {{[0-9]+}}(%rsp) +; CHECK-NEXT: ldtilecfg (%rsp) +; CHECK-NEXT: movl $buf, %eax +; CHECK-NEXT: movl $32, %ecx +; CHECK-NEXT: movw $8, %r14w +; CHECK-NEXT: tileloadd (%rax,%rcx), %tmm0 +; CHECK-NEXT: movabsq $64, %rax +; CHECK-NEXT: tilestored %tmm0, 1088(%rsp,%rax) # 1024-byte Folded Spill +; CHECK-NEXT: movl $buf+1024, %eax +; CHECK-NEXT: tileloadd (%rax,%rcx), %tmm1 +; CHECK-NEXT: movabsq $64, %rax +; CHECK-NEXT: tilestored %tmm1, 64(%rsp,%rax) # 1024-byte Folded Spill +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: callq foo +; CHECK-NEXT: ldtilecfg (%rsp) +; CHECK-NEXT: movabsq $64, %rax +; CHECK-NEXT: tileloadd 64(%rsp,%rax), %tmm1 # 1024-byte Folded Reload +; CHECK-NEXT: tilemovrow $2, %tmm1, %zmm0 +; CHECK-NEXT: tileloadd 1088(%rsp,%rax), %tmm0 # 1024-byte Folded Reload +; CHECK-NEXT: tilemovrow $2, %tmm0, %zmm1 +; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0 +; CHECK-NEXT: addq $2112, %rsp # imm = 0x840 +; CHECK-NEXT: popq %rbx +; CHECK-NEXT: popq %r14 +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: tilerelease +; CHECK-NEXT: retq +; +; IPRA-LABEL: test_api: +; IPRA: # %bb.0: +; IPRA-NEXT: subq $72, %rsp +; IPRA-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; IPRA-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp) +; IPRA-NEXT: movb $1, {{[0-9]+}}(%rsp) +; IPRA-NEXT: movw $8, {{[0-9]+}}(%rsp) +; IPRA-NEXT: movb $8, {{[0-9]+}}(%rsp) +; IPRA-NEXT: movw %si, {{[0-9]+}}(%rsp) +; IPRA-NEXT: movb %dil, {{[0-9]+}}(%rsp) +; IPRA-NEXT: ldtilecfg {{[0-9]+}}(%rsp) +; IPRA-NEXT: movl $buf, %eax +; IPRA-NEXT: movl $32, %ecx +; IPRA-NEXT: movw $8, %dx +; IPRA-NEXT: tileloadd (%rax,%rcx), %tmm0 +; IPRA-NEXT: movl $buf+1024, %eax +; IPRA-NEXT: tileloadd (%rax,%rcx), %tmm1 +; IPRA-NEXT: callq foo +; IPRA-NEXT: tilemovrow $2, %tmm1, %zmm0 +; IPRA-NEXT: tilemovrow $2, %tmm0, %zmm1 +; IPRA-NEXT: vpaddd %zmm1, %zmm0, %zmm0 +; IPRA-NEXT: addq $72, %rsp +; IPRA-NEXT: tilerelease +; IPRA-NEXT: retq +; +; O0-LABEL: test_api: +; O0: # %bb.0: +; O0-NEXT: pushq %rbp +; O0-NEXT: movq %rsp, %rbp +; O0-NEXT: andq $-1024, %rsp # imm = 0xFC00 +; O0-NEXT: subq $4096, %rsp # imm = 0x1000 +; O0-NEXT: vpxor %xmm0, %xmm0, %xmm0 +; O0-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp) +; O0-NEXT: movb $1, {{[0-9]+}}(%rsp) +; O0-NEXT: movw %si, %cx +; O0-NEXT: movw %cx, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill +; O0-NEXT: movw %di, %ax +; O0-NEXT: movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill +; O0-NEXT: movl $buf, %esi +; O0-NEXT: movl $32, %edi +; O0-NEXT: movw $8, %dx +; O0-NEXT: # implicit-def: $al +; O0-NEXT: movb %al, {{[0-9]+}}(%rsp) +; O0-NEXT: movw %dx, {{[0-9]+}}(%rsp) +; O0-NEXT: ldtilecfg {{[0-9]+}}(%rsp) +; O0-NEXT: tileloadd (%rsi,%rdi), %tmm0 +; O0-NEXT: movl $64, %edi +; O0-NEXT: leaq {{[0-9]+}}(%rsp), %rsi +; O0-NEXT: movw $8, %dx +; O0-NEXT: tilestored %tmm0, (%rsi,%rdi) +; O0-NEXT: movl $32, %esi +; O0-NEXT: movl $buf+1024, %edx +; O0-NEXT: movw $8, %ax +; O0-NEXT: # implicit-def: $al +; O0-NEXT: movb %al, {{[0-9]+}}(%rsp) +; O0-NEXT: movw %cx, {{[0-9]+}}(%rsp) +; O0-NEXT: ldtilecfg {{[0-9]+}}(%rsp) +; O0-NEXT: tileloadd (%rdx,%rsi), %tmm0 +; O0-NEXT: movl $64, %esi +; O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx +; O0-NEXT: movw $8, %ax +; O0-NEXT: tilestored %tmm0, (%rdx,%rsi) +; O0-NEXT: vzeroupper +; O0-NEXT: callq foo +; O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %dx # 2-byte Reload +; O0-NEXT: movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload +; O0-NEXT: movl $64, %edi +; O0-NEXT: leaq {{[0-9]+}}(%rsp), %rsi +; O0-NEXT: movw $8, %cx +; O0-NEXT: # implicit-def: $cl +; O0-NEXT: movb %cl, {{[0-9]+}}(%rsp) +; O0-NEXT: movw %dx, {{[0-9]+}}(%rsp) +; O0-NEXT: ldtilecfg {{[0-9]+}}(%rsp) +; O0-NEXT: tileloadd (%rsi,%rdi), %tmm0 +; O0-NEXT: movw $8, %cx +; O0-NEXT: tilemovrow $2, %tmm0, %zmm0 +; O0-NEXT: movl $64, %esi +; O0-NEXT: leaq {{[0-9]+}}(%rsp), %rdx +; O0-NEXT: movw $8, %cx +; O0-NEXT: # implicit-def: $al +; O0-NEXT: movb %al, {{[0-9]+}}(%rsp) +; O0-NEXT: movw %cx, {{[0-9]+}}(%rsp) +; O0-NEXT: ldtilecfg {{[0-9]+}}(%rsp) +; O0-NEXT: tileloadd (%rdx,%rsi), %tmm0 +; O0-NEXT: movw $8, %cx +; O0-NEXT: tilemovrow $2, %tmm0, %zmm1 +; O0-NEXT: vpaddd %zmm1, %zmm0, %zmm0 +; O0-NEXT: movq %rbp, %rsp +; O0-NEXT: popq %rbp +; O0-NEXT: tilerelease +; O0-NEXT: retq + %3 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 %0, i16 8, ptr @buf, i64 32) + %4 = tail call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 %1, ptr getelementptr inbounds ([3072 x i8], ptr @buf, i64 0, i64 1024), i64 32) + call void @foo() + %5 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 8, i16 %1, x86_amx %4, i32 2) + %6 = call <16 x i32> @llvm.x86.tilemovrow.internal(i16 %0, i16 8, x86_amx %3, i32 2) + %7 = add <16 x i32> %5, %6 + ret <16 x i32> %7 +} + + +declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, ptr, i64) +declare x86_amx @llvm.x86.tdpbssd.internal(i16, i16, i16, x86_amx, x86_amx, x86_amx) +declare void @llvm.x86.tilestored64.internal(i16, i16, ptr, i64, x86_amx) +declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32) diff --git a/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll b/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll new file mode 100644 index 00000000000000..da7fedee88821b --- /dev/null +++ b/llvm/test/CodeGen/X86/amx-avx512-intrinsics.ll @@ -0,0 +1,116 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -O0 -verify-machineinstrs -mtriple=x86_64-unknown-unknown --show-mc-encoding -mattr=+amx-tile,+amx-avx512,+avx10.2-512 | FileCheck %s + +define <16 x float> @test_tcvtrowd2ps(i32 %A) { +; CHECK-LABEL: test_tcvtrowd2ps: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowd2ps %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x46,0x48,0x4a,0xc1] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <16 x float> @llvm.x86.tcvtrowd2ps(i8 1, i32 %A) + ret <16 x float> %ret +} + +define <16 x float> @test_tcvtrowd2psi() { +; CHECK-LABEL: test_tcvtrowd2psi: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowd2ps $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x07,0xc1,0x7f] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <16 x float> @llvm.x86.tcvtrowd2ps(i8 1, i32 127) + ret <16 x float> %ret +} +declare <16 x float> @llvm.x86.tcvtrowd2ps(i8 %A, i32 %B) + +define <32 x bfloat> @test_tcvtrowps2pbf16h(i32 %A) { +; CHECK-LABEL: test_tcvtrowps2pbf16h: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2pbf16h %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x47,0x48,0x6d,0xc1] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h(i8 1, i32 %A) + ret <32 x bfloat> %ret +} + +define <32 x bfloat> @test_tcvtrowps2pbf16hi() { +; CHECK-LABEL: test_tcvtrowps2pbf16hi: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2pbf16h $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x07,0xc1,0x7f] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h(i8 1, i32 127) + ret <32 x bfloat> %ret +} +declare <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h(i8 %A, i32 %B) + +define <32 x bfloat> @test_tcvtrowps2pbf16l(i32 %A) { +; CHECK-LABEL: test_tcvtrowps2pbf16l: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2pbf16l %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x46,0x48,0x6d,0xc1] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l(i8 1, i32 %A) + ret <32 x bfloat> %ret +} + +define <32 x bfloat> @test_tcvtrowps2pbf16li() { +; CHECK-LABEL: test_tcvtrowps2pbf16li: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2pbf16l $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7e,0x48,0x77,0xc1,0x7f] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l(i8 1, i32 127) + ret <32 x bfloat> %ret +} +declare <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l(i8 %A, i32 %B) + +define <32 x half> @test_tcvtrowps2phh(i32 %A) { +; CHECK-LABEL: test_tcvtrowps2phh: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2phh %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x44,0x48,0x6d,0xc1] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x half> @llvm.x86.tcvtrowps2phh(i8 1, i32 %A) + ret <32 x half> %ret +} + +define <32 x half> @test_tcvtrowps2phhi() { +; CHECK-LABEL: test_tcvtrowps2phhi: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2phh $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7c,0x48,0x07,0xc1,0x7f] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x half> @llvm.x86.tcvtrowps2phh(i8 1, i32 127) + ret <32 x half> %ret +} +declare <32 x half> @llvm.x86.tcvtrowps2phh(i8 %A, i32 %B) + +define <32 x half> @test_tcvtrowps2phl(i32 %A) { +; CHECK-LABEL: test_tcvtrowps2phl: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2phl %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x45,0x48,0x6d,0xc1] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x half> @llvm.x86.tcvtrowps2phl(i8 1, i32 %A) + ret <32 x half> %ret +} + +define <32 x half> @test_tcvtrowps2phli() { +; CHECK-LABEL: test_tcvtrowps2phli: +; CHECK: # %bb.0: +; CHECK-NEXT: tcvtrowps2phl $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7f,0x48,0x77,0xc1,0x7f] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <32 x half> @llvm.x86.tcvtrowps2phl(i8 1, i32 127) + ret <32 x half> %ret +} +declare <32 x half> @llvm.x86.tcvtrowps2phl(i8 %A, i32 %B) + +define <16 x i32> @test_tilemovrow(i32 %A) { +; CHECK-LABEL: test_tilemovrow: +; CHECK: # %bb.0: +; CHECK-NEXT: tilemovrow %edi, %tmm1, %zmm0 # encoding: [0x62,0xf2,0x45,0x48,0x4a,0xc1] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <16 x i32> @llvm.x86.tilemovrow(i8 1, i32 %A) + ret <16 x i32> %ret +} + +define <16 x i32> @test_tilemovrowi() { +; CHECK-LABEL: test_tilemovrowi: +; CHECK: # %bb.0: +; CHECK-NEXT: tilemovrow $127, %tmm1, %zmm0 # encoding: [0x62,0xf3,0x7d,0x48,0x07,0xc1,0x7f] +; CHECK-NEXT: retq # encoding: [0xc3] + %ret = call <16 x i32> @llvm.x86.tilemovrow(i8 1, i32 127) + ret <16 x i32> %ret +} +declare <16 x i32> @llvm.x86.tilemovrow(i8 %A, i32 %B) diff --git a/llvm/test/CodeGen/X86/amx-tile-avx512-internals.ll b/llvm/test/CodeGen/X86/amx-tile-avx512-internals.ll new file mode 100644 index 00000000000000..b4a5c90bbea330 --- /dev/null +++ b/llvm/test/CodeGen/X86/amx-tile-avx512-internals.ll @@ -0,0 +1,61 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+amx-tile,+amx-bf16,+avx10.2-512, \ +; RUN: -mattr=+amx-avx512 -verify-machineinstrs | FileCheck %s + +define void @test_amx(i8* %pointer, i8* %base, i32 %index, i64 %stride) { +; CHECK-LABEL: test_amx: +; CHECK: # %bb.0: +; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; CHECK-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp) +; CHECK-NEXT: movb $1, -{{[0-9]+}}(%rsp) +; CHECK-NEXT: movb $8, -{{[0-9]+}}(%rsp) +; CHECK-NEXT: movw $8, -{{[0-9]+}}(%rsp) +; CHECK-NEXT: ldtilecfg -{{[0-9]+}}(%rsp) +; CHECK-NEXT: movw $8, %ax +; CHECK-NEXT: tileloadd (%rsi,%rcx), %tmm0 +; CHECK-NEXT: tcvtrowd2ps %edx, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowd2ps $16, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2pbf16h %edx, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2pbf16h $16, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2pbf16l %edx, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2pbf16l $16, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2phh %edx, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2phh $16, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2phl %edx, %tmm0, %zmm0 +; CHECK-NEXT: tcvtrowps2phl $16, %tmm0, %zmm0 +; CHECK-NEXT: tilemovrow %edx, %tmm0, %zmm0 +; CHECK-NEXT: tilemovrow $16, %tmm0, %zmm0 +; CHECK-NEXT: tilestored %tmm0, (%rdi,%rcx) +; CHECK-NEXT: tilerelease +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq + + %a = call x86_amx @llvm.x86.tileloadd64.internal(i16 8, i16 8, i8* %base, i64 %stride) + call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 8, i16 8, x86_amx %a, i32 %index) + call <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16 8, i16 8, x86_amx %a, i32 16) + call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h.internal(i16 8, i16 8, x86_amx %a, i32 %index) + call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h.internal(i16 8, i16 8, x86_amx %a, i32 16) + call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l.internal(i16 8, i16 8, x86_amx %a, i32 %index) + call <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l.internal(i16 8, i16 8, x86_amx %a, i32 16) + call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 8, i16 8, x86_amx %a, i32 %index) + call <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16 8, i16 8, x86_amx %a, i32 16) + call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 8, i16 8, x86_amx %a, i32 %index) + call <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16 8, i16 8, x86_amx %a, i32 16) + call <16 x i32> @llvm.x86.tilemovrow.internal(i16 8, i16 8, x86_amx %a, i32 %index) + call <16 x i32> @llvm.x86.tilemovrow.internal(i16 8, i16 8, x86_amx %a, i32 16) + + call void @llvm.x86.tilestored64.internal(i16 8, i16 8, i8* %pointer, i64 %stride, x86_amx %a) + ret void +} + +declare x86_amx @llvm.x86.tilezero.internal(i16, i16) +declare x86_amx @llvm.x86.tileloadd64.internal(i16, i16, i8*, i64) +declare x86_amx @llvm.x86.tileloaddt164.internal(i16, i16, i8*, i64) +declare void @llvm.x86.tilestored64.internal(i16, i16, i8*, i64, x86_amx) + +declare <16 x float> @llvm.x86.tcvtrowd2ps.internal(i16, i16, x86_amx, i32) +declare <32 x bfloat> @llvm.x86.tcvtrowps2pbf16h.internal(i16, i16, x86_amx, i32) +declare <32 x bfloat> @llvm.x86.tcvtrowps2pbf16l.internal(i16, i16, x86_amx, i32) +declare <32 x half> @llvm.x86.tcvtrowps2phh.internal(i16, i16, x86_amx, i32) +declare <32 x half> @llvm.x86.tcvtrowps2phl.internal(i16, i16, x86_amx, i32) +declare <16 x i32> @llvm.x86.tilemovrow.internal(i16, i16, x86_amx, i32) diff --git a/llvm/test/MC/Disassembler/X86/amx-avx512.txt b/llvm/test/MC/Disassembler/X86/amx-avx512.txt new file mode 100644 index 00000000000000..0a162af1b4bc02 --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/amx-avx512.txt @@ -0,0 +1,106 @@ +# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT +# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL + +# ATT: tcvtrowd2ps %ecx, %tmm5, %zmm22 +# INTEL: tcvtrowd2ps zmm22, tmm5, ecx +0x62,0xe2,0x76,0x48,0x4a,0xf5 + +# ATT: tcvtrowd2ps %ecx, %tmm2, %zmm22 +# INTEL: tcvtrowd2ps zmm22, tmm2, ecx +0x62,0xe2,0x76,0x48,0x4a,0xf2 + +# ATT: tcvtrowd2ps $123, %tmm5, %zmm22 +# INTEL: tcvtrowd2ps zmm22, tmm5, 123 +0x62,0xe3,0x7e,0x48,0x07,0xf5,0x7b + +# ATT: tcvtrowd2ps $123, %tmm2, %zmm22 +# INTEL: tcvtrowd2ps zmm22, tmm2, 123 +0x62,0xe3,0x7e,0x48,0x07,0xf2,0x7b + +# ATT: tcvtrowps2pbf16h %ecx, %tmm5, %zmm22 +# INTEL: tcvtrowps2pbf16h zmm22, tmm5, ecx +0x62,0xe2,0x77,0x48,0x6d,0xf5 + +# ATT: tcvtrowps2pbf16h %ecx, %tmm2, %zmm22 +# INTEL: tcvtrowps2pbf16h zmm22, tmm2, ecx +0x62,0xe2,0x77,0x48,0x6d,0xf2 + +# ATT: tcvtrowps2pbf16h $123, %tmm5, %zmm22 +# INTEL: tcvtrowps2pbf16h zmm22, tmm5, 123 +0x62,0xe3,0x7f,0x48,0x07,0xf5,0x7b + +# ATT: tcvtrowps2pbf16h $123, %tmm2, %zmm22 +# INTEL: tcvtrowps2pbf16h zmm22, tmm2, 123 +0x62,0xe3,0x7f,0x48,0x07,0xf2,0x7b + +# ATT: tcvtrowps2pbf16l %ecx, %tmm5, %zmm22 +# INTEL: tcvtrowps2pbf16l zmm22, tmm5, ecx +0x62,0xe2,0x76,0x48,0x6d,0xf5 + +# ATT: tcvtrowps2pbf16l %ecx, %tmm2, %zmm22 +# INTEL: tcvtrowps2pbf16l zmm22, tmm2, ecx +0x62,0xe2,0x76,0x48,0x6d,0xf2 + +# ATT: tcvtrowps2pbf16l $123, %tmm5, %zmm22 +# INTEL: tcvtrowps2pbf16l zmm22, tmm5, 123 +0x62,0xe3,0x7e,0x48,0x77,0xf5,0x7b + +# ATT: tcvtrowps2pbf16l $123, %tmm2, %zmm22 +# INTEL: tcvtrowps2pbf16l zmm22, tmm2, 123 +0x62,0xe3,0x7e,0x48,0x77,0xf2,0x7b + +# ATT: tcvtrowps2phh %ecx, %tmm5, %zmm22 +# INTEL: tcvtrowps2phh zmm22, tmm5, ecx +0x62,0xe2,0x74,0x48,0x6d,0xf5 + +# ATT: tcvtrowps2phh %ecx, %tmm2, %zmm22 +# INTEL: tcvtrowps2phh zmm22, tmm2, ecx +0x62,0xe2,0x74,0x48,0x6d,0xf2 + +# ATT: tcvtrowps2phh $123, %tmm5, %zmm22 +# INTEL: tcvtrowps2phh zmm22, tmm5, 123 +0x62,0xe3,0x7c,0x48,0x07,0xf5,0x7b + +# ATT: tcvtrowps2phh $123, %tmm2, %zmm22 +# INTEL: tcvtrowps2phh zmm22, tmm2, 123 +0x62,0xe3,0x7c,0x48,0x07,0xf2,0x7b + +# ATT: tcvtrowps2phl %ecx, %tmm5, %zmm22 +# INTEL: tcvtrowps2phl zmm22, tmm5, ecx +0x62,0xe2,0x75,0x48,0x6d,0xf5 + +# ATT: tcvtrowps2phl %ecx, %tmm2, %zmm22 +# INTEL: tcvtrowps2phl zmm22, tmm2, ecx +0x62,0xe2,0x75,0x48,0x6d,0xf2 + +# ATT: tcvtrowps2phl $123, %tmm5, %zmm22 +# INTEL: tcvtrowps2phl zmm22, tmm5, 123 +0x62,0xe3,0x7f,0x48,0x77,0xf5,0x7b + +# ATT: tcvtrowps2phl $123, %tmm2, %zmm22 +# INTEL: tcvtrowps2phl zmm22, tmm2, 123 +0x62,0xe3,0x7f,0x48,0x77,0xf2,0x7b + +# ATT: tilemovrow %ecx, %tmm3, %zmm22 +# INTEL: tilemovrow zmm22, tmm3, ecx +0x62,0xe2,0x75,0x48,0x4a,0xf3 + +# ATT: tilemovrow %ecx, %tmm2, %zmm22 +# INTEL: tilemovrow zmm22, tmm2, ecx +0x62,0xe2,0x75,0x48,0x4a,0xf2 + +# ATT: tilemovrow $123, %tmm3, %zmm22 +# INTEL: tilemovrow zmm22, tmm3, 123 +0x62,0xe3,0x7d,0x48,0x07,0xf3,0x7b + +# ATT: tilemovrow $123, %tmm2, %zmm22 +# INTEL: tilemovrow zmm22, tmm2, 123 +0x62,0xe3,0x7d,0x48,0x07,0xf2,0x7b + +# ATT: tilemovrow %edx, %tmm0, %zmm22 +# INTEL: tilemovrow zmm22, tmm0, edx +0x62,0xe2,0x6d,0x48,0x4a,0xf0 + +# ATT: tilemovrow $123, %tmm0, %zmm22 +# INTEL: tilemovrow zmm22, tmm0, 123 +0x62,0xe3,0x7d,0x48,0x07,0xf0,0x7b diff --git a/llvm/test/MC/X86/amx-avx512-att.s b/llvm/test/MC/X86/amx-avx512-att.s new file mode 100644 index 00000000000000..6da4ede82c6217 --- /dev/null +++ b/llvm/test/MC/X86/amx-avx512-att.s @@ -0,0 +1,105 @@ +// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding < %s | FileCheck %s + +// CHECK: tcvtrowd2ps %ecx, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x4a,0xf5] + tcvtrowd2ps %ecx, %tmm5, %zmm22 + +// CHECK: tcvtrowd2ps %ecx, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x4a,0xf2] + tcvtrowd2ps %ecx, %tmm2, %zmm22 + +// CHECK: tcvtrowd2ps $123, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x07,0xf5,0x7b] + tcvtrowd2ps $123, %tmm5, %zmm22 + +// CHECK: tcvtrowd2ps $123, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x07,0xf2,0x7b] + tcvtrowd2ps $123, %tmm2, %zmm22 + +// CHECK: tcvtrowps2pbf16h %ecx, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x77,0x48,0x6d,0xf5] + tcvtrowps2pbf16h %ecx, %tmm5, %zmm22 + +// CHECK: tcvtrowps2pbf16h %ecx, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x77,0x48,0x6d,0xf2] + tcvtrowps2pbf16h %ecx, %tmm2, %zmm22 + +// CHECK: tcvtrowps2pbf16h $123, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x07,0xf5,0x7b] + tcvtrowps2pbf16h $123, %tmm5, %zmm22 + +// CHECK: tcvtrowps2pbf16h $123, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x07,0xf2,0x7b] + tcvtrowps2pbf16h $123, %tmm2, %zmm22 + +// CHECK: tcvtrowps2pbf16l %ecx, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x6d,0xf5] + tcvtrowps2pbf16l %ecx, %tmm5, %zmm22 + +// CHECK: tcvtrowps2pbf16l %ecx, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x6d,0xf2] + tcvtrowps2pbf16l %ecx, %tmm2, %zmm22 + +// CHECK: tcvtrowps2pbf16l $123, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x77,0xf5,0x7b] + tcvtrowps2pbf16l $123, %tmm5, %zmm22 + +// CHECK: tcvtrowps2pbf16l $123, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x77,0xf2,0x7b] + tcvtrowps2pbf16l $123, %tmm2, %zmm22 + +// CHECK: tcvtrowps2phh %ecx, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x74,0x48,0x6d,0xf5] + tcvtrowps2phh %ecx, %tmm5, %zmm22 + +// CHECK: tcvtrowps2phh %ecx, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x74,0x48,0x6d,0xf2] + tcvtrowps2phh %ecx, %tmm2, %zmm22 + +// CHECK: tcvtrowps2phh $123, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7c,0x48,0x07,0xf5,0x7b] + tcvtrowps2phh $123, %tmm5, %zmm22 + +// CHECK: tcvtrowps2phh $123, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7c,0x48,0x07,0xf2,0x7b] + tcvtrowps2phh $123, %tmm2, %zmm22 + +// CHECK: tcvtrowps2phl %ecx, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x6d,0xf5] + tcvtrowps2phl %ecx, %tmm5, %zmm22 + +// CHECK: tcvtrowps2phl %ecx, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x6d,0xf2] + tcvtrowps2phl %ecx, %tmm2, %zmm22 + +// CHECK: tcvtrowps2phl $123, %tmm5, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x77,0xf5,0x7b] + tcvtrowps2phl $123, %tmm5, %zmm22 + +// CHECK: tcvtrowps2phl $123, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x77,0xf2,0x7b] + tcvtrowps2phl $123, %tmm2, %zmm22 + +// CHECK: tilemovrow %ecx, %tmm3, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x4a,0xf3] + tilemovrow %ecx, %tmm3, %zmm22 + +// CHECK: tilemovrow %ecx, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x4a,0xf2] + tilemovrow %ecx, %tmm2, %zmm22 + +// CHECK: tilemovrow $123, %tmm3, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7d,0x48,0x07,0xf3,0x7b] + tilemovrow $123, %tmm3, %zmm22 + +// CHECK: tilemovrow $123, %tmm2, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7d,0x48,0x07,0xf2,0x7b] + tilemovrow $123, %tmm2, %zmm22 + +// CHECK: tilemovrow %edx, %tmm0, %zmm22 +// CHECK: encoding: [0x62,0xe2,0x6d,0x48,0x4a,0xf0] + tilemovrow %edx, %tmm0, %zmm22 + +// CHECK: tilemovrow $123, %tmm0, %zmm22 +// CHECK: encoding: [0x62,0xe3,0x7d,0x48,0x07,0xf0,0x7b] + tilemovrow $123, %tmm0, %zmm22 diff --git a/llvm/test/MC/X86/amx-avx512-intel.s b/llvm/test/MC/X86/amx-avx512-intel.s new file mode 100644 index 00000000000000..3a517a6cd1aabb --- /dev/null +++ b/llvm/test/MC/X86/amx-avx512-intel.s @@ -0,0 +1,105 @@ +// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s + +// CHECK: tcvtrowd2ps zmm22, tmm5, ecx +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x4a,0xf5] + tcvtrowd2ps zmm22, tmm5, ecx + +// CHECK: tcvtrowd2ps zmm22, tmm2, ecx +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x4a,0xf2] + tcvtrowd2ps zmm22, tmm2, ecx + +// CHECK: tcvtrowd2ps zmm22, tmm5, 123 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x07,0xf5,0x7b] + tcvtrowd2ps zmm22, tmm5, 123 + +// CHECK: tcvtrowd2ps zmm22, tmm2, 123 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x07,0xf2,0x7b] + tcvtrowd2ps zmm22, tmm2, 123 + +// CHECK: tcvtrowps2pbf16h zmm22, tmm5, ecx +// CHECK: encoding: [0x62,0xe2,0x77,0x48,0x6d,0xf5] + tcvtrowps2pbf16h zmm22, tmm5, ecx + +// CHECK: tcvtrowps2pbf16h zmm22, tmm2, ecx +// CHECK: encoding: [0x62,0xe2,0x77,0x48,0x6d,0xf2] + tcvtrowps2pbf16h zmm22, tmm2, ecx + +// CHECK: tcvtrowps2pbf16h zmm22, tmm5, 123 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x07,0xf5,0x7b] + tcvtrowps2pbf16h zmm22, tmm5, 123 + +// CHECK: tcvtrowps2pbf16h zmm22, tmm2, 123 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x07,0xf2,0x7b] + tcvtrowps2pbf16h zmm22, tmm2, 123 + +// CHECK: tcvtrowps2pbf16l zmm22, tmm5, ecx +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x6d,0xf5] + tcvtrowps2pbf16l zmm22, tmm5, ecx + +// CHECK: tcvtrowps2pbf16l zmm22, tmm2, ecx +// CHECK: encoding: [0x62,0xe2,0x76,0x48,0x6d,0xf2] + tcvtrowps2pbf16l zmm22, tmm2, ecx + +// CHECK: tcvtrowps2pbf16l zmm22, tmm5, 123 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x77,0xf5,0x7b] + tcvtrowps2pbf16l zmm22, tmm5, 123 + +// CHECK: tcvtrowps2pbf16l zmm22, tmm2, 123 +// CHECK: encoding: [0x62,0xe3,0x7e,0x48,0x77,0xf2,0x7b] + tcvtrowps2pbf16l zmm22, tmm2, 123 + +// CHECK: tcvtrowps2phh zmm22, tmm5, ecx +// CHECK: encoding: [0x62,0xe2,0x74,0x48,0x6d,0xf5] + tcvtrowps2phh zmm22, tmm5, ecx + +// CHECK: tcvtrowps2phh zmm22, tmm2, ecx +// CHECK: encoding: [0x62,0xe2,0x74,0x48,0x6d,0xf2] + tcvtrowps2phh zmm22, tmm2, ecx + +// CHECK: tcvtrowps2phh zmm22, tmm5, 123 +// CHECK: encoding: [0x62,0xe3,0x7c,0x48,0x07,0xf5,0x7b] + tcvtrowps2phh zmm22, tmm5, 123 + +// CHECK: tcvtrowps2phh zmm22, tmm2, 123 +// CHECK: encoding: [0x62,0xe3,0x7c,0x48,0x07,0xf2,0x7b] + tcvtrowps2phh zmm22, tmm2, 123 + +// CHECK: tcvtrowps2phl zmm22, tmm5, ecx +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x6d,0xf5] + tcvtrowps2phl zmm22, tmm5, ecx + +// CHECK: tcvtrowps2phl zmm22, tmm2, ecx +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x6d,0xf2] + tcvtrowps2phl zmm22, tmm2, ecx + +// CHECK: tcvtrowps2phl zmm22, tmm5, 123 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x77,0xf5,0x7b] + tcvtrowps2phl zmm22, tmm5, 123 + +// CHECK: tcvtrowps2phl zmm22, tmm2, 123 +// CHECK: encoding: [0x62,0xe3,0x7f,0x48,0x77,0xf2,0x7b] + tcvtrowps2phl zmm22, tmm2, 123 + +// CHECK: tilemovrow zmm22, tmm3, ecx +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x4a,0xf3] + tilemovrow zmm22, tmm3, ecx + +// CHECK: tilemovrow zmm22, tmm2, ecx +// CHECK: encoding: [0x62,0xe2,0x75,0x48,0x4a,0xf2] + tilemovrow zmm22, tmm2, ecx + +// CHECK: tilemovrow zmm22, tmm3, 123 +// CHECK: encoding: [0x62,0xe3,0x7d,0x48,0x07,0xf3,0x7b] + tilemovrow zmm22, tmm3, 123 + +// CHECK: tilemovrow zmm22, tmm2, 123 +// CHECK: encoding: [0x62,0xe3,0x7d,0x48,0x07,0xf2,0x7b] + tilemovrow zmm22, tmm2, 123 + +// CHECK: tilemovrow zmm22, tmm0, edx +// CHECK: encoding: [0x62,0xe2,0x6d,0x48,0x4a,0xf0] + tilemovrow zmm22, tmm0, edx + +// CHECK: tilemovrow zmm22, tmm0, 123 +// CHECK: encoding: [0x62,0xe3,0x7d,0x48,0x07,0xf0,0x7b] + tilemovrow zmm22, tmm0, 123 >From c38da4e614434b02158444f31f50aee61f9879f6 Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" <phoebe.w...@intel.com> Date: Wed, 6 Nov 2024 17:55:34 +0800 Subject: [PATCH 2/6] Address review comments --- clang/lib/Headers/amxavx512intrin.h | 24 ++++++++++++------------ llvm/lib/Target/X86/X86ExpandPseudo.cpp | 4 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/clang/lib/Headers/amxavx512intrin.h b/clang/lib/Headers/amxavx512intrin.h index f819696f8086b7..9bfa868cf447a0 100644 --- a/clang/lib/Headers/amxavx512intrin.h +++ b/clang/lib/Headers/amxavx512intrin.h @@ -18,7 +18,7 @@ __attribute__((__always_inline__, __nodebug__, __target__("amx-avx512"))) /// Moves a row from a tile register to a zmm destination register, converting -/// the int32 source elements to fp32. The row of the tile is selected by an +/// the int32 source elements to fp32. The row of the tile is selected by a /// 32b GPR. /// /// \headerfile <x86intrin.h> @@ -46,7 +46,7 @@ /// This intrinsic corresponds to the \c TCVTROWD2PS instruction. /// /// \param tsrc -/// The 1st source tile. Max size is 1024 Bytes. +/// The source tile. Max size is 1024 Bytes. /// \param row /// The row of the source tile #define _tile_cvtrowd2ps(tsrc, row) __builtin_ia32_tcvtrowd2ps(tsrc, row) @@ -54,7 +54,7 @@ /// Moves a row from a tile register to a zmm destination register, converting /// the fp32 source elements to bf16. It places the resulting bf16 elements /// in the high 16 bits within each dword. The row of the tile is selected -/// by an 32b GPR. +/// by a 32b GPR. /// /// \headerfile <x86intrin.h> /// @@ -91,7 +91,7 @@ /// Moves a row from a tile register to a zmm destination register, converting /// the fp32 source elements to bf16. It places the resulting bf16 elements /// in the low 16 bits within each dword. The row of the tile is selected -/// by an 32b GPR. +/// by a 32b GPR. /// /// \headerfile <x86intrin.h> /// @@ -128,7 +128,7 @@ /// Moves a row from a tile register to a zmm destination register, converting /// the fp32 source elements to fp16. It places the resulting fp16 elements /// in the high 16 bits within each dword. The row of the tile is selected -/// by an 32b GPR. +/// by a 32b GPR. /// /// \headerfile <x86intrin.h> /// @@ -164,7 +164,7 @@ /// Moves a row from a tile register to a zmm destination register, converting /// the fp32 source elements to fp16. It places the resulting fp16 elements /// in the low 16 bits within each dword. The row of the tile is selected -/// by an 32b GPR. +/// by a 32b GPR. /// /// \headerfile <x86intrin.h> /// @@ -275,7 +275,7 @@ static __inline__ __m512i __DEFAULT_FN_ATTRS_AVX512 _tile_movrow_internal( /// \param src0 /// The 1st source tile. Max size is 1024 Bytes. /// \param src1 -/// The 1st source r32. Size is 4 Bytes. +/// The 2nd source r32. Size is 4 Bytes. /// \returns /// The destination v16f32 data. Size is 64 Bytes. __DEFAULT_FN_ATTRS_AVX512 @@ -294,7 +294,7 @@ static __m512 __tile_cvtrowd2ps(__tile1024i src0, unsigned src1) { /// \param src0 /// The 1st source tile. Max size is 1024 Bytes. /// \param src1 -/// The 1st source r32. Size is 4 Bytes. +/// The 2nd source r32. Size is 4 Bytes. /// \returns /// The destination v32bf16 data. Size is 64 Bytes. __DEFAULT_FN_ATTRS_AVX512 @@ -313,7 +313,7 @@ static __m512bh __tile_cvtrowps2pbf16h(__tile1024i src0, unsigned src1) { /// \param src0 /// The 1st source tile. Max size is 1024 Bytes. /// \param src1 -/// The 1st source r32. Size is 4 Bytes. +/// The 2nd source r32. Size is 4 Bytes. /// \returns /// The destination v32bf16 data. Size is 64 Bytes. __DEFAULT_FN_ATTRS_AVX512 @@ -332,7 +332,7 @@ static __m512bh __tile_cvtrowps2pbf16l(__tile1024i src0, unsigned src1) { /// \param src0 /// The 1st source tile. Max size is 1024 Bytes. /// \param src1 -/// The 1st source r32. Size is 4 Bytes. +/// The 2nd source r32. Size is 4 Bytes. /// \returns /// The destination v32fp16 data. Size is 64 Bytes. __DEFAULT_FN_ATTRS_AVX512 @@ -351,7 +351,7 @@ static __m512h __tile_cvtrowps2phh(__tile1024i src0, unsigned src1) { /// \param src0 /// The 1st source tile. Max size is 1024 Bytes. /// \param src1 -/// The 1st source r32. Size is 4 Bytes. +/// The 2nd source r32. Size is 4 Bytes. /// \returns /// The destination v32fp16 data. Size is 64 Bytes. __DEFAULT_FN_ATTRS_AVX512 @@ -369,7 +369,7 @@ static __m512h __tile_cvtrowps2phl(__tile1024i src0, unsigned src1) { /// \param src0 /// The 1st source tile. Max size is 1024 Bytes. /// \param src1 -/// The 1st source r32. Size is 4 Bytes. +/// The 2nd source r32. Size is 4 Bytes. /// \returns /// The destination v16i32 data. Size is 64 Bytes. __DEFAULT_FN_ATTRS_AVX512 diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index b878516d279df0..52519f49e7014d 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -619,7 +619,7 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, Opc = X86::TILEMOVROWrri; break; default: - llvm_unreachable("Impossible Opcode!"); + llvm_unreachable("Unexpected Opcode"); } MI.setDesc(TII->get(Opc)); return true; @@ -770,7 +770,7 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, case X86::PTDPBUUDV: Opc = X86::TDPBUUD; break; case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS; break; case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS; break; - default: llvm_unreachable("Impossible Opcode!"); + default: llvm_unreachable("Unexpected Opcode"); } MI.setDesc(TII->get(Opc)); MI.tieOperands(0, 1); >From 1b93a9a559a94874ae39d66998a319d957b01945 Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" <phoebe.w...@intel.com> Date: Thu, 7 Nov 2024 15:56:30 +0800 Subject: [PATCH 3/6] Add avx10.2-512 predicate --- clang/include/clang/Basic/BuiltinsX86_64.def | 12 ++++++------ llvm/lib/Target/X86/X86.td | 2 +- llvm/lib/Target/X86/X86ExpandPseudo.cpp | 3 ++- llvm/lib/Target/X86/X86InstrAMX.td | 4 ++-- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def index d2128b8e913832..29f86e4f84b679 100644 --- a/clang/include/clang/Basic/BuiltinsX86_64.def +++ b/clang/include/clang/Basic/BuiltinsX86_64.def @@ -165,12 +165,12 @@ TARGET_BUILTIN(__builtin_ia32_t2rpntlvwz1, "vIUcvC*z", "n", "amx-transpose") TARGET_BUILTIN(__builtin_ia32_t2rpntlvwz1t1, "vIUcvC*z", "n","amx-transpose") TARGET_BUILTIN(__builtin_ia32_ttransposed, "vIUcIUc", "n", "amx-transpose") -TARGET_BUILTIN(__builtin_ia32_tcvtrowd2ps, "V16fIUcUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16h, "V32yIUcUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16l, "V32yIUcUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phh, "V32xIUcUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phl, "V32xIUcUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tilemovrow, "V16iIUcUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowd2ps, "V16fIUcUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16h, "V32yIUcUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16l, "V32yIUcUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phh, "V32xIUcUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phl, "V32xIUcUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tilemovrow, "V16iIUcUi", "n", "amx-avx512,avx10.2-512") TARGET_BUILTIN(__builtin_ia32_prefetchi, "vvC*Ui", "nc", "prefetchi") TARGET_BUILTIN(__builtin_ia32_cmpccxadd32, "Siv*SiSiIi", "n", "cmpccxadd") diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td index 43129d4bcf1d10..59780ba5b99fcf 100644 --- a/llvm/lib/Target/X86/X86.td +++ b/llvm/lib/Target/X86/X86.td @@ -279,7 +279,7 @@ def FeatureAMXTRANSPOSE : SubtargetFeature<"amx-transpose", "HasAMXTRANSPOSE", " def FeatureAMXAVX512 : SubtargetFeature<"amx-avx512", "HasAMXAVX512", "true", "Support AMX-AVX512 instructions", - [FeatureAMXTILE, FeatureAVX512]>; + [FeatureAMXTILE]>; def FeatureCMPCCXADD : SubtargetFeature<"cmpccxadd", "HasCMPCCXADD", "true", "Support CMPCCXADD instructions">; def FeatureRAOINT : SubtargetFeature<"raoint", "HasRAOINT", "true", diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp index 52519f49e7014d..9511a82f0e97d2 100644 --- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp +++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp @@ -770,7 +770,8 @@ bool X86ExpandPseudo::expandMI(MachineBasicBlock &MBB, case X86::PTDPBUUDV: Opc = X86::TDPBUUD; break; case X86::PTDPBF16PSV: Opc = X86::TDPBF16PS; break; case X86::PTDPFP16PSV: Opc = X86::TDPFP16PS; break; - default: llvm_unreachable("Unexpected Opcode"); + default: + llvm_unreachable("Unexpected Opcode"); } MI.setDesc(TII->get(Opc)); MI.tieOperands(0, 1); diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td index 512cdcb27926bc..16b3be9c396f8d 100644 --- a/llvm/lib/Target/X86/X86InstrAMX.td +++ b/llvm/lib/Target/X86/X86InstrAMX.td @@ -479,7 +479,7 @@ defm TCVTROWPS2PBF16H : AMXAVX512_BASE<0x6d, 0x07, "tcvtrowps2pbf16h", XD, XD>; defm TCVTROWPS2PBF16L : AMXAVX512_BASE<0x6d, 0x77, "tcvtrowps2pbf16l", XS, XS>; multiclass m_tilemovrow { - let Predicates = [HasAMXAVX512, In64BitMode] in { + let Predicates = [HasAMXAVX512, HasAVX10_2_512, In64BitMode] in { let SchedRW = [WriteSystem] in { def rri : Ii8<0x7, MRMSrcReg, (outs VR512:$dst), (ins TILE:$src1, u8imm:$src2), @@ -495,7 +495,7 @@ multiclass m_tilemovrow { defm TILEMOVROW : m_tilemovrow; -let Predicates = [HasAMXAVX512, In64BitMode] in { +let Predicates = [HasAMXAVX512, HasAVX10_2_512, In64BitMode] in { let SchedRW = [WriteSystem] in { let usesCustomInserter = 1 in { def PTILEMOVROWrri : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2), >From 656080631510dd641f4c1c4380f55f6ddad28283 Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" <phoebe.w...@intel.com> Date: Fri, 8 Nov 2024 10:59:58 +0800 Subject: [PATCH 4/6] Add missing parts --- clang/include/clang/Basic/BuiltinsX86_64.def | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/clang/include/clang/Basic/BuiltinsX86_64.def b/clang/include/clang/Basic/BuiltinsX86_64.def index 29f86e4f84b679..9f7462b1e0d962 100644 --- a/clang/include/clang/Basic/BuiltinsX86_64.def +++ b/clang/include/clang/Basic/BuiltinsX86_64.def @@ -133,12 +133,12 @@ TARGET_BUILTIN(__builtin_ia32_t2rpntlvwz0t1_internal, "vUsUsUsV256i*V256i*vC*z", TARGET_BUILTIN(__builtin_ia32_t2rpntlvwz1_internal, "vUsUsUsV256i*V256i*vC*z", "n", "amx-transpose") TARGET_BUILTIN(__builtin_ia32_t2rpntlvwz1t1_internal, "vUsUsUsV256i*V256i*vC*z", "n", "amx-transpose") TARGET_BUILTIN(__builtin_ia32_ttransposed_internal, "V256iUsUsV256i", "n", "amx-transpose") -TARGET_BUILTIN(__builtin_ia32_tcvtrowd2ps_internal, "V16fUsUsV256iUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16h_internal, "V32yUsUsV256iUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16l_internal, "V32yUsUsV256iUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phh_internal, "V32xUsUsV256iUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phl_internal, "V32xUsUsV256iUi", "n", "amx-avx512") -TARGET_BUILTIN(__builtin_ia32_tilemovrow_internal, "V16iUsUsV256iUi", "n", "amx-avx512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowd2ps_internal, "V16fUsUsV256iUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16h_internal, "V32yUsUsV256iUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2pbf16l_internal, "V32yUsUsV256iUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phh_internal, "V32xUsUsV256iUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tcvtrowps2phl_internal, "V32xUsUsV256iUi", "n", "amx-avx512,avx10.2-512") +TARGET_BUILTIN(__builtin_ia32_tilemovrow_internal, "V16iUsUsV256iUi", "n", "amx-avx512,avx10.2-512") // AMX TARGET_BUILTIN(__builtin_ia32_tile_loadconfig, "vvC*", "n", "amx-tile") TARGET_BUILTIN(__builtin_ia32_tile_storeconfig, "vvC*", "n", "amx-tile") >From 6a78b18e65740cd866b66cb57c1505833d6cc274 Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" <phoebe.w...@intel.com> Date: Fri, 8 Nov 2024 11:03:23 +0800 Subject: [PATCH 5/6] Add avx10.2-512 --- clang/lib/Headers/amxavx512intrin.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/clang/lib/Headers/amxavx512intrin.h b/clang/lib/Headers/amxavx512intrin.h index 9bfa868cf447a0..945edea543e706 100644 --- a/clang/lib/Headers/amxavx512intrin.h +++ b/clang/lib/Headers/amxavx512intrin.h @@ -15,7 +15,8 @@ #ifdef __x86_64__ #define __DEFAULT_FN_ATTRS_AVX512 \ - __attribute__((__always_inline__, __nodebug__, __target__("amx-avx512"))) + __attribute__((__always_inline__, __nodebug__, \ + __target__("amx-avx512,avx10.2-512"))) /// Moves a row from a tile register to a zmm destination register, converting /// the int32 source elements to fp32. The row of the tile is selected by a >From ffcb6a7452a8ef41e62e47ce71747559eca1c83a Mon Sep 17 00:00:00 2001 From: "Wang, Phoebe" <phoebe.w...@intel.com> Date: Fri, 8 Nov 2024 15:20:18 +0800 Subject: [PATCH 6/6] Add HasAVX10_2_512 --- llvm/lib/Target/X86/X86InstrAMX.td | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Target/X86/X86InstrAMX.td b/llvm/lib/Target/X86/X86InstrAMX.td index 16b3be9c396f8d..b954c977f8c6c9 100644 --- a/llvm/lib/Target/X86/X86InstrAMX.td +++ b/llvm/lib/Target/X86/X86InstrAMX.td @@ -371,7 +371,7 @@ let Predicates = [HasAMXTRANSPOSE, In64BitMode] in { } // HasAMXTILE, HasAMXTRANSPOSE multiclass m_tcvtrowd2ps { - let Predicates = [HasAMXAVX512, In64BitMode] in { + let Predicates = [HasAMXAVX512, HasAVX10_2_512, In64BitMode] in { let SchedRW = [WriteSystem] in { def rri : Ii8<0x7, MRMSrcReg, (outs VR512:$dst), (ins TILE:$src1, i32u8imm:$src2), @@ -382,12 +382,12 @@ multiclass m_tcvtrowd2ps { "tcvtrowd2ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, T8,XS, EVEX, VVVV, EVEX_V512; } - } // HasAMXAVX512, In64BitMode + } // HasAMXAVX512, HasAVX10_2_512, In64BitMode } defm TCVTROWD2PS : m_tcvtrowd2ps; -let Predicates = [HasAMXAVX512, In64BitMode] in { +let Predicates = [HasAMXAVX512, HasAVX10_2_512, In64BitMode] in { let SchedRW = [WriteSystem] in { let usesCustomInserter = 1 in { def PTCVTROWD2PSrri : PseudoI<(outs VR512:$dst), (ins u8imm:$src1, i32u8imm:$src2), @@ -451,7 +451,7 @@ let Predicates = [HasAMXAVX512, In64BitMode] in { multiclass AMXAVX512_BASE<bits<8> Opcode1, bits<8> Opcode2, string Opstr, Prefix P1, Prefix P2> { - let Predicates = [HasAMXAVX512, In64BitMode], SchedRW = [WriteSystem] in { + let Predicates = [HasAMXAVX512, HasAVX10_2_512, In64BitMode], SchedRW = [WriteSystem] in { let OpPrefix = P1 in def rre : I<Opcode1, MRMSrcReg4VOp3, (outs VR512:$dst), (ins TILE:$src1, GR32:$src2), @@ -490,7 +490,7 @@ multiclass m_tilemovrow { "tilemovrow\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, T8,PD, EVEX, VVVV, EVEX_V512; } - } // HasAMXAVX512, In64BitMode + } // HasAMXAVX512, HasAVX10_2_512, In64BitMode } defm TILEMOVROW : m_tilemovrow; _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits