================ @@ -0,0 +1,766 @@ +/*===-------------- acev1intrin.h - ACEV1 intrinsics -*- C/C++ -*------------=== + * + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + * + *===------------------------------------------------------------------------=== + */ + +#ifndef __IMMINTRIN_H +#error "Never use <acev1intrin.h> directly; include <immintrin.h> instead." +#endif /* __IMMINTRIN_H */ + +#ifndef __ACEV1INTRIN_H +#define __ACEV1INTRIN_H +#ifdef __x86_64__ + +/* Define the default attributes for the functions in this file. */ +#define __DEFAULT_FN_ATTRS_ACE \ + __attribute__((__always_inline__, __nodebug__, __target__("acev1"))) + +/// Load tile configuration from a 64-byte memory location. For ACE +/// (Palette 2), the palette_id byte must be 2. Unlike AMX (Palette 1), +/// ACE tiles have fixed dimensions of 16 rows × 64 bytes, so per-tile +/// row/column configuration bytes are ignored. If palette_id is zero, +/// tiles return to init state and are zeroed. Invalid configurations +/// result in #GP fault. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> LDTILECFG </c> instruction. +/// +/// \param __config +/// A pointer to 64-byte tile configuration (use __ace_tile_config). +static __inline__ void __DEFAULT_FN_ATTRS_ACE +_tile_ace_loadconfig(const void *__config) { + __builtin_ia32_tile_loadconfig(__config); +} + +/// Store the current tile configuration to a 64-byte memory location. +/// For ACE (Palette 2), the stored palette_id will be 2 and per-tile +/// configuration bytes reflect the fixed 16×64 dimensions. If tiles +/// are not configured, all zeroes are stored. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> STTILECFG </c> instruction. +/// +/// \param __config +/// A pointer to 64-byte tile configuration buffer. +static __inline__ void __DEFAULT_FN_ATTRS_ACE +_tile_ace_storeconfig(void *__config) { + __builtin_ia32_tile_storeconfig(__config); +} + +/// Release the tile configuration to return to init state, releasing +/// all tile storage. After this, tiles must be reconfigured with +/// _tile_ace_loadconfig before use. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILERELEASE </c> instruction. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _tile_ace_release(void) { + __builtin_ia32_tilerelease(); +} + +/// Zero the ACE tile specified by "tile". Sets all 1024 bytes +/// (16 rows × 64 bytes) of the tile register to zero. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEZERO </c> instruction. +/// +/// \param tile +/// Destination tile register ID (0-7). +#define _tile_ace_zero(tile) __builtin_ia32_tilezero((tile)) + +/// Move a 64-byte ZMM vector to a tile column. The 16 doublewords from +/// the ZMM are written as a vertical column in the tile at the specified +/// index. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEMOVCOL </c> instruction. +/// +/// \param dst +/// Destination tile register ID (0-7). +/// \param src +/// Source ZMM register ID (0-31). +/// \param idx +/// Column index (0-15). Immediate or register form selected automatically. +#define _tile_setcol(dst, src, idx) \ + __builtin_ia32_tilesetcol((dst), (src), (idx)) + +/// Move a 64-byte ZMM vector to a tile row. The ZMM contents are written +/// as a horizontal row in the tile at the specified index. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TILEMOVROW </c> instruction. +/// +/// \param dst +/// Destination tile register ID (0-7). +/// \param src +/// Source ZMM register ID (0-31). +/// \param idx +/// Row index (0-15). Immediate or register form selected automatically. +#define _tile_setrow(dst, src, idx) \ + __builtin_ia32_tilesetrow((dst), (src), (idx)) + +/// Initialize the Block Scale Register (BSR) to zero. The BSR holds +/// 32 scaling factors used by mixed-precision outer product instructions +/// (TOP4MX* variants). Must be called before using BSR-scaled operations. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRINIT </c> instruction. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_init(void) { + __builtin_ia32_bsrinit(); +} + +/// Load the full BSR (32 scale factors) from two ZMM registers. The low +/// half (16 factors) comes from __src1, high half from __src2. Each +/// doubleword contains one scale factor. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVF </c> instruction. +/// +/// \param __src1 +/// ZMM with scale factors for BSR low half (factors 0-15). +/// \param __src2 +/// ZMM with scale factors for BSR high half (factors 16-31). +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movf(__m512i __src1, + __m512i __src2) { + __builtin_ia32_bsrmovf((__v16si)__src1, (__v16si)__src2); +} + +/// Load the high half of BSR (scale factors 16-31) from a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction. +/// +/// \param __src +/// ZMM with 16 scale factors to write to BSR high half. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movh_set(__m512i __src) { + __builtin_ia32_bsrmovh_set((__v16si)__src); +} + +/// Read the high half of BSR (scale factors 16-31) to a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVH </c> instruction. +/// +/// \returns +/// ZMM containing 16 scale factors from BSR high half. +static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movh_get(void) { + return (__m512i)__builtin_ia32_bsrmovh_get(); +} + +/// Load the low half of BSR (scale factors 0-15) from a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction. +/// +/// \param __src +/// ZMM with 16 scale factors to write to BSR low half. +static __inline__ void __DEFAULT_FN_ATTRS_ACE _bsr0_movl_set(__m512i __src) { + __builtin_ia32_bsrmovl_set((__v16si)__src); +} + +/// Read the low half of BSR (scale factors 0-15) to a ZMM register. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> BSRMOVL </c> instruction. +/// +/// \returns +/// ZMM containing 16 scale factors from BSR low half. +static __inline__ __m512i __DEFAULT_FN_ATTRS_ACE _bsr0_movl_get(void) { + return (__m512i)__builtin_ia32_bsrmovl_get(); +} + +/// Compute 2-way outer product of BF16 pairs, accumulating to FP32. +/// Each BF16 pair from src1 and src2 produces two FP32 products that +/// are accumulated into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP2BF16PS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing 32 BF16 values. +/// \param src2 +/// Second source ZMM register ID (0-31) containing 32 BF16 values. +#define _tile_top2bf16ps(dst, src1, src2) \ + __builtin_ia32_top2bf16ps((dst), (src1), (src2)) + +/// Compute 4-way outer product of unsigned×unsigned bytes to INT32. +/// Each group of 4 unsigned byte pairs from src1 and src2 produces +/// 4 products accumulated into the destination tile as 32-bit integers. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4BUUD </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing 64 unsigned bytes. +/// \param src2 +/// Second source ZMM register ID (0-31) containing 64 unsigned bytes. +#define _tile_top4buud(dst, src1, src2) \ + __builtin_ia32_top4buud((dst), (src1), (src2)) + +/// Compute 4-way outer product of unsigned×signed bytes to INT32. +/// Each group of 4 byte pairs (unsigned from src1, signed from src2) +/// produces 4 products accumulated into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4BUSD </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing 64 unsigned bytes. +/// \param src2 +/// Second source ZMM register ID (0-31) containing 64 signed bytes. +#define _tile_top4busd(dst, src1, src2) \ + __builtin_ia32_top4busd((dst), (src1), (src2)) + +/// Compute 4-way outer product of signed×signed bytes to INT32. +/// Each group of 4 signed byte pairs produces 4 products accumulated +/// into the destination tile as 32-bit integers. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4BSSD </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing 64 signed bytes. +/// \param src2 +/// Second source ZMM register ID (0-31) containing 64 signed bytes. +#define _tile_top4bssd(dst, src1, src2) \ + __builtin_ia32_top4bssd((dst), (src1), (src2)) + +/// Compute 4-way outer product of signed×unsigned bytes to INT32. +/// Each group of 4 byte pairs (signed from src1, unsigned from src2) +/// produces 4 products accumulated into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4BSUD </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing 64 signed bytes. +/// \param src2 +/// Second source ZMM register ID (0-31) containing 64 unsigned bytes. +#define _tile_top4bsud(dst, src1, src2) \ + __builtin_ia32_top4bsud((dst), (src1), (src2)) + +/// Compute 4-way mixed precision outer product with HF8 (E4M3) format. +/// Multiplies HF8 values from src1 with BF8 values from src2, converting +/// to FP32 and accumulating into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4MXHF8PS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing HF8 values. +/// \param src2 +/// Second source ZMM register ID (0-31) containing BF8 values. +/// \param imm +/// 8-bit control immediate for scaling/rounding options. +#define _tile_top4mxhf8ps(dst, src1, src2, imm) \ + __builtin_ia32_top4mxhf8ps((dst), (src1), (src2), (imm)) + +/// Compute 4-way mixed precision outer product with BF8/HF8 format. +/// Multiplies BF8 values from src1 with HF8 (E4M3) values from src2, +/// converting to FP32 and accumulating into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4MXBHF8PS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing BF8 values. +/// \param src2 +/// Second source ZMM register ID (0-31) containing HF8 values. +/// \param imm +/// 8-bit control immediate for scaling/rounding options. +#define _tile_top4mxbhf8ps(dst, src1, src2, imm) \ + __builtin_ia32_top4mxbhf8ps((dst), (src1), (src2), (imm)) + +/// Compute 4-way mixed precision outer product with HF8/BF8 format. +/// Multiplies HF8 (E4M3) values from src1 with BF8 values from src2, +/// converting to FP32 and accumulating into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4MXHBF8PS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing HF8 values. +/// \param src2 +/// Second source ZMM register ID (0-31) containing BF8 values. +/// \param imm +/// 8-bit control immediate for scaling/rounding options. +#define _tile_top4mxhbf8ps(dst, src1, src2, imm) \ + __builtin_ia32_top4mxhbf8ps((dst), (src1), (src2), (imm)) + +/// Compute 4-way mixed precision outer product with BF8 (E5M2) format. +/// Multiplies BF8 values from both sources, converting to FP32 and +/// accumulating into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4MXBF8PS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing BF8 values. +/// \param src2 +/// Second source ZMM register ID (0-31) containing BF8 values. +/// \param imm +/// 8-bit control immediate for scaling/rounding options. +#define _tile_top4mxbf8ps(dst, src1, src2, imm) \ + __builtin_ia32_top4mxbf8ps((dst), (src1), (src2), (imm)) + +/// Compute 4-way mixed precision outer product of signed INT8 with BSR +/// scaling. Multiplies signed bytes, applies scale factors from the BSR, +/// and accumulates FP32 results into the destination tile. +/// +/// \headerfile <immintrin.h> +/// +/// This intrinsic corresponds to the <c> TOP4MXBSSPS </c> instruction. +/// +/// \param dst +/// Destination/accumulator tile register ID (0-7). +/// \param src1 +/// First source ZMM register ID (0-31) containing signed bytes. +/// \param src2 +/// Second source ZMM register ID (0-31) containing signed bytes. +/// \param imm +/// 8-bit immediate selecting BSR scale factors to apply. +#define _tile_top4mxbssps(dst, src1, src2, imm) \ + __builtin_ia32_top4mxbssps((dst), (src1), (src2), (imm)) + +/// ACE tile type with fixed dimensions (16 rows × 64 bytes = 1024 bytes). +/// ACE Palette 2 uses fixed tile dimensions, unlike AMX Palette 1. +typedef int __acetile __attribute__((__vector_size__(1024), __aligned__(64))); + +/// ACE Palette 2 tile configuration structure (64 bytes). +/// For ACE, only byte 0 (palette_id = 2) is significant; bytes 1-63 +/// must be zero. Unlike AMX Palette 1, tile dimensions are fixed. +typedef struct __attribute__((__packed__, __aligned__(64))) { + unsigned char palette_id; + unsigned char reserved[63]; +} __ace_tile_config; + +/// Initialize an ACE tile configuration structure for Palette 2. +/// Sets palette_id to 2 and clears all reserved bytes to zero. +/// Use with _tile_ace_loadconfig to configure tiles for ACE operations. +static __inline__ void __DEFAULT_FN_ATTRS_ACE +__ace_init_config(__ace_tile_config *cfg) { + for (int i = 0; i < 64; i++) ---------------- mahesh-attarde wrote:
How about using `__builtin_memset(cfg, 0, 64);` ? Usual expectation from intrinsic is to map directly to hardware instructions, With loop we are at mercy of Opt level to optimize that to memset. Clang builtin is always available. https://github.com/llvm/llvm-project/pull/208408 _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
