https://github.com/kieroxide updated https://github.com/llvm/llvm-project/pull/238416
>From bbd6dc712eba5731c71f5ded998f3124105fd5ff Mon Sep 17 00:00:00 2001 From: Kieran Bailey <[email protected]> Date: Tue, 3 Nov 2026 16:48:34 +0000 Subject: [PATCH] [AArch64] Implement STNP/LDNP intrinsics Implement Non-temporal load/store STNP/LDNP intrinsics specified in the ACLE --- clang/include/clang/Basic/arm_neon.td | 6 + clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 33 ++ clang/test/CodeGen/AArch64/vldn_intrinsics.c | 343 +++++++++++++++++++ clang/test/CodeGen/AArch64/vstn_intrinsics.c | 282 +++++++++++++++ llvm/include/llvm/IR/IntrinsicsAArch64.td | 10 + llvm/lib/Target/AArch64/AArch64InstrInfo.td | 26 ++ llvm/test/CodeGen/AArch64/vldn_intrinsics.ll | 143 ++++++++ llvm/test/CodeGen/AArch64/vstn_intrinsics.ll | 159 +++++++++ 8 files changed, 1002 insertions(+) create mode 100644 clang/test/CodeGen/AArch64/vldn_intrinsics.c create mode 100644 clang/test/CodeGen/AArch64/vstn_intrinsics.c create mode 100644 llvm/test/CodeGen/AArch64/vldn_intrinsics.ll create mode 100644 llvm/test/CodeGen/AArch64/vstn_intrinsics.ll diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td index b74d3805e950a9..3597a37d7e6900 100644 --- a/clang/include/clang/Basic/arm_neon.td +++ b/clang/include/clang/Basic/arm_neon.td @@ -2245,3 +2245,9 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp8,neon" in { def FSCALE_V128 : WInst<"vscale", "..(.S)", "QdQfQh">; def FSCALE_V64 : WInst<"vscale", "(.q)(.q)(.qS)", "fh">; } + +let ArchGuard = "defined(__aarch64__)", TargetGuard = "fp-armv8" in { + // Non-temporal pair load/stores + def VSTNQ_X2 : SInst<"vstnq_x2", "v(*)(2!)", "HcHsHiHlHUcHUsHUiHUlHhHbHfHdHm">; + def VLDNQ_X2 : SInst<"vldnq_x2", "2(c*!)", "HcHsHiHlHUcHUsHUiHUlHhHbHfHdHm">; +} diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index 4a7bdba6d917ee..fd537a4281e3e9 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -6893,6 +6893,39 @@ Value *CodeGenFunction::EmitAArch64BuiltinExpr(unsigned BuiltinID, return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_st4lane, Tys), Ops, ""); } + case NEON::BI__builtin_neon_vstnq_s8_x2: + case NEON::BI__builtin_neon_vstnq_s16_x2: + case NEON::BI__builtin_neon_vstnq_s32_x2: + case NEON::BI__builtin_neon_vstnq_s64_x2: + case NEON::BI__builtin_neon_vstnq_u8_x2: + case NEON::BI__builtin_neon_vstnq_u16_x2: + case NEON::BI__builtin_neon_vstnq_u32_x2: + case NEON::BI__builtin_neon_vstnq_u64_x2: + case NEON::BI__builtin_neon_vstnq_mf8_x2: + case NEON::BI__builtin_neon_vstnq_f16_x2: + case NEON::BI__builtin_neon_vstnq_bf16_x2: + case NEON::BI__builtin_neon_vstnq_f32_x2: + case NEON::BI__builtin_neon_vstnq_f64_x2: + std::rotate(Ops.begin(), Ops.begin() + 1, Ops.end()); + return EmitNeonCall(CGM.getIntrinsic(Intrinsic::aarch64_neon_vstnq_x2, Ty), + Ops, ""); + case NEON::BI__builtin_neon_vldnq_s8_x2: + case NEON::BI__builtin_neon_vldnq_s16_x2: + case NEON::BI__builtin_neon_vldnq_s32_x2: + case NEON::BI__builtin_neon_vldnq_s64_x2: + case NEON::BI__builtin_neon_vldnq_u8_x2: + case NEON::BI__builtin_neon_vldnq_u16_x2: + case NEON::BI__builtin_neon_vldnq_u32_x2: + case NEON::BI__builtin_neon_vldnq_u64_x2: + case NEON::BI__builtin_neon_vldnq_mf8_x2: + case NEON::BI__builtin_neon_vldnq_f16_x2: + case NEON::BI__builtin_neon_vldnq_bf16_x2: + case NEON::BI__builtin_neon_vldnq_f32_x2: + case NEON::BI__builtin_neon_vldnq_f64_x2: + Ops[1] = Builder.CreateCall( + CGM.getIntrinsic(Intrinsic::aarch64_neon_vldnq_x2, VTy), Ops[1], + "vldnq_x2"); + return Builder.CreateDefaultAlignedStore(Ops[1], Ops[0]); case NEON::BI__builtin_neon_vtrn_v: case NEON::BI__builtin_neon_vtrnq_v: { Ops[1] = Builder.CreateBitCast(Ops[1], Ty); diff --git a/clang/test/CodeGen/AArch64/vldn_intrinsics.c b/clang/test/CodeGen/AArch64/vldn_intrinsics.c new file mode 100644 index 00000000000000..959cc883e79736 --- /dev/null +++ b/clang/test/CodeGen/AArch64/vldn_intrinsics.c @@ -0,0 +1,343 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 4 +#include <arm_neon.h> + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +fp-armv8 -O2 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK-LE +// RUN: %clang_cc1 -triple aarch64_be-none-linux-gnu -target-feature +fp-armv8 -O2 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK-BE + +// CHECK-LE-LABEL: define dso_local %struct.int8x16x2_t @test_vldnq_s8_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T:%.*]] poison, <16 x i8> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.int8x16x2_t @test_vldnq_s8_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VLDNQ_X2_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <16 x i8> [[VLDNQ_X2_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T:%.*]] poison, <16 x i8> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_INT8X16X2_T]] [[DOTFCA_0_1_INSERT]] +// +int8x16x2_t test_vldnq_s8_x2(const int8_t* ptr){ + return vldnq_s8_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.int16x8x2_t @test_vldnq_s16_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldnq.x2.v8i16(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T:%.*]] poison, <8 x i16> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.int16x8x2_t @test_vldnq_s16_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldnq.x2.v8i16(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[VLDNQ_X2_FCA_0_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x i16> [[VLDNQ_X2_FCA_1_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T:%.*]] poison, <8 x i16> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_INT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +int16x8x2_t test_vldnq_s16_x2(const int16_t* ptr){ + return vldnq_s16_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.int32x4x2_t @test_vldnq_s32_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldnq.x2.v4i32(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T:%.*]] poison, <4 x i32> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.int32x4x2_t @test_vldnq_s32_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldnq.x2.v4i32(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VLDNQ_X2_FCA_0_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <4 x i32> [[VLDNQ_X2_FCA_1_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T:%.*]] poison, <4 x i32> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_INT32X4X2_T]] [[DOTFCA_0_1_INSERT]] +// +int32x4x2_t test_vldnq_s32_x2(const int32_t* ptr){ + return vldnq_s32_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.int64x2x2_t @test_vldnq_s64_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldnq.x2.v2i64(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T:%.*]] poison, <2 x i64> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.int64x2x2_t @test_vldnq_s64_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldnq.x2.v2i64(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[VLDNQ_X2_FCA_0_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <2 x i64> [[VLDNQ_X2_FCA_1_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T:%.*]] poison, <2 x i64> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_INT64X2X2_T]] [[DOTFCA_0_1_INSERT]] +// +int64x2x2_t test_vldnq_s64_x2(const int64_t* ptr){ + return vldnq_s64_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.uint8x16x2_t @test_vldnq_u8_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T:%.*]] poison, <16 x i8> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.uint8x16x2_t @test_vldnq_u8_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VLDNQ_X2_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <16 x i8> [[VLDNQ_X2_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T:%.*]] poison, <16 x i8> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_UINT8X16X2_T]] [[DOTFCA_0_1_INSERT]] +// +uint8x16x2_t test_vldnq_u8_x2(const uint8_t* ptr){ + return vldnq_u8_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.uint16x8x2_t @test_vldnq_u16_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldnq.x2.v8i16(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T:%.*]] poison, <8 x i16> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.uint16x8x2_t @test_vldnq_u16_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldnq.x2.v8i16(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x i16>, <8 x i16> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[VLDNQ_X2_FCA_0_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x i16> [[VLDNQ_X2_FCA_1_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T:%.*]] poison, <8 x i16> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x i16> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_UINT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +uint16x8x2_t test_vldnq_u16_x2(const uint16_t* ptr){ + return vldnq_u16_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.uint32x4x2_t @test_vldnq_u32_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldnq.x2.v4i32(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T:%.*]] poison, <4 x i32> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.uint32x4x2_t @test_vldnq_u32_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldnq.x2.v4i32(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x i32>, <4 x i32> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VLDNQ_X2_FCA_0_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <4 x i32> [[VLDNQ_X2_FCA_1_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T:%.*]] poison, <4 x i32> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x i32> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_UINT32X4X2_T]] [[DOTFCA_0_1_INSERT]] +// +uint32x4x2_t test_vldnq_u32_x2(const uint32_t* ptr){ + return vldnq_u32_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.uint64x2x2_t @test_vldnq_u64_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldnq.x2.v2i64(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T:%.*]] poison, <2 x i64> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.uint64x2x2_t @test_vldnq_u64_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldnq.x2.v2i64(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x i64>, <2 x i64> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[VLDNQ_X2_FCA_0_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <2 x i64> [[VLDNQ_X2_FCA_1_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T:%.*]] poison, <2 x i64> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x i64> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_UINT64X2X2_T]] [[DOTFCA_0_1_INSERT]] +// +uint64x2x2_t test_vldnq_u64_x2(const uint64_t* ptr){ + return vldnq_u64_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.float16x8x2_t @test_vldnq_f16_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x half>, <8 x half> } @llvm.aarch64.neon.vldnq.x2.v8f16(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T:%.*]] poison, <8 x half> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x half> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.float16x8x2_t @test_vldnq_f16_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x half>, <8 x half> } @llvm.aarch64.neon.vldnq.x2.v8f16(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x half>, <8 x half> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x half> [[VLDNQ_X2_FCA_0_EXTRACT]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x half> [[VLDNQ_X2_FCA_1_EXTRACT]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T:%.*]] poison, <8 x half> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x half> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_FLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +float16x8x2_t test_vldnq_f16_x2(const float16_t* ptr){ + return vldnq_f16_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.bfloat16x8x2_t @test_vldnq_bf16_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.vldnq.x2.v8bf16(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.bfloat16x8x2_t @test_vldnq_bf16_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.vldnq.x2.v8bf16(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <8 x bfloat>, <8 x bfloat> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x bfloat> [[VLDNQ_X2_FCA_0_EXTRACT]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <8 x bfloat> [[VLDNQ_X2_FCA_1_EXTRACT]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T:%.*]] poison, <8 x bfloat> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_0_INSERT]], <8 x bfloat> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_BFLOAT16X8X2_T]] [[DOTFCA_0_1_INSERT]] +// +bfloat16x8x2_t test_vldnq_bf16_x2(const bfloat16_t* ptr){ + return vldnq_bf16_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.float32x4x2_t @test_vldnq_f32_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.aarch64.neon.vldnq.x2.v4f32(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T:%.*]] poison, <4 x float> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x float> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.float32x4x2_t @test_vldnq_f32_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <4 x float>, <4 x float> } @llvm.aarch64.neon.vldnq.x2.v4f32(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <4 x float>, <4 x float> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x float> [[VLDNQ_X2_FCA_0_EXTRACT]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <4 x float> [[VLDNQ_X2_FCA_1_EXTRACT]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T:%.*]] poison, <4 x float> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_0_INSERT]], <4 x float> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_FLOAT32X4X2_T]] [[DOTFCA_0_1_INSERT]] +// +float32x4x2_t test_vldnq_f32_x2(const float32_t* ptr){ + return vldnq_f32_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.float64x2x2_t @test_vldnq_f64_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <2 x double>, <2 x double> } @llvm.aarch64.neon.vldnq.x2.v2f64(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T:%.*]] poison, <2 x double> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x double> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.float64x2x2_t @test_vldnq_f64_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <2 x double>, <2 x double> } @llvm.aarch64.neon.vldnq.x2.v2f64(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <2 x double>, <2 x double> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x double> [[VLDNQ_X2_FCA_0_EXTRACT]], <2 x double> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <2 x double> [[VLDNQ_X2_FCA_1_EXTRACT]], <2 x double> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T:%.*]] poison, <2 x double> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_0_INSERT]], <2 x double> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_FLOAT64X2X2_T]] [[DOTFCA_0_1_INSERT]] +// +float64x2x2_t test_vldnq_f64_x2(const float64_t* ptr){ + return vldnq_f64_x2(ptr); +} + +// CHECK-LE-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vldnq_mf8_x2( +// CHECK-LE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: entry: +// CHECK-LE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr [[PTR]]) +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 0 +// CHECK-LE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 1 +// CHECK-LE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[VLDNQ_X2_FCA_0_EXTRACT]], 0, 0 +// CHECK-LE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[VLDNQ_X2_FCA_1_EXTRACT]], 0, 1 +// CHECK-LE-NEXT: ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]] +// +// CHECK-BE-LABEL: define dso_local %struct.mfloat8x16x2_t @test_vldnq_mf8_x2( +// CHECK-BE-SAME: ptr noundef [[PTR:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: entry: +// CHECK-BE-NEXT: [[VLDNQ_X2:%.*]] = tail call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr [[PTR]]) +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_0_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 0 +// CHECK-BE-NEXT: [[VLDNQ_X2_FCA_1_EXTRACT:%.*]] = extractvalue { <16 x i8>, <16 x i8> } [[VLDNQ_X2]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VLDNQ_X2_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE9:%.*]] = shufflevector <16 x i8> [[VLDNQ_X2_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[DOTFCA_0_0_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T:%.*]] poison, <16 x i8> [[SHUFFLE]], 0, 0 +// CHECK-BE-NEXT: [[DOTFCA_0_1_INSERT:%.*]] = insertvalue [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_0_INSERT]], <16 x i8> [[SHUFFLE9]], 0, 1 +// CHECK-BE-NEXT: ret [[STRUCT_MFLOAT8X16X2_T]] [[DOTFCA_0_1_INSERT]] +// +mfloat8x16x2_t test_vldnq_mf8_x2(const mfloat8_t* ptr){ + return vldnq_mf8_x2(ptr); +} diff --git a/clang/test/CodeGen/AArch64/vstn_intrinsics.c b/clang/test/CodeGen/AArch64/vstn_intrinsics.c new file mode 100644 index 00000000000000..57615f2f9c0184 --- /dev/null +++ b/clang/test/CodeGen/AArch64/vstn_intrinsics.c @@ -0,0 +1,282 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6 +#include <arm_neon.h> + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +fp-armv8 -O2 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK-LE +// RUN: %clang_cc1 -triple aarch64_be-none-linux-gnu -target-feature +fp-armv8 -O2 -emit-llvm -o - %s | FileCheck %s --check-prefixes=CHECK-BE + + +// CHECK-LE-LABEL: define dso_local void @test_vstnq_s8_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <16 x i8>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> [[VAL_COERCE_FCA_0_EXTRACT]], <16 x i8> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_s8_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <16 x i8>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VAL_COERCE_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <16 x i8> [[VAL_COERCE_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> [[SHUFFLE]], <16 x i8> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_s8_x2(int8_t *ptr, int8x16x2_t val){ + vstnq_s8_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_s16_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x i16>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8i16(<8 x i16> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x i16> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_s16_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x i16>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <8 x i16> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8i16(<8 x i16> [[SHUFFLE]], <8 x i16> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_s16_x2(int16_t *ptr, int16x8x2_t val){ + vstnq_s16_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_s32_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <4 x i32>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v4i32(<4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_s32_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <4 x i32>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v4i32(<4 x i32> [[SHUFFLE]], <4 x i32> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_s32_x2(int32_t *ptr, int32x4x2_t val){ + vstnq_s32_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_s64_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <2 x i64>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v2i64(<2 x i64> [[VAL_COERCE_FCA_0_EXTRACT]], <2 x i64> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_s64_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <2 x i64>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[VAL_COERCE_FCA_0_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <2 x i64> [[VAL_COERCE_FCA_1_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v2i64(<2 x i64> [[SHUFFLE]], <2 x i64> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_s64_x2(int64_t *ptr, int64x2x2_t val){ + vstnq_s64_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_u8_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <16 x i8>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> [[VAL_COERCE_FCA_0_EXTRACT]], <16 x i8> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_u8_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <16 x i8>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VAL_COERCE_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <16 x i8> [[VAL_COERCE_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> [[SHUFFLE]], <16 x i8> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_u8_x2(uint8_t *ptr, uint8x16x2_t val){ + vstnq_u8_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_u16_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x i16>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8i16(<8 x i16> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x i16> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_u16_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x i16>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x i16>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <8 x i16> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <8 x i16> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x i16> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8i16(<8 x i16> [[SHUFFLE]], <8 x i16> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_u16_x2(uint16_t *ptr, uint16x8x2_t val){ + vstnq_u16_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_u32_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <4 x i32>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v4i32(<4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_u32_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <4 x i32>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x i32>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <4 x i32> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <4 x i32> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x i32> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v4i32(<4 x i32> [[SHUFFLE]], <4 x i32> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_u32_x2(uint32_t *ptr, uint32x4x2_t val){ + vstnq_u32_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_u64_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <2 x i64>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v2i64(<2 x i64> [[VAL_COERCE_FCA_0_EXTRACT]], <2 x i64> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_u64_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <2 x i64>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x i64>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <2 x i64> [[VAL_COERCE_FCA_0_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <2 x i64> [[VAL_COERCE_FCA_1_EXTRACT]], <2 x i64> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v2i64(<2 x i64> [[SHUFFLE]], <2 x i64> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_u64_x2(uint64_t *ptr, uint64x2x2_t val){ + vstnq_u64_x2(ptr, val); +} + +// CHECK-LE-LABEL: define dso_local void @test_vstnq_mf8_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <16 x i8>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> [[VAL_COERCE_FCA_0_EXTRACT]], <16 x i8> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_mf8_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <16 x i8>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <16 x i8>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[SHUFFLE:%.*]] = shufflevector <16 x i8> [[VAL_COERCE_FCA_0_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[SHUFFLE10:%.*]] = shufflevector <16 x i8> [[VAL_COERCE_FCA_1_EXTRACT]], <16 x i8> poison, <16 x i32> <i32 15, i32 14, i32 13, i32 12, i32 11, i32 10, i32 9, i32 8, i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> [[SHUFFLE]], <16 x i8> [[SHUFFLE10]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_mf8_x2(mfloat8_t *ptr, mfloat8x16x2_t val){ + vstnq_mf8_x2(ptr, val); +} + +// CHECK-LE-LABEL: define dso_local void @test_vstnq_f16_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x half>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x half>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x half>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8f16(<8 x half> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x half> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_f16_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x half>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x half>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x half>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <8 x half> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <8 x half> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x half> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8f16(<8 x half> [[TMP0]], <8 x half> [[TMP1]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_f16_x2(float16_t *ptr, float16x8x2_t val){ + vstnq_f16_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_bf16_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x bfloat>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8bf16(<8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_bf16_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <8 x bfloat>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <8 x bfloat>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <8 x bfloat> [[VAL_COERCE_FCA_0_EXTRACT]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <8 x bfloat> [[VAL_COERCE_FCA_1_EXTRACT]], <8 x bfloat> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v8bf16(<8 x bfloat> [[TMP0]], <8 x bfloat> [[TMP1]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_bf16_x2(bfloat16_t *ptr, bfloat16x8x2_t val){ + vstnq_bf16_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_f32_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <4 x float>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x float>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x float>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v4f32(<4 x float> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x float> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_f32_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <4 x float>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <4 x float>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <4 x float>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <4 x float> [[VAL_COERCE_FCA_0_EXTRACT]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <4 x float> [[VAL_COERCE_FCA_1_EXTRACT]], <4 x float> poison, <4 x i32> <i32 3, i32 2, i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v4f32(<4 x float> [[TMP0]], <4 x float> [[TMP1]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_f32_x2(float32_t *ptr, float32x4x2_t val){ + vstnq_f32_x2(ptr, val); +} +// CHECK-LE-LABEL: define dso_local void @test_vstnq_f64_x2( +// CHECK-LE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <2 x double>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-LE-NEXT: [[ENTRY:.*:]] +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x double>] [[VAL_COERCE]], 0 +// CHECK-LE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x double>] [[VAL_COERCE]], 1 +// CHECK-LE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v2f64(<2 x double> [[VAL_COERCE_FCA_0_EXTRACT]], <2 x double> [[VAL_COERCE_FCA_1_EXTRACT]], ptr [[PTR]]) +// CHECK-LE-NEXT: ret void +// +// CHECK-BE-LABEL: define dso_local void @test_vstnq_f64_x2( +// CHECK-BE-SAME: ptr nofree noundef writeonly captures(none) [[PTR:%.*]], [2 x <2 x double>] alignstack(16) [[VAL_COERCE:%.*]]) local_unnamed_addr #[[ATTR0]] { +// CHECK-BE-NEXT: [[ENTRY:.*:]] +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_0_EXTRACT:%.*]] = extractvalue [2 x <2 x double>] [[VAL_COERCE]], 0 +// CHECK-BE-NEXT: [[VAL_COERCE_FCA_1_EXTRACT:%.*]] = extractvalue [2 x <2 x double>] [[VAL_COERCE]], 1 +// CHECK-BE-NEXT: [[TMP0:%.*]] = shufflevector <2 x double> [[VAL_COERCE_FCA_0_EXTRACT]], <2 x double> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[VAL_COERCE_FCA_1_EXTRACT]], <2 x double> poison, <2 x i32> <i32 1, i32 0> +// CHECK-BE-NEXT: tail call void @llvm.aarch64.neon.vstnq.x2.v2f64(<2 x double> [[TMP0]], <2 x double> [[TMP1]], ptr [[PTR]]) +// CHECK-BE-NEXT: ret void +// +void test_vstnq_f64_x2(float64_t *ptr, float64x2x2_t val){ + vstnq_f64_x2(ptr, val); +} diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index ff4e20b4e3f5dc..4c59917783d710 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -693,6 +693,16 @@ def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic; def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic; def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic; +// Non-Temporal Pair Load/Store +let TargetPrefix = "aarch64" in { +def int_aarch64_neon_vstnq_x2: DefaultAttrsIntrinsic<[], + [llvm_anyvector_ty, LLVMMatchType<0>, llvm_ptr_ty], + [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; +def int_aarch64_neon_vldnq_x2 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], + [llvm_ptr_ty], [IntrReadMem, IntrArgMemOnly]>; +} + + let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". class AdvSIMD_Tbl1_Intrinsic : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>], diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td index 8b21cffe440ada..a06ea515a829ee 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td @@ -3943,6 +3943,19 @@ defm LDNPD : LoadPairNoAlloc<0b01, 1, FPR64Op, simm7s8, "ldnp">; defm LDNPQ : LoadPairNoAlloc<0b10, 1, FPR128Op, simm7s16, "ldnp">; } +multiclass LDNPQLoadIntrinsic<ValueType VecTy> { + def : Pat<([VecTy, VecTy] (int_aarch64_neon_vldnq_x2 GPR64sp:$Rn)), + ([VecTy, VecTy] (LDNPQi GPR64sp:$Rn, 0))>; +} +defm : LDNPQLoadIntrinsic<v16i8>; +defm : LDNPQLoadIntrinsic<v8i16>; +defm : LDNPQLoadIntrinsic<v4i32>; +defm : LDNPQLoadIntrinsic<v2i64>; +defm : LDNPQLoadIntrinsic<v8f16>; +defm : LDNPQLoadIntrinsic<v8bf16>; +defm : LDNPQLoadIntrinsic<v4f32>; +defm : LDNPQLoadIntrinsic<v2f64>; + def : Pat<(AArch64ldp (am_indexed7s64 GPR64sp:$Rn, simm7s8:$offset)), (LDPXi GPR64sp:$Rn, simm7s8:$offset)>; @@ -4845,6 +4858,19 @@ defm STNPD : StorePairNoAlloc<0b01, 1, FPR64Op, simm7s8, "stnp">; defm STNPQ : StorePairNoAlloc<0b10, 1, FPR128Op, simm7s16, "stnp">; } +multiclass STNPQStoreIntrinsic<ValueType VecTy> { + def : Pat<(int_aarch64_neon_vstnq_x2 (VecTy FPR128:$Rt1), (VecTy FPR128:$Rt2), GPR64sp:$Rn), + (STNPQi FPR128:$Rt1, FPR128:$Rt2, GPR64sp:$Rn, 0)>; +} +defm : STNPQStoreIntrinsic<v16i8>; +defm : STNPQStoreIntrinsic<v8i16>; +defm : STNPQStoreIntrinsic<v4i32>; +defm : STNPQStoreIntrinsic<v2i64>; +defm : STNPQStoreIntrinsic<v8f16>; +defm : STNPQStoreIntrinsic<v8bf16>; +defm : STNPQStoreIntrinsic<v4f32>; +defm : STNPQStoreIntrinsic<v2f64>; + // Armv9.6-a Load/store pair (FEAT_LSUI) let Predicates = [HasLSUI] in { defm LDTP : LoadPairOffset<0b11, 0, GPR64z, simm7s8, "ldtp">; diff --git a/llvm/test/CodeGen/AArch64/vldn_intrinsics.ll b/llvm/test/CodeGen/AArch64/vldn_intrinsics.ll new file mode 100644 index 00000000000000..20bed791284a8f --- /dev/null +++ b/llvm/test/CodeGen/AArch64/vldn_intrinsics.ll @@ -0,0 +1,143 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6 +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK-LE +; RUN: llc -mtriple=aarch64_be -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK-BE + +define { <16 x i8>, <16 x i8> } @test_vldnq_i8_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_i8_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_i8_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: rev64 v0.16b, v0.16b +; CHECK-BE-NEXT: rev64 v1.16b, v1.16b +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <16 x i8>, <16 x i8> } @llvm.aarch64.neon.vldnq.x2.v16i8(ptr %0) + ret { <16 x i8>, <16 x i8> } %2 +} + +define { <8 x i16>, <8 x i16> } @test_vldnq_s16_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_s16_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_s16_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: rev64 v0.8h, v0.8h +; CHECK-BE-NEXT: rev64 v1.8h, v1.8h +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <8 x i16>, <8 x i16> } @llvm.aarch64.neon.vldnq.x2.v8i16(ptr %0) + ret { <8 x i16>, <8 x i16> } %2 +} + +define { <4 x i32>, <4 x i32> } @test_vldnq_s32_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_s32_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_s32_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: rev64 v0.4s, v0.4s +; CHECK-BE-NEXT: rev64 v1.4s, v1.4s +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <4 x i32>, <4 x i32> } @llvm.aarch64.neon.vldnq.x2.v4i32(ptr %0) + ret { <4 x i32>, <4 x i32> } %2 +} + +define { <2 x i64>, <2 x i64> } @test_vldnq_s64_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_s64_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_s64_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <2 x i64>, <2 x i64> } @llvm.aarch64.neon.vldnq.x2.v2i64(ptr %0) + ret { <2 x i64>, <2 x i64> } %2 +} + +define { <8 x half>, <8 x half> } @test_vldnq_f16_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_f16_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_f16_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: rev64 v0.8h, v0.8h +; CHECK-BE-NEXT: rev64 v1.8h, v1.8h +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <8 x half>, <8 x half> } @llvm.aarch64.neon.vldnq.x2.v8f16(ptr %0) + ret { <8 x half>, <8 x half> } %2 +} + +define { <8 x bfloat>, <8 x bfloat> } @test_vldnq_bf16_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_bf16_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_bf16_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: rev64 v0.8h, v0.8h +; CHECK-BE-NEXT: rev64 v1.8h, v1.8h +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <8 x bfloat>, <8 x bfloat> } @llvm.aarch64.neon.vldnq.x2.v8bf16(ptr %0) + ret { <8 x bfloat>, <8 x bfloat> } %2 +} + +define { <4 x float>, <4 x float> } @test_vldnq_f32_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_f32_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_f32_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: rev64 v0.4s, v0.4s +; CHECK-BE-NEXT: rev64 v1.4s, v1.4s +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <4 x float>, <4 x float> } @llvm.aarch64.neon.vldnq.x2.v4f32(ptr %0) + ret { <4 x float>, <4 x float> } %2 +} + +define { <2 x double>, <2 x double> } @test_vldnq_f64_x2(ptr %0) { +; CHECK-LE-LABEL: test_vldnq_f64_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: ldnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vldnq_f64_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ldnp q0, q1, [x0] +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ret + %2 = call { <2 x double>, <2 x double> } @llvm.aarch64.neon.vldnq.x2.v2f64(ptr %0) + ret { <2 x double>, <2 x double> } %2 +} diff --git a/llvm/test/CodeGen/AArch64/vstn_intrinsics.ll b/llvm/test/CodeGen/AArch64/vstn_intrinsics.ll new file mode 100644 index 00000000000000..209d0fd05024a2 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/vstn_intrinsics.ll @@ -0,0 +1,159 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK-LE +; RUN: llc -mtriple=aarch64_be -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK-BE + +define void @test_vstnq_i8_x2(ptr %0, [2 x <16 x i8>] %1) { +; CHECK-LE-LABEL: test_vstnq_i8_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_i8_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: rev64 v1.16b, v1.16b +; CHECK-BE-NEXT: rev64 v0.16b, v0.16b +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <16 x i8>] %1, 0 + %4 = extractvalue [2 x <16 x i8>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v16i8(<16 x i8> %3, <16 x i8> %4, ptr %0) + ret void +} + +define void @test_vstnq_i16_x2(ptr %0, [2 x <8 x i16>] %1) { +; CHECK-LE-LABEL: test_vstnq_i16_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_i16_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: rev64 v1.8h, v1.8h +; CHECK-BE-NEXT: rev64 v0.8h, v0.8h +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <8 x i16>] %1, 0 + %4 = extractvalue [2 x <8 x i16>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v8i16(<8 x i16> %3, <8 x i16> %4, ptr %0) + ret void +} + +define void @test_vstnq_i32_x2(ptr %0, [2 x <4 x i32>] %1) { +; CHECK-LE-LABEL: test_vstnq_i32_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_i32_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: rev64 v1.4s, v1.4s +; CHECK-BE-NEXT: rev64 v0.4s, v0.4s +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <4 x i32>] %1, 0 + %4 = extractvalue [2 x <4 x i32>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v4i32(<4 x i32> %3, <4 x i32> %4, ptr %0) + ret void +} + +define void @test_vstnq_i64_x2(ptr %0, [2 x <2 x i64>] %1) { +; CHECK-LE-LABEL: test_vstnq_i64_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_i64_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <2 x i64>] %1, 0 + %4 = extractvalue [2 x <2 x i64>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v2i64(<2 x i64> %3, <2 x i64> %4, ptr %0) + ret void +} + +define void @test_vstnq_f16_x2(ptr %0, [2 x <8 x half>] %1) { +; CHECK-LE-LABEL: test_vstnq_f16_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_f16_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: rev64 v1.8h, v1.8h +; CHECK-BE-NEXT: rev64 v0.8h, v0.8h +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <8 x half>] %1, 0 + %4 = extractvalue [2 x <8 x half>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v8f16(<8 x half> %3, <8 x half> %4, ptr %0) + ret void +} + +define void @test_vstnq_bf16_x2(ptr %0, [2 x <8 x bfloat>] %1) { +; CHECK-LE-LABEL: test_vstnq_bf16_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_bf16_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: rev64 v1.8h, v1.8h +; CHECK-BE-NEXT: rev64 v0.8h, v0.8h +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <8 x bfloat>] %1, 0 + %4 = extractvalue [2 x <8 x bfloat>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v8bf16(<8 x bfloat> %3, <8 x bfloat> %4, ptr %0) + ret void +} + +define void @test_vstnq_f32_x2(ptr %0, [2 x <4 x float>] %1) { +; CHECK-LE-LABEL: test_vstnq_f32_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_f32_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: rev64 v1.4s, v1.4s +; CHECK-BE-NEXT: rev64 v0.4s, v0.4s +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <4 x float>] %1, 0 + %4 = extractvalue [2 x <4 x float>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v4f32(<4 x float> %3, <4 x float> %4, ptr %0) + ret void +} + +define void @test_vstnq_f64_x2(ptr %0, [2 x <2 x double>] %1) { +; CHECK-LE-LABEL: test_vstnq_f64_x2: +; CHECK-LE: // %bb.0: +; CHECK-LE-NEXT: stnp q0, q1, [x0] +; CHECK-LE-NEXT: ret +; +; CHECK-BE-LABEL: test_vstnq_f64_x2: +; CHECK-BE: // %bb.0: +; CHECK-BE-NEXT: ext v1.16b, v1.16b, v1.16b, #8 +; CHECK-BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8 +; CHECK-BE-NEXT: stnp q0, q1, [x0] +; CHECK-BE-NEXT: ret + %3 = extractvalue [2 x <2 x double>] %1, 0 + %4 = extractvalue [2 x <2 x double>] %1, 1 + call void @llvm.aarch64.neon.vstnq.x2.v2f64(<2 x double> %3, <2 x double> %4, ptr %0) + ret void +} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
