https://github.com/yairbenavraham updated https://github.com/llvm/llvm-project/pull/210359
>From 11194dfe13fd1583b2ada0f2bc7474042813f229 Mon Sep 17 00:00:00 2001 From: Yair Ben Avraham <[email protected]> Date: Fri, 17 Jul 2026 13:04:53 +0300 Subject: [PATCH 1/4] [CIR][AArch64] Move FP16 vfms vector tests Move vfms_f16 and vfmsq_f16 into the focused Neon test with direct LLVM, CIR-to-LLVM, and CIR coverage. Remove the superseded non-constrained tests from v8.2a-neon-intrinsics.c. --- .../AArch64/neon/fused-multiple-fullfp16.c | 44 +++++++++++++++++++ .../CodeGen/AArch64/v8.2a-neon-intrinsics.c | 40 ----------------- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c index b6980049fe52f..b2a9ecf936b77 100644 --- a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c +++ b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c @@ -69,6 +69,50 @@ float16x8_t test_vfmaq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { return vfmaq_f16(a, b, c); } +// LLVM-LABEL: @test_vfms_f16( +// CIR-LABEL: @vfms_f16( +float16x4_t test_vfms_f16(float16x4_t a, float16x4_t b, float16x4_t c) { +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: cir.call @vfma_f16(%{{.*}}, [[FNEG]], %{{.*}}) : + +// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM: [[FNEG:%.*]] = fneg <4 x half> [[B]] +// LLVM-NEXT: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> +// LLVM-NEXT: [[B_I:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> +// LLVM-NEXT: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> +// LLVM-NEXT: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> +// LLVM-NEXT: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> +// LLVM-NEXT: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM-NEXT: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half> +// LLVM-NEXT: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half> +// LLVM-NEXT: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> +// LLVM-NEXT: [[FMA:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[B_CAST]], <4 x half> [[C_CAST]], <4 x half> [[A_CAST]]) +// LLVM-NEXT: ret <4 x half> [[FMA]] + return vfms_f16(a, b, c); +} + +// LLVM-LABEL: @test_vfmsq_f16( +// CIR-LABEL: @vfmsq_f16( +float16x8_t test_vfmsq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: cir.call @vfmaq_f16(%{{.*}}, [[FNEG]], %{{.*}}) : + +// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM: [[FNEG:%.*]] = fneg <8 x half> [[B]] +// LLVM-NEXT: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> +// LLVM-NEXT: [[B_I:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> +// LLVM-NEXT: [[C_I:%.*]] = bitcast <8 x half> [[C]] to <8 x i16> +// LLVM-NEXT: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> +// LLVM-NEXT: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> +// LLVM-NEXT: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8> +// LLVM-NEXT: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half> +// LLVM-NEXT: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half> +// LLVM-NEXT: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half> +// LLVM-NEXT: [[FMA:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B_CAST]], <8 x half> [[C_CAST]], <8 x half> [[A_CAST]]) +// LLVM-NEXT: ret <8 x half> [[FMA]] + return vfmsq_f16(a, b, c); +} + // ALL-LABEL: @test_vfma_lane_f16( float16x4_t test_vfma_lane_f16(float16x4_t a, float16x4_t b, float16x4_t c) { diff --git a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c index a58ade4c4c29f..5c602c983fb29 100644 --- a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c +++ b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c @@ -1550,46 +1550,6 @@ float16x8_t test_vsubq_f16(float16x8_t a, float16x8_t b) { return vsubq_f16(a, b); } -// CHECK-LABEL: define {{[^@]+}}@test_vfms_f16 -// CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[FNEG_I:%.*]] = fneg <4 x half> [[B]] -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[FNEG_I]] to <4 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half> -// CHECK-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]]) -// CHECK-NEXT: ret <4 x half> [[TMP9]] -// -float16x4_t test_vfms_f16(float16x4_t a, float16x4_t b, float16x4_t c) { - return vfms_f16(a, b, c); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmsq_f16 -// CHECK-SAME: (<8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[FNEG_I:%.*]] = fneg <8 x half> [[B]] -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[FNEG_I]] to <8 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half> -// CHECK-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[TMP7]], <8 x half> [[TMP8]], <8 x half> [[TMP6]]) -// CHECK-NEXT: ret <8 x half> [[TMP9]] -// -float16x8_t test_vfmsq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { - return vfmsq_f16(a, b, c); -} - // CHECK-LABEL: define {{[^@]+}}@test_vfma_n_f16 // CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], half noundef [[C:%.*]]) #[[ATTR0]] { // CHECK-NEXT: entry: >From f9dc2548afefd0893ed4bb28a7ba1c470be2f434 Mon Sep 17 00:00:00 2001 From: Yair Ben Avraham <[email protected]> Date: Fri, 17 Jul 2026 13:06:07 +0300 Subject: [PATCH 2/4] [CIR][AArch64] Move remaining FP16 FMA tests Move the remaining ten FP16 fused multiply-accumulate wrappers into the focused Neon test with direct LLVM, CIR-to-LLVM, and CIR coverage. Remove their superseded non-constrained legacy tests. --- .../AArch64/neon/fused-multiple-fullfp16.c | 221 +++++++++++++++++- .../CodeGen/AArch64/v8.2a-neon-intrinsics.c | 214 ----------------- 2 files changed, 216 insertions(+), 219 deletions(-) diff --git a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c index b2a9ecf936b77..4f17037e55cff 100644 --- a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c +++ b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c @@ -12,11 +12,10 @@ // This file contains fullfp16 tests that were originally located in: // * clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c // The main difference is the use of RUN lines that enable ClangIR lowering. -// This file currently covers the f16 wrappers that lower through -// BI__builtin_neon_vfmaq_v, BI__builtin_neon_vfmaq_lane_v, -// BI__builtin_neon_vfmaq_laneq_v, BI__builtin_neon_vfma_laneq_v, -// BI__builtin_neon_vfmah_lane_f16, and -// BI__builtin_neon_vfmah_laneq_f16. +// This file currently covers the f16 ACLE wrappers from the +// fused-multiply-accumulate section. Several wrappers are header-level forms +// over the existing lowered vfma/vfmah builtins, such as vfma_n_f16, +// vfms_f16, vfms_lane_f16, vfms_n_f16, and vfmsh_lane_f16. // // ACLE section headings based on v2025Q2 of the ACLE specification: // * https://arm-software.github.io/acle/neon_intrinsics/advsimd.html#fused-multiply-accumulate-2 @@ -201,6 +200,48 @@ float16x8_t test_vfmaq_laneq_f16(float16x8_t a, float16x8_t b, return vfmaq_laneq_f16(a, b, c, 7); } +// ALL-LABEL: @test_vfma_n_f16( +float16x4_t test_vfma_n_f16(float16x4_t a, float16x4_t b, float16_t c) { +// CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<4 x !cir.f16> +// CIR: cir.call @vfma_f16(%{{.*}}, %{{.*}}, %{{.*}}) : + +// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], half {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[SPLAT:%.*]] = insertelement <4 x half> {{.*}}, half [[C]], i{{32|64}} 3 +// LLVM-DAG: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <4 x half> [[B]] to <4 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <4 x half> [[SPLAT]] to <4 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> +// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[B_CAST]], <4 x half> [[C_CAST]], <4 x half> [[A_CAST]]) +// LLVM: ret <4 x half> [[FMA]] + return vfma_n_f16(a, b, c); +} + +// ALL-LABEL: @test_vfmaq_n_f16( +float16x8_t test_vfmaq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { +// CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<8 x !cir.f16> +// CIR: cir.call @vfmaq_f16(%{{.*}}, %{{.*}}, %{{.*}}) : + +// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], half {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[SPLAT:%.*]] = insertelement <8 x half> {{.*}}, half [[C]], i{{32|64}} 7 +// LLVM-DAG: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <8 x half> [[B]] to <8 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <8 x half> [[SPLAT]] to <8 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half> +// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B_CAST]], <8 x half> [[C_CAST]], <8 x half> [[A_CAST]]) +// LLVM: ret <8 x half> [[FMA]] + return vfmaq_n_f16(a, b, c); +} + // ALL-LABEL: @test_vfmah_lane_f16( float16_t test_vfmah_lane_f16(float16_t a, float16_t b, float16x4_t c) { // CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<4 x !cir.f16> @@ -224,3 +265,173 @@ float16_t test_vfmah_laneq_f16(float16_t a, float16_t b, float16x8_t c) { // LLVM: ret half [[FMA]] return vfmah_laneq_f16(a, b, c, 7); } + +// ALL-LABEL: @test_vfms_lane_f16( +float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, + float16x4_t c) { +// CIR: cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<4 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> + +// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[FNEG:%.*]] = fneg <4 x half> [[B]] +// LLVM-DAG: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half> +// LLVM-DAG: [[LANE:%.*]] = shufflevector <4 x half> [[C_CAST]], <4 x half> {{.*}}, <4 x i32> <i32 3, i32 3, i32 3, i32 3> +// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[B_CAST]], <4 x half> [[LANE]], <4 x half> [[A_CAST]]) +// LLVM: ret <4 x half> [[FMA]] + return vfms_lane_f16(a, b, c, 3); +} + +// ALL-LABEL: @test_vfmsq_lane_f16( +float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, + float16x4_t c) { +// CIR: cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<8 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> + +// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[FNEG:%.*]] = fneg <8 x half> [[B]] +// LLVM-DAG: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half> +// LLVM-DAG: [[LANE:%.*]] = shufflevector <4 x half> [[C_CAST]], <4 x half> {{.*}}, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3> +// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B_CAST]], <8 x half> [[LANE]], <8 x half> [[A_CAST]]) +// LLVM: ret <8 x half> [[FMA]] + return vfmsq_lane_f16(a, b, c, 3); +} + +// ALL-LABEL: @test_vfms_laneq_f16( +float16x4_t test_vfms_laneq_f16(float16x4_t a, float16x4_t b, + float16x8_t c) { +// CIR: cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<4 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" [[LANE]], %{{.*}}, %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> + +// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[FNEG:%.*]] = fneg <4 x half> [[B]] +// LLVM-DAG: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <8 x half> [[C]] to <8 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half> +// LLVM-DAG: [[LANE:%.*]] = shufflevector <8 x half> [[C_CAST]], <8 x half> {{.*}}, <4 x i32> <i32 7, i32 7, i32 7, i32 7> +// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[LANE]], <4 x half> [[B_CAST]], <4 x half> [[A_CAST]]) +// LLVM: ret <4 x half> [[FMA]] + return vfms_laneq_f16(a, b, c, 7); +} + +// ALL-LABEL: @test_vfmsq_laneq_f16( +float16x8_t test_vfmsq_laneq_f16(float16x8_t a, float16x8_t b, + float16x8_t c) { +// CIR: cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<8 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" [[LANE]], %{{.*}}, %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> + +// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[FNEG:%.*]] = fneg <8 x half> [[B]] +// LLVM-DAG: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <8 x half> [[C]] to <8 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half> +// LLVM-DAG: [[LANE:%.*]] = shufflevector <8 x half> [[C_CAST]], <8 x half> {{.*}}, <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7> +// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[LANE]], <8 x half> [[B_CAST]], <8 x half> [[A_CAST]]) +// LLVM: ret <8 x half> [[FMA]] + return vfmsq_laneq_f16(a, b, c, 7); +} + +// ALL-LABEL: @test_vfms_n_f16( +float16x4_t test_vfms_n_f16(float16x4_t a, float16x4_t b, float16_t c) { +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<4 x !cir.f16> +// CIR: cir.call @vfma_f16(%{{.*}}, [[FNEG]], %{{.*}}) : + +// LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], half {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[FNEG:%.*]] = fneg <4 x half> [[B]] +// LLVM-DAG: [[SPLAT:%.*]] = insertelement <4 x half> {{.*}}, half [[C]], i{{32|64}} 3 +// LLVM-DAG: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <4 x half> [[SPLAT]] to <4 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> +// LLVM: [[FMA:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[B_CAST]], <4 x half> [[C_CAST]], <4 x half> [[A_CAST]]) +// LLVM: ret <4 x half> [[FMA]] + return vfms_n_f16(a, b, c); +} + +// ALL-LABEL: @test_vfmsq_n_f16( +float16x8_t test_vfmsq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<8 x !cir.f16> +// CIR: cir.call @vfmaq_f16(%{{.*}}, [[FNEG]], %{{.*}}) : + +// LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], half {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[FNEG:%.*]] = fneg <8 x half> [[B]] +// LLVM-DAG: [[SPLAT:%.*]] = insertelement <8 x half> {{.*}}, half [[C]], i{{32|64}} 7 +// LLVM-DAG: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> +// LLVM-DAG: [[B_I:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> +// LLVM-DAG: [[C_I:%.*]] = bitcast <8 x half> [[SPLAT]] to <8 x i16> +// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> +// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> +// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <8 x i16> [[C_I]] to <16 x i8> +// LLVM-DAG: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half> +// LLVM-DAG: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half> +// LLVM-DAG: [[C_CAST:%.*]] = bitcast <16 x i8> [[C_BYTES]] to <8 x half> +// LLVM: [[FMA:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[B_CAST]], <8 x half> [[C_CAST]], <8 x half> [[A_CAST]]) +// LLVM: ret <8 x half> [[FMA]] + return vfmsq_n_f16(a, b, c); +} + +// ALL-LABEL: @test_vfmsh_lane_f16( +float16_t test_vfmsh_lane_f16(float16_t a, float16_t b, float16x4_t c) { +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<4 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 + +// LLVM-SAME: half {{.*}} [[A:%.*]], half {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[LANE:%.*]] = extractelement <4 x half> [[C]], i{{32|64}} 3 +// LLVM-DAG: [[NEG:%.*]] = {{(fptrunc float .* to half|fneg half .*)}} +// LLVM: [[FMA:%.*]] = call half @llvm.fma.f16(half [[NEG]], half [[LANE]], half [[A]]) +// LLVM: ret half [[FMA]] + return vfmsh_lane_f16(a, b, c, 3); +} + +// ALL-LABEL: @test_vfmsh_laneq_f16( +float16_t test_vfmsh_laneq_f16(float16_t a, float16_t b, float16x8_t c) { +// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<8 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 + +// LLVM-SAME: half {{.*}} [[A:%.*]], half {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { +// LLVM-DAG: [[LANE:%.*]] = extractelement <8 x half> [[C]], i{{32|64}} 7 +// LLVM-DAG: [[NEG:%.*]] = {{(fptrunc float .* to half|fneg half .*)}} +// LLVM: [[FMA:%.*]] = call half @llvm.fma.f16(half [[NEG]], half [[LANE]], half [[A]]) +// LLVM: ret half [[FMA]] + return vfmsh_laneq_f16(a, b, c, 7); +} diff --git a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c index 5c602c983fb29..2a017370079bc 100644 --- a/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c +++ b/clang/test/CodeGen/AArch64/v8.2a-neon-intrinsics.c @@ -1550,220 +1550,6 @@ float16x8_t test_vsubq_f16(float16x8_t a, float16x8_t b) { return vsubq_f16(a, b); } -// CHECK-LABEL: define {{[^@]+}}@test_vfma_n_f16 -// CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], half noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <4 x half> poison, half [[C]], i32 0 -// CHECK-NEXT: [[VECINIT1:%.*]] = insertelement <4 x half> [[VECINIT]], half [[C]], i32 1 -// CHECK-NEXT: [[VECINIT2:%.*]] = insertelement <4 x half> [[VECINIT1]], half [[C]], i32 2 -// CHECK-NEXT: [[VECINIT3:%.*]] = insertelement <4 x half> [[VECINIT2]], half [[C]], i32 3 -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[B]] to <4 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[VECINIT3]] to <4 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half> -// CHECK-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]]) -// CHECK-NEXT: ret <4 x half> [[TMP9]] -// -float16x4_t test_vfma_n_f16(float16x4_t a, float16x4_t b, float16_t c) { - return vfma_n_f16(a, b, c); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmaq_n_f16 -// CHECK-SAME: (<8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], half noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <8 x half> poison, half [[C]], i32 0 -// CHECK-NEXT: [[VECINIT1:%.*]] = insertelement <8 x half> [[VECINIT]], half [[C]], i32 1 -// CHECK-NEXT: [[VECINIT2:%.*]] = insertelement <8 x half> [[VECINIT1]], half [[C]], i32 2 -// CHECK-NEXT: [[VECINIT3:%.*]] = insertelement <8 x half> [[VECINIT2]], half [[C]], i32 3 -// CHECK-NEXT: [[VECINIT4:%.*]] = insertelement <8 x half> [[VECINIT3]], half [[C]], i32 4 -// CHECK-NEXT: [[VECINIT5:%.*]] = insertelement <8 x half> [[VECINIT4]], half [[C]], i32 5 -// CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <8 x half> [[VECINIT5]], half [[C]], i32 6 -// CHECK-NEXT: [[VECINIT7:%.*]] = insertelement <8 x half> [[VECINIT6]], half [[C]], i32 7 -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[B]] to <8 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[VECINIT7]] to <8 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half> -// CHECK-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[TMP7]], <8 x half> [[TMP8]], <8 x half> [[TMP6]]) -// CHECK-NEXT: ret <8 x half> [[TMP9]] -// -float16x8_t test_vfmaq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { - return vfmaq_n_f16(a, b, c); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfms_lane_f16 -// CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> -// CHECK-NEXT: [[FNEG:%.*]] = fneg <4 x half> [[B]] -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half> -// CHECK-NEXT: [[LANE:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP6]], <4 x i32> <i32 3, i32 3, i32 3, i32 3> -// CHECK-NEXT: [[FMLA:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half> -// CHECK-NEXT: [[FMLA1:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half> -// CHECK-NEXT: [[FMLA2:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[FMLA]], <4 x half> [[LANE]], <4 x half> [[FMLA1]]) -// CHECK-NEXT: ret <4 x half> [[FMLA2]] -// -float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, float16x4_t c) { - return vfms_lane_f16(a, b, c, 3); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmsq_lane_f16 -// CHECK-SAME: (<8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> -// CHECK-NEXT: [[FNEG:%.*]] = fneg <8 x half> [[B]] -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half> -// CHECK-NEXT: [[LANE:%.*]] = shufflevector <4 x half> [[TMP6]], <4 x half> [[TMP6]], <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3> -// CHECK-NEXT: [[FMLA:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half> -// CHECK-NEXT: [[FMLA1:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half> -// CHECK-NEXT: [[FMLA2:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[FMLA]], <8 x half> [[LANE]], <8 x half> [[FMLA1]]) -// CHECK-NEXT: ret <8 x half> [[FMLA2]] -// -float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, float16x4_t c) { - return vfmsq_lane_f16(a, b, c, 3); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfms_laneq_f16 -// CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> -// CHECK-NEXT: [[FNEG:%.*]] = fneg <4 x half> [[B]] -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half> -// CHECK-NEXT: [[LANE:%.*]] = shufflevector <8 x half> [[TMP8]], <8 x half> [[TMP8]], <4 x i32> <i32 7, i32 7, i32 7, i32 7> -// CHECK-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[LANE]], <4 x half> [[TMP7]], <4 x half> [[TMP6]]) -// CHECK-NEXT: ret <4 x half> [[TMP9]] -// -float16x4_t test_vfms_laneq_f16(float16x4_t a, float16x4_t b, float16x8_t c) { - return vfms_laneq_f16(a, b, c, 7); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmsq_laneq_f16 -// CHECK-SAME: (<8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> -// CHECK-NEXT: [[FNEG:%.*]] = fneg <8 x half> [[B]] -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[C]] to <8 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half> -// CHECK-NEXT: [[LANE:%.*]] = shufflevector <8 x half> [[TMP8]], <8 x half> [[TMP8]], <8 x i32> <i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7, i32 7> -// CHECK-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[LANE]], <8 x half> [[TMP7]], <8 x half> [[TMP6]]) -// CHECK-NEXT: ret <8 x half> [[TMP9]] -// -float16x8_t test_vfmsq_laneq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { - return vfmsq_laneq_f16(a, b, c, 7); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfms_n_f16 -// CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]], half noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[FNEG:%.*]] = fneg <4 x half> [[B]] -// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <4 x half> poison, half [[C]], i32 0 -// CHECK-NEXT: [[VECINIT1:%.*]] = insertelement <4 x half> [[VECINIT]], half [[C]], i32 1 -// CHECK-NEXT: [[VECINIT2:%.*]] = insertelement <4 x half> [[VECINIT1]], half [[C]], i32 2 -// CHECK-NEXT: [[VECINIT3:%.*]] = insertelement <4 x half> [[VECINIT2]], half [[C]], i32 3 -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x half> [[VECINIT3]] to <4 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i16> [[TMP0]] to <8 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x i16> [[TMP1]] to <8 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x i16> [[TMP2]] to <8 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <8 x i8> [[TMP3]] to <4 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <8 x i8> [[TMP4]] to <4 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <8 x i8> [[TMP5]] to <4 x half> -// CHECK-NEXT: [[TMP9:%.*]] = call <4 x half> @llvm.fma.v4f16(<4 x half> [[TMP7]], <4 x half> [[TMP8]], <4 x half> [[TMP6]]) -// CHECK-NEXT: ret <4 x half> [[TMP9]] -// -float16x4_t test_vfms_n_f16(float16x4_t a, float16x4_t b, float16_t c) { - return vfms_n_f16(a, b, c); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmsq_n_f16 -// CHECK-SAME: (<8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]], half noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[FNEG:%.*]] = fneg <8 x half> [[B]] -// CHECK-NEXT: [[VECINIT:%.*]] = insertelement <8 x half> poison, half [[C]], i32 0 -// CHECK-NEXT: [[VECINIT1:%.*]] = insertelement <8 x half> [[VECINIT]], half [[C]], i32 1 -// CHECK-NEXT: [[VECINIT2:%.*]] = insertelement <8 x half> [[VECINIT1]], half [[C]], i32 2 -// CHECK-NEXT: [[VECINIT3:%.*]] = insertelement <8 x half> [[VECINIT2]], half [[C]], i32 3 -// CHECK-NEXT: [[VECINIT4:%.*]] = insertelement <8 x half> [[VECINIT3]], half [[C]], i32 4 -// CHECK-NEXT: [[VECINIT5:%.*]] = insertelement <8 x half> [[VECINIT4]], half [[C]], i32 5 -// CHECK-NEXT: [[VECINIT6:%.*]] = insertelement <8 x half> [[VECINIT5]], half [[C]], i32 6 -// CHECK-NEXT: [[VECINIT7:%.*]] = insertelement <8 x half> [[VECINIT6]], half [[C]], i32 7 -// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> -// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> -// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[VECINIT7]] to <8 x i16> -// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8> -// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8> -// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8> -// CHECK-NEXT: [[TMP6:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half> -// CHECK-NEXT: [[TMP7:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half> -// CHECK-NEXT: [[TMP8:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half> -// CHECK-NEXT: [[TMP9:%.*]] = call <8 x half> @llvm.fma.v8f16(<8 x half> [[TMP7]], <8 x half> [[TMP8]], <8 x half> [[TMP6]]) -// CHECK-NEXT: ret <8 x half> [[TMP9]] -// -float16x8_t test_vfmsq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { - return vfmsq_n_f16(a, b, c); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmsh_lane_f16 -// CHECK-SAME: (half noundef [[A:%.*]], half noundef [[B:%.*]], <4 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[CONV:%.*]] = fpext half [[B]] to float -// CHECK-NEXT: [[FNEG:%.*]] = fneg float [[CONV]] -// CHECK-NEXT: [[CONV1:%.*]] = fptrunc float [[FNEG]] to half -// CHECK-NEXT: [[EXTRACT:%.*]] = extractelement <4 x half> [[C]], i32 3 -// CHECK-NEXT: [[TMP0:%.*]] = call half @llvm.fma.f16(half [[CONV1]], half [[EXTRACT]], half [[A]]) -// CHECK-NEXT: ret half [[TMP0]] -// -float16_t test_vfmsh_lane_f16(float16_t a, float16_t b, float16x4_t c) { - return vfmsh_lane_f16(a, b, c, 3); -} - -// CHECK-LABEL: define {{[^@]+}}@test_vfmsh_laneq_f16 -// CHECK-SAME: (half noundef [[A:%.*]], half noundef [[B:%.*]], <8 x half> noundef [[C:%.*]]) #[[ATTR0]] { -// CHECK-NEXT: entry: -// CHECK-NEXT: [[CONV:%.*]] = fpext half [[B]] to float -// CHECK-NEXT: [[FNEG:%.*]] = fneg float [[CONV]] -// CHECK-NEXT: [[CONV1:%.*]] = fptrunc float [[FNEG]] to half -// CHECK-NEXT: [[EXTRACT:%.*]] = extractelement <8 x half> [[C]], i32 7 -// CHECK-NEXT: [[TMP0:%.*]] = call half @llvm.fma.f16(half [[CONV1]], half [[EXTRACT]], half [[A]]) -// CHECK-NEXT: ret half [[TMP0]] -// -float16_t test_vfmsh_laneq_f16(float16_t a, float16_t b, float16x8_t c) { - return vfmsh_laneq_f16(a, b, c, 7); -} - // CHECK-LABEL: define {{[^@]+}}@test_vmul_lane_f16 // CHECK-SAME: (<4 x half> noundef [[A:%.*]], <4 x half> noundef [[B:%.*]]) #[[ATTR0]] { // CHECK-NEXT: entry: >From 82ebc1b3623e4879dcf9c7ae0674fa96b2e7ba32 Mon Sep 17 00:00:00 2001 From: Yair Ben Avraham <[email protected]> Date: Fri, 17 Jul 2026 13:06:32 +0300 Subject: [PATCH 3/4] [CIR][AArch64] Strengthen FP16 FMA checks Track negated operands through the CIR store, cast, and load paths into the final FMA calls for the FP16 subtract lane forms. Also tie the scalar LLVM negation sequence to the original b argument before checking the FMA operand. --- .../AArch64/neon/fused-multiple-fullfp16.c | 56 ++++++++++++++----- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c index 4f17037e55cff..f51c3a55d6c6b 100644 --- a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c +++ b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c @@ -269,9 +269,13 @@ float16_t test_vfmah_laneq_f16(float16_t a, float16_t b, float16x8_t c) { // ALL-LABEL: @test_vfms_lane_f16( float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, float16x4_t c) { -// CIR: cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: cir.store align(8) [[FNEG]], [[NEG_SLOT:%.*]] : !cir.vector<4 x !cir.f16>, !cir.ptr<!cir.vector<4 x !cir.f16>> +// CIR: [[NEG_PTR:%.*]] = cir.cast bitcast [[NEG_SLOT]] : !cir.ptr<!cir.vector<4 x !cir.f16>> -> !cir.ptr<!cir.vector<8 x !s8i>> +// CIR: [[NEG_BYTES:%.*]] = cir.load align(8) [[NEG_PTR]] : !cir.ptr<!cir.vector<8 x !s8i>>, !cir.vector<8 x !s8i> +// CIR: [[NEG_CAST:%.*]] = cir.cast bitcast [[NEG_BYTES]] : !cir.vector<8 x !s8i> -> !cir.vector<4 x !cir.f16> // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<4 x !cir.f16> -// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" [[NEG_CAST]], [[LANE]], %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> // LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { // LLVM-DAG: [[FNEG:%.*]] = fneg <4 x half> [[B]] @@ -293,9 +297,13 @@ float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, // ALL-LABEL: @test_vfmsq_lane_f16( float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, float16x4_t c) { -// CIR: cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: cir.store align(16) [[FNEG]], [[NEG_SLOT:%.*]] : !cir.vector<8 x !cir.f16>, !cir.ptr<!cir.vector<8 x !cir.f16>> +// CIR: [[NEG_PTR:%.*]] = cir.cast bitcast [[NEG_SLOT]] : !cir.ptr<!cir.vector<8 x !cir.f16>> -> !cir.ptr<!cir.vector<16 x !s8i>> +// CIR: [[NEG_BYTES:%.*]] = cir.load align(16) [[NEG_PTR]] : !cir.ptr<!cir.vector<16 x !s8i>>, !cir.vector<16 x !s8i> +// CIR: [[NEG_CAST:%.*]] = cir.cast bitcast [[NEG_BYTES]] : !cir.vector<16 x !s8i> -> !cir.vector<8 x !cir.f16> // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<8 x !cir.f16> -// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" [[NEG_CAST]], [[LANE]], %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> // LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { // LLVM-DAG: [[FNEG:%.*]] = fneg <8 x half> [[B]] @@ -317,9 +325,13 @@ float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, // ALL-LABEL: @test_vfms_laneq_f16( float16x4_t test_vfms_laneq_f16(float16x4_t a, float16x4_t b, float16x8_t c) { -// CIR: cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> +// CIR: cir.store align(8) [[FNEG]], [[NEG_SLOT:%.*]] : !cir.vector<4 x !cir.f16>, !cir.ptr<!cir.vector<4 x !cir.f16>> +// CIR: [[NEG_PTR:%.*]] = cir.cast bitcast [[NEG_SLOT]] : !cir.ptr<!cir.vector<4 x !cir.f16>> -> !cir.ptr<!cir.vector<8 x !s8i>> +// CIR: [[NEG_BYTES:%.*]] = cir.load align(8) [[NEG_PTR]] : !cir.ptr<!cir.vector<8 x !s8i>>, !cir.vector<8 x !s8i> +// CIR: [[NEG_CAST:%.*]] = cir.cast bitcast [[NEG_BYTES]] : !cir.vector<8 x !s8i> -> !cir.vector<4 x !cir.f16> // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<4 x !cir.f16> -// CIR: cir.call_llvm_intrinsic "fma" [[LANE]], %{{.*}}, %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" [[LANE]], [[NEG_CAST]], %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> // LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { // LLVM-DAG: [[FNEG:%.*]] = fneg <4 x half> [[B]] @@ -341,9 +353,13 @@ float16x4_t test_vfms_laneq_f16(float16x4_t a, float16x4_t b, // ALL-LABEL: @test_vfmsq_laneq_f16( float16x8_t test_vfmsq_laneq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { -// CIR: cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> +// CIR: cir.store align(16) [[FNEG]], [[NEG_SLOT:%.*]] : !cir.vector<8 x !cir.f16>, !cir.ptr<!cir.vector<8 x !cir.f16>> +// CIR: [[NEG_PTR:%.*]] = cir.cast bitcast [[NEG_SLOT]] : !cir.ptr<!cir.vector<8 x !cir.f16>> -> !cir.ptr<!cir.vector<16 x !s8i>> +// CIR: [[NEG_BYTES:%.*]] = cir.load align(16) [[NEG_PTR]] : !cir.ptr<!cir.vector<16 x !s8i>>, !cir.vector<16 x !s8i> +// CIR: [[NEG_CAST:%.*]] = cir.cast bitcast [[NEG_BYTES]] : !cir.vector<16 x !s8i> -> !cir.vector<8 x !cir.f16> // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<8 x !cir.f16> -// CIR: cir.call_llvm_intrinsic "fma" [[LANE]], %{{.*}}, %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> +// CIR: cir.call_llvm_intrinsic "fma" [[LANE]], [[NEG_CAST]], %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> // LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { // LLVM-DAG: [[FNEG:%.*]] = fneg <8 x half> [[B]] @@ -410,13 +426,19 @@ float16x8_t test_vfmsq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { // ALL-LABEL: @test_vfmsh_lane_f16( float16_t test_vfmsh_lane_f16(float16_t a, float16_t b, float16x4_t c) { -// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: [[CONV:%.*]] = cir.cast floating %{{.*}} : !cir.f16 -> !cir.float +// CIR: [[FNEG:%.*]] = cir.fneg [[CONV]] : !cir.float +// CIR: [[NEG:%.*]] = cir.cast floating [[FNEG]] : !cir.float -> !cir.f16 +// CIR: cir.store align(2) [[NEG]], [[NEG_SLOT:%.*]] : !cir.f16, !cir.ptr<!cir.f16> +// CIR: [[NEG_ARG:%.*]] = cir.load align(2) [[NEG_SLOT]] : !cir.ptr<!cir.f16>, !cir.f16 // CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<4 x !cir.f16> -// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 +// CIR: cir.call_llvm_intrinsic "fma" [[NEG_ARG]], [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 // LLVM-SAME: half {{.*}} [[A:%.*]], half {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { // LLVM-DAG: [[LANE:%.*]] = extractelement <4 x half> [[C]], i{{32|64}} 3 -// LLVM-DAG: [[NEG:%.*]] = {{(fptrunc float .* to half|fneg half .*)}} +// LLVM-DAG: [[CONV:%.*]] = fpext half [[B]] to float +// LLVM-DAG: [[FNEG:%.*]] = fneg float [[CONV]] +// LLVM-DAG: [[NEG:%.*]] = fptrunc float [[FNEG]] to half // LLVM: [[FMA:%.*]] = call half @llvm.fma.f16(half [[NEG]], half [[LANE]], half [[A]]) // LLVM: ret half [[FMA]] return vfmsh_lane_f16(a, b, c, 3); @@ -424,13 +446,19 @@ float16_t test_vfmsh_lane_f16(float16_t a, float16_t b, float16x4_t c) { // ALL-LABEL: @test_vfmsh_laneq_f16( float16_t test_vfmsh_laneq_f16(float16_t a, float16_t b, float16x8_t c) { -// CIR: cir.fneg %{{.*}} : !cir.float +// CIR: [[CONV:%.*]] = cir.cast floating %{{.*}} : !cir.f16 -> !cir.float +// CIR: [[FNEG:%.*]] = cir.fneg [[CONV]] : !cir.float +// CIR: [[NEG:%.*]] = cir.cast floating [[FNEG]] : !cir.float -> !cir.f16 +// CIR: cir.store align(2) [[NEG]], [[NEG_SLOT:%.*]] : !cir.f16, !cir.ptr<!cir.f16> +// CIR: [[NEG_ARG:%.*]] = cir.load align(2) [[NEG_SLOT]] : !cir.ptr<!cir.f16>, !cir.f16 // CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<8 x !cir.f16> -// CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 +// CIR: cir.call_llvm_intrinsic "fma" [[NEG_ARG]], [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 // LLVM-SAME: half {{.*}} [[A:%.*]], half {{.*}} [[B:%.*]], <8 x half> {{.*}} [[C:%.*]]) {{.*}} { // LLVM-DAG: [[LANE:%.*]] = extractelement <8 x half> [[C]], i{{32|64}} 7 -// LLVM-DAG: [[NEG:%.*]] = {{(fptrunc float .* to half|fneg half .*)}} +// LLVM-DAG: [[CONV:%.*]] = fpext half [[B]] to float +// LLVM-DAG: [[FNEG:%.*]] = fneg float [[CONV]] +// LLVM-DAG: [[NEG:%.*]] = fptrunc float [[FNEG]] to half // LLVM: [[FMA:%.*]] = call half @llvm.fma.f16(half [[NEG]], half [[LANE]], half [[A]]) // LLVM: ret half [[FMA]] return vfmsh_laneq_f16(a, b, c, 7); >From ab1e973ca2c10f2863542cdb769de74707d30cf9 Mon Sep 17 00:00:00 2001 From: Yair Ben Avraham <[email protected]> Date: Tue, 28 Jul 2026 06:54:02 +0300 Subject: [PATCH 4/4] [CIR][AArch64] Refine FP16 FMA checks Use separate LLVM and CIR labels throughout the FP16 FMA tests so each check block is explicitly anchored to the emitted function. Use LLVM-NEXT for the shared-order vfms lane setup and retain LLVM-DAG only where direct LLVM and CIR-to-LLVM emit the casts and lane shuffle in different relative orders. --- .../AArch64/neon/fused-multiple-fullfp16.c | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c index f51c3a55d6c6b..ef9cca5ecc8e9 100644 --- a/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c +++ b/clang/test/CodeGen/AArch64/neon/fused-multiple-fullfp16.c @@ -112,7 +112,8 @@ float16x8_t test_vfmsq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { return vfmsq_f16(a, b, c); } -// ALL-LABEL: @test_vfma_lane_f16( +// LLVM-LABEL: @test_vfma_lane_f16( +// CIR-LABEL: @test_vfma_lane_f16( float16x4_t test_vfma_lane_f16(float16x4_t a, float16x4_t b, float16x4_t c) { // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<4 x !cir.f16> @@ -134,7 +135,8 @@ float16x4_t test_vfma_lane_f16(float16x4_t a, float16x4_t b, return vfma_lane_f16(a, b, c, 3); } -// ALL-LABEL: @test_vfmaq_lane_f16( +// LLVM-LABEL: @test_vfmaq_lane_f16( +// CIR-LABEL: @test_vfmaq_lane_f16( float16x8_t test_vfmaq_lane_f16(float16x8_t a, float16x8_t b, float16x4_t c) { // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<4 x !cir.f16>) [#cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i, #cir.int<3> : !s32i] : !cir.vector<8 x !cir.f16> @@ -156,7 +158,8 @@ float16x8_t test_vfmaq_lane_f16(float16x8_t a, float16x8_t b, return vfmaq_lane_f16(a, b, c, 3); } -// ALL-LABEL: @test_vfma_laneq_f16( +// LLVM-LABEL: @test_vfma_laneq_f16( +// CIR-LABEL: @test_vfma_laneq_f16( float16x4_t test_vfma_laneq_f16(float16x4_t a, float16x4_t b, float16x8_t c) { // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<4 x !cir.f16> @@ -178,7 +181,8 @@ float16x4_t test_vfma_laneq_f16(float16x4_t a, float16x4_t b, return vfma_laneq_f16(a, b, c, 7); } -// ALL-LABEL: @test_vfmaq_laneq_f16( +// LLVM-LABEL: @test_vfmaq_laneq_f16( +// CIR-LABEL: @test_vfmaq_laneq_f16( float16x8_t test_vfmaq_laneq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { // CIR: [[LANE:%.*]] = cir.vec.shuffle(%{{.*}}, %{{.*}} : !cir.vector<8 x !cir.f16>) [#cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i, #cir.int<7> : !s32i] : !cir.vector<8 x !cir.f16> @@ -200,7 +204,8 @@ float16x8_t test_vfmaq_laneq_f16(float16x8_t a, float16x8_t b, return vfmaq_laneq_f16(a, b, c, 7); } -// ALL-LABEL: @test_vfma_n_f16( +// LLVM-LABEL: @test_vfma_n_f16( +// CIR-LABEL: @test_vfma_n_f16( float16x4_t test_vfma_n_f16(float16x4_t a, float16x4_t b, float16_t c) { // CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<4 x !cir.f16> // CIR: cir.call @vfma_f16(%{{.*}}, %{{.*}}, %{{.*}}) : @@ -221,7 +226,8 @@ float16x4_t test_vfma_n_f16(float16x4_t a, float16x4_t b, float16_t c) { return vfma_n_f16(a, b, c); } -// ALL-LABEL: @test_vfmaq_n_f16( +// LLVM-LABEL: @test_vfmaq_n_f16( +// CIR-LABEL: @test_vfmaq_n_f16( float16x8_t test_vfmaq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { // CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<8 x !cir.f16> // CIR: cir.call @vfmaq_f16(%{{.*}}, %{{.*}}, %{{.*}}) : @@ -242,7 +248,8 @@ float16x8_t test_vfmaq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { return vfmaq_n_f16(a, b, c); } -// ALL-LABEL: @test_vfmah_lane_f16( +// LLVM-LABEL: @test_vfmah_lane_f16( +// CIR-LABEL: @test_vfmah_lane_f16( float16_t test_vfmah_lane_f16(float16_t a, float16_t b, float16x4_t c) { // CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<4 x !cir.f16> // CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 @@ -254,7 +261,8 @@ float16_t test_vfmah_lane_f16(float16_t a, float16_t b, float16x4_t c) { return vfmah_lane_f16(a, b, c, 3); } -// ALL-LABEL: @test_vfmah_laneq_f16( +// LLVM-LABEL: @test_vfmah_laneq_f16( +// CIR-LABEL: @test_vfmah_laneq_f16( float16_t test_vfmah_laneq_f16(float16_t a, float16_t b, float16x8_t c) { // CIR: [[LANE:%.*]] = cir.vec.extract %{{.*}}[%{{.*}} : !u64i] : !cir.vector<8 x !cir.f16> // CIR: cir.call_llvm_intrinsic "fma" %{{.*}}, [[LANE]], %{{.*}} : (!cir.f16, !cir.f16, !cir.f16) -> !cir.f16 @@ -266,7 +274,8 @@ float16_t test_vfmah_laneq_f16(float16_t a, float16_t b, float16x8_t c) { return vfmah_laneq_f16(a, b, c, 7); } -// ALL-LABEL: @test_vfms_lane_f16( +// LLVM-LABEL: @test_vfms_lane_f16( +// CIR-LABEL: @test_vfms_lane_f16( float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, float16x4_t c) { // CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> @@ -278,13 +287,13 @@ float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, // CIR: cir.call_llvm_intrinsic "fma" [[NEG_CAST]], [[LANE]], %{{.*}} : (!cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>, !cir.vector<4 x !cir.f16>) -> !cir.vector<4 x !cir.f16> // LLVM-SAME: <4 x half> {{.*}} [[A:%.*]], <4 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { -// LLVM-DAG: [[FNEG:%.*]] = fneg <4 x half> [[B]] -// LLVM-DAG: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> -// LLVM-DAG: [[B_I:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> -// LLVM-DAG: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> -// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> -// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> -// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM: [[A_I:%.*]] = bitcast <4 x half> [[A]] to <4 x i16> +// LLVM-NEXT: [[FNEG:%.*]] = fneg <4 x half> [[B]] +// LLVM-NEXT: [[B_I:%.*]] = bitcast <4 x half> [[FNEG]] to <4 x i16> +// LLVM-NEXT: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> +// LLVM-NEXT: [[A_BYTES:%.*]] = bitcast <4 x i16> [[A_I]] to <8 x i8> +// LLVM-NEXT: [[B_BYTES:%.*]] = bitcast <4 x i16> [[B_I]] to <8 x i8> +// LLVM-NEXT: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> // LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> // LLVM-DAG: [[B_CAST:%.*]] = bitcast <8 x i8> [[B_BYTES]] to <4 x half> // LLVM-DAG: [[A_CAST:%.*]] = bitcast <8 x i8> [[A_BYTES]] to <4 x half> @@ -294,7 +303,8 @@ float16x4_t test_vfms_lane_f16(float16x4_t a, float16x4_t b, return vfms_lane_f16(a, b, c, 3); } -// ALL-LABEL: @test_vfmsq_lane_f16( +// LLVM-LABEL: @test_vfmsq_lane_f16( +// CIR-LABEL: @test_vfmsq_lane_f16( float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, float16x4_t c) { // CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> @@ -306,13 +316,13 @@ float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, // CIR: cir.call_llvm_intrinsic "fma" [[NEG_CAST]], [[LANE]], %{{.*}} : (!cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>, !cir.vector<8 x !cir.f16>) -> !cir.vector<8 x !cir.f16> // LLVM-SAME: <8 x half> {{.*}} [[A:%.*]], <8 x half> {{.*}} [[B:%.*]], <4 x half> {{.*}} [[C:%.*]]) {{.*}} { -// LLVM-DAG: [[FNEG:%.*]] = fneg <8 x half> [[B]] -// LLVM-DAG: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> -// LLVM-DAG: [[B_I:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> -// LLVM-DAG: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> -// LLVM-DAG: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> -// LLVM-DAG: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> -// LLVM-DAG: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> +// LLVM: [[A_I:%.*]] = bitcast <8 x half> [[A]] to <8 x i16> +// LLVM-NEXT: [[FNEG:%.*]] = fneg <8 x half> [[B]] +// LLVM-NEXT: [[B_I:%.*]] = bitcast <8 x half> [[FNEG]] to <8 x i16> +// LLVM-NEXT: [[C_I:%.*]] = bitcast <4 x half> [[C]] to <4 x i16> +// LLVM-NEXT: [[A_BYTES:%.*]] = bitcast <8 x i16> [[A_I]] to <16 x i8> +// LLVM-NEXT: [[B_BYTES:%.*]] = bitcast <8 x i16> [[B_I]] to <16 x i8> +// LLVM-NEXT: [[C_BYTES:%.*]] = bitcast <4 x i16> [[C_I]] to <8 x i8> // LLVM-DAG: [[C_CAST:%.*]] = bitcast <8 x i8> [[C_BYTES]] to <4 x half> // LLVM-DAG: [[B_CAST:%.*]] = bitcast <16 x i8> [[B_BYTES]] to <8 x half> // LLVM-DAG: [[A_CAST:%.*]] = bitcast <16 x i8> [[A_BYTES]] to <8 x half> @@ -322,7 +332,8 @@ float16x8_t test_vfmsq_lane_f16(float16x8_t a, float16x8_t b, return vfmsq_lane_f16(a, b, c, 3); } -// ALL-LABEL: @test_vfms_laneq_f16( +// LLVM-LABEL: @test_vfms_laneq_f16( +// CIR-LABEL: @test_vfms_laneq_f16( float16x4_t test_vfms_laneq_f16(float16x4_t a, float16x4_t b, float16x8_t c) { // CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> @@ -350,7 +361,8 @@ float16x4_t test_vfms_laneq_f16(float16x4_t a, float16x4_t b, return vfms_laneq_f16(a, b, c, 7); } -// ALL-LABEL: @test_vfmsq_laneq_f16( +// LLVM-LABEL: @test_vfmsq_laneq_f16( +// CIR-LABEL: @test_vfmsq_laneq_f16( float16x8_t test_vfmsq_laneq_f16(float16x8_t a, float16x8_t b, float16x8_t c) { // CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> @@ -378,7 +390,8 @@ float16x8_t test_vfmsq_laneq_f16(float16x8_t a, float16x8_t b, return vfmsq_laneq_f16(a, b, c, 7); } -// ALL-LABEL: @test_vfms_n_f16( +// LLVM-LABEL: @test_vfms_n_f16( +// CIR-LABEL: @test_vfms_n_f16( float16x4_t test_vfms_n_f16(float16x4_t a, float16x4_t b, float16_t c) { // CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<4 x !cir.f16> // CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<4 x !cir.f16> @@ -401,7 +414,8 @@ float16x4_t test_vfms_n_f16(float16x4_t a, float16x4_t b, float16_t c) { return vfms_n_f16(a, b, c); } -// ALL-LABEL: @test_vfmsq_n_f16( +// LLVM-LABEL: @test_vfmsq_n_f16( +// CIR-LABEL: @test_vfmsq_n_f16( float16x8_t test_vfmsq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { // CIR: [[FNEG:%.*]] = cir.fneg %{{.*}} : !cir.vector<8 x !cir.f16> // CIR: {{%.*}} = cir.vec.create({{.*}}) : !cir.vector<8 x !cir.f16> @@ -424,7 +438,8 @@ float16x8_t test_vfmsq_n_f16(float16x8_t a, float16x8_t b, float16_t c) { return vfmsq_n_f16(a, b, c); } -// ALL-LABEL: @test_vfmsh_lane_f16( +// LLVM-LABEL: @test_vfmsh_lane_f16( +// CIR-LABEL: @test_vfmsh_lane_f16( float16_t test_vfmsh_lane_f16(float16_t a, float16_t b, float16x4_t c) { // CIR: [[CONV:%.*]] = cir.cast floating %{{.*}} : !cir.f16 -> !cir.float // CIR: [[FNEG:%.*]] = cir.fneg [[CONV]] : !cir.float @@ -444,7 +459,8 @@ float16_t test_vfmsh_lane_f16(float16_t a, float16_t b, float16x4_t c) { return vfmsh_lane_f16(a, b, c, 3); } -// ALL-LABEL: @test_vfmsh_laneq_f16( +// LLVM-LABEL: @test_vfmsh_laneq_f16( +// CIR-LABEL: @test_vfmsh_laneq_f16( float16_t test_vfmsh_laneq_f16(float16_t a, float16_t b, float16x8_t c) { // CIR: [[CONV:%.*]] = cir.cast floating %{{.*}} : !cir.f16 -> !cir.float // CIR: [[FNEG:%.*]] = cir.fneg [[CONV]] : !cir.float _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
