https://github.com/dyung updated https://github.com/llvm/llvm-project/pull/222588
>From 21a77e7bb6bf4971cfa0428b785798a61e23b3c4 Mon Sep 17 00:00:00 2001 From: Paul Walker <[email protected]> Date: Thu, 10 Sep 2026 10:57:26 +0100 Subject: [PATCH] [LLVM][CodeGen][SME] Don't use non-streaming zeroing instruction when in streaming mode. (#222353) When selecting scalable vector bf16-f32 converts we emit zip(0,vec). Unfortunately, the typical zero-latency zeroing instruction is not available when in streaming mode. (cherry picked from commit 9679fbfea993cb2e815fe0fa5eefde310f917349) --- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 13 +- llvm/test/CodeGen/AArch64/sve-bf16-arith.ll | 395 ++++++--- .../test/CodeGen/AArch64/sve-bf16-compares.ll | 798 ++++++++++++------ .../test/CodeGen/AArch64/sve-bf16-converts.ll | 34 +- .../CodeGen/AArch64/sve-bf16-int-converts.ll | 260 ++++-- .../CodeGen/AArch64/sve-bf16-reductions.ll | 155 ++-- .../test/CodeGen/AArch64/sve-bf16-rounding.ll | 263 ++++-- 7 files changed, 1297 insertions(+), 621 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 26b0a38f6bae0..0d8fac0d95706 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -2653,9 +2653,9 @@ let Predicates = [HasSVE_or_SME] in { // A common sequence that occurs when promoting nxv8bf16 operations. def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 0)), undef)), - (ZIP1_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; + (ZIP1_ZZZ_H (DUP_ZI_H 0, 0), $Zs)>; def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 4)), undef)), - (ZIP2_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; + (ZIP2_ZZZ_H (DUP_ZI_H 0, 0), $Zs)>; // Signed integer -> Floating-point def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg), @@ -2715,6 +2715,15 @@ let Predicates = [HasSVE_or_SME] in { defm FSQRT_ZPmZ : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt", AArch64fsqrt_mt>; } // End HasSVE_or_SME +let Predicates = [HasSVE, HasNEON], AddedComplexity = 1 in { + // A common sequence that occurs when promoting nxv8bf16 operations. + // NOTE: MOVIv2d_ns is not available in streaming-mode. + def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 0)), undef)), + (ZIP1_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; + def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 4)), undef)), + (ZIP2_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; +} + let Predicates = [HasBF16, HasSVE_or_SME] in { defm BFDOT_ZZZ : sve_float_dot<0b1, 0b0, ZPR32, ZPR16, "bfdot", nxv8bf16, int_aarch64_sve_bfdot>; defm BFDOT_ZZI : sve_float_dot_indexed<0b1, 0b00, ZPR16, ZPR3b16, "bfdot", nxv8bf16, int_aarch64_sve_bfdot_lane_v2>; diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll index c916f81edf669..6337fa8ee7fe8 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-NONSTREAMING -; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,B16B16 -; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-STREAMING -; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16 +; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING,NOB16B16,NOB16B16-NONSTREAMING +; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING,B16B16 +; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING,NOB16B16,NOB16B16-STREAMING +; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING,B16B16 target triple = "aarch64-unknown-linux-gnu" @@ -83,25 +83,40 @@ define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fadd_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fadd_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fadd z2.s, z4.s, z3.s -; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fadd_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fadd z2.s, z4.s, z3.s +; NOB16B16-NONSTREAMING-NEXT: fadd z0.s, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fadd_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: bfadd z0.h, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fadd_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fadd z2.s, z4.s, z3.s +; NOB16B16-STREAMING-NEXT: fadd z0.s, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = fadd <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x bfloat> %res } @@ -137,20 +152,35 @@ define <vscale x 4 x bfloat> @fdiv_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fdiv_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fdiv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fdivr z3.s, p0/m, z3.s, z4.s -; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z3.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fdiv_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fdivr z3.s, p0/m, z3.s, z4.s +; NONSTREAMING-NEXT: fdiv z0.s, p0/m, z0.s, z1.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fdiv_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fdivr z3.s, p0/m, z3.s, z4.s +; STREAMING-NEXT: fdiv z0.s, p0/m, z0.s, z1.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = fdiv <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x bfloat> %res } @@ -198,26 +228,41 @@ define <vscale x 4 x bfloat> @fmax_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fmax_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fmax_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fmax z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fmax z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmax_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fmax z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fmax z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmax_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmax z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmax_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fmax z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fmax z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.maximum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -265,26 +310,41 @@ define <vscale x 4 x bfloat> @fmaxnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale } define <vscale x 8 x bfloat> @fmaxnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fmaxnm_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmaxnm_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmaxnm_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmaxnm z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmaxnm_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.maxnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -332,26 +392,41 @@ define <vscale x 4 x bfloat> @fmin_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fmin_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fmin_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fmin z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fmin z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmin_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fmin z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fmin z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmin_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmin z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmin_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fmin z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fmin z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.minimum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -399,26 +474,41 @@ define <vscale x 4 x bfloat> @fminnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale } define <vscale x 8 x bfloat> @fminnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fminnm_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fminnm z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fminnm z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fminnm_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fminnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fminnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fminnm_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfminnm z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fminnm_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fminnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fminnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.minnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -466,28 +556,45 @@ define <vscale x 4 x bfloat> @fmla_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) { -; NOB16B16-LABEL: fmla_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v3.2d, #0000000000000000 -; NOB16B16-NEXT: uunpkhi z4.s, z1.h -; NOB16B16-NEXT: uunpkhi z5.s, z0.h -; NOB16B16-NEXT: uunpklo z1.s, z1.h -; NOB16B16-NEXT: uunpklo z0.s, z0.h -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z6.h, z3.h, z2.h -; NOB16B16-NEXT: zip1 z2.h, z3.h, z2.h -; NOB16B16-NEXT: bfmlalb z6.s, z5.h, z4.h -; NOB16B16-NEXT: bfmlalb z2.s, z0.h, z1.h -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z6.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s -; NOB16B16-NEXT: uzp1 z0.h, z1.h, z0.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmla_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v3.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: uunpkhi z4.s, z1.h +; NOB16B16-NONSTREAMING-NEXT: uunpkhi z5.s, z0.h +; NOB16B16-NONSTREAMING-NEXT: uunpklo z1.s, z1.h +; NOB16B16-NONSTREAMING-NEXT: uunpklo z0.s, z0.h +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z6.h, z3.h, z2.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z2.h, z3.h, z2.h +; NOB16B16-NONSTREAMING-NEXT: bfmlalb z6.s, z5.h, z4.h +; NOB16B16-NONSTREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z6.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z1.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmla_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmla_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z3.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: uunpkhi z4.s, z1.h +; NOB16B16-STREAMING-NEXT: uunpkhi z5.s, z0.h +; NOB16B16-STREAMING-NEXT: uunpklo z1.s, z1.h +; NOB16B16-STREAMING-NEXT: uunpklo z0.s, z0.h +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z6.h, z3.h, z2.h +; NOB16B16-STREAMING-NEXT: zip1 z2.h, z3.h, z2.h +; NOB16B16-STREAMING-NEXT: bfmlalb z6.s, z5.h, z4.h +; NOB16B16-STREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z6.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z1.h, z0.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.fma.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) ret <vscale x 8 x bfloat> %res } @@ -655,18 +762,31 @@ define <vscale x 4 x bfloat> @fsqrt_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @fsqrt_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fsqrt_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fsqrt z2.s, p0/m, z2.s -; CHECK-NEXT: fsqrt z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fsqrt_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: fsqrt z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: fsqrt z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fsqrt_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: fsqrt z2.s, p0/m, z2.s +; STREAMING-NEXT: fsqrt z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.sqrt.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -714,25 +834,40 @@ define <vscale x 4 x bfloat> @fsub_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fsub_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fsub_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fsub z2.s, z4.s, z3.s -; NOB16B16-NEXT: fsub z0.s, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fsub_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fsub z2.s, z4.s, z3.s +; NOB16B16-NONSTREAMING-NEXT: fsub z0.s, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fsub_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: bfsub z0.h, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fsub_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fsub z2.s, z4.s, z3.s +; NOB16B16-STREAMING-NEXT: fsub z0.s, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = fsub <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x bfloat> %res } diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll b/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll index c2f74068b17d6..10e76c358a4a7 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub -; RUN: llc -mattr=+sve < %s | FileCheck %s -; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s +; RUN: llc -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING +; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING target triple = "aarch64-unknown-linux-gnu" @@ -33,18 +33,31 @@ define <vscale x 4 x i1> @fcmp_oeq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_oeq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_oeq_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_oeq_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_oeq_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp oeq <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -78,18 +91,31 @@ define <vscale x 4 x i1> @fcmp_ogt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ogt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ogt_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ogt_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ogt_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ogt <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -123,18 +149,31 @@ define <vscale x 4 x i1> @fcmp_oge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_oge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_oge_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_oge_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmge p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_oge_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmge p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp oge <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -168,18 +207,31 @@ define <vscale x 4 x i1> @fcmp_olt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_olt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_olt_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_olt_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_olt_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp olt <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -213,18 +265,31 @@ define <vscale x 4 x i1> @fcmp_ole_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ole_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ole_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ole_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmge p0.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ole_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmge p0.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ole <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -262,22 +327,39 @@ define <vscale x 4 x i1> @fcmp_one_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_one_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_one_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p2.s, p0/z, z3.s, z4.s -; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z0.s -; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: mov p1.b, p2/m, p2.b -; CHECK-NEXT: sel p0.b, p0, p0.b, p3.b -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_one_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p2.s, p0/z, z3.s, z4.s +; NONSTREAMING-NEXT: fcmgt p3.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: mov p1.b, p2/m, p2.b +; NONSTREAMING-NEXT: sel p0.b, p0, p0.b, p3.b +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_one_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p2.s, p0/z, z3.s, z4.s +; STREAMING-NEXT: fcmgt p3.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: mov p1.b, p2/m, p2.b +; STREAMING-NEXT: sel p0.b, p0, p0.b, p3.b +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp one <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -313,20 +395,35 @@ define <vscale x 4 x i1> @fcmp_ord_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ord_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ord_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmuo p2.s, p0/z, z0.s, z1.s -; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ord_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmuo p2.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: not p1.b, p0/z, p1.b +; NONSTREAMING-NEXT: not p2.b, p0/z, p2.b +; NONSTREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ord_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmuo p2.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: not p1.b, p0/z, p1.b +; STREAMING-NEXT: not p2.b, p0/z, p2.b +; STREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ord <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -364,22 +461,39 @@ define <vscale x 4 x i1> @fcmp_ueq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ueq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ueq_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmeq p2.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmuo p3.s, p0/z, z0.s, z1.s -; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: mov p1.b, p2/m, p2.b -; CHECK-NEXT: sel p0.b, p0, p0.b, p3.b -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ueq_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmeq p2.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmuo p3.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: mov p1.b, p2/m, p2.b +; NONSTREAMING-NEXT: sel p0.b, p0, p0.b, p3.b +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ueq_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmeq p2.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmuo p3.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: mov p1.b, p2/m, p2.b +; STREAMING-NEXT: sel p0.b, p0, p0.b, p3.b +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ueq <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -415,20 +529,35 @@ define <vscale x 4 x i1> @fcmp_ugt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ugt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ugt_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z0.s -; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ugt_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmge p2.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: not p1.b, p0/z, p1.b +; NONSTREAMING-NEXT: not p2.b, p0/z, p2.b +; NONSTREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ugt_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmge p2.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: not p1.b, p0/z, p1.b +; STREAMING-NEXT: not p2.b, p0/z, p2.b +; STREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ugt <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -464,20 +593,35 @@ define <vscale x 4 x i1> @fcmp_uge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_uge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_uge_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p2.s, p0/z, z1.s, z0.s -; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_uge_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p2.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: not p1.b, p0/z, p1.b +; NONSTREAMING-NEXT: not p2.b, p0/z, p2.b +; NONSTREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_uge_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p2.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: not p1.b, p0/z, p1.b +; STREAMING-NEXT: not p2.b, p0/z, p2.b +; STREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; STREAMING-NEXT: ret %res = fcmp uge <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -513,20 +657,35 @@ define <vscale x 4 x i1> @fcmp_ult_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ult_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ult_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z1.s -; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ult_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmge p2.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: not p1.b, p0/z, p1.b +; NONSTREAMING-NEXT: not p2.b, p0/z, p2.b +; NONSTREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ult_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmge p2.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: not p1.b, p0/z, p1.b +; STREAMING-NEXT: not p2.b, p0/z, p2.b +; STREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ult <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -562,20 +721,35 @@ define <vscale x 4 x i1> @fcmp_ule_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_ule_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ule_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p2.s, p0/z, z0.s, z1.s -; CHECK-NEXT: not p1.b, p0/z, p1.b -; CHECK-NEXT: not p2.b, p0/z, p2.b -; CHECK-NEXT: uzp1 p0.h, p2.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ule_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p2.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: not p1.b, p0/z, p1.b +; NONSTREAMING-NEXT: not p2.b, p0/z, p2.b +; NONSTREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ule_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p2.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: not p1.b, p0/z, p1.b +; STREAMING-NEXT: not p2.b, p0/z, p2.b +; STREAMING-NEXT: uzp1 p0.h, p2.h, p1.h +; STREAMING-NEXT: ret %res = fcmp ule <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -609,18 +783,31 @@ define <vscale x 4 x i1> @fcmp_une_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_une_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_une_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmne p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_une_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmne p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmne p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_une_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmne p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmne p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp une <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -654,18 +841,31 @@ define <vscale x 4 x i1> @fcmp_uno_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x i1> @fcmp_uno_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_uno_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_uno_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_uno_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmuo p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp uno <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -699,18 +899,31 @@ define <vscale x 4 x i1> @fcmp_eq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 } define <vscale x 8 x i1> @fcmp_eq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_eq_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_eq_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_eq_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmeq p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp fast oeq <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -744,18 +957,31 @@ define <vscale x 4 x i1> @fcmp_gt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 } define <vscale x 8 x i1> @fcmp_gt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_gt_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_gt_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_gt_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp fast ogt <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -789,18 +1015,31 @@ define <vscale x 4 x i1> @fcmp_ge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 } define <vscale x 8 x i1> @fcmp_ge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ge_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ge_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmge p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ge_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmge p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp fast oge <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -834,18 +1073,31 @@ define <vscale x 4 x i1> @fcmp_lt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 } define <vscale x 8 x i1> @fcmp_lt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_lt_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_lt_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_lt_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmgt p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp fast olt <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -879,18 +1131,31 @@ define <vscale x 4 x i1> @fcmp_le_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 } define <vscale x 8 x i1> @fcmp_le_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_le_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z0.h -; CHECK-NEXT: zip2 z4.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: fcmge p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_le_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmge p0.s, p0/z, z1.s, z0.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_le_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z0.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: fcmge p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmge p0.s, p0/z, z1.s, z0.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp fast ole <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } @@ -924,18 +1189,31 @@ define <vscale x 4 x i1> @fcmp_ne_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 } define <vscale x 8 x i1> @fcmp_ne_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fcmp_ne_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fcmne p1.s, p0/z, z4.s, z3.s -; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s -; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fcmp_ne_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fcmne p1.s, p0/z, z4.s, z3.s +; NONSTREAMING-NEXT: fcmne p0.s, p0/z, z0.s, z1.s +; NONSTREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fcmp_ne_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fcmne p1.s, p0/z, z4.s, z3.s +; STREAMING-NEXT: fcmne p0.s, p0/z, z0.s, z1.s +; STREAMING-NEXT: uzp1 p0.h, p0.h, p1.h +; STREAMING-NEXT: ret %res = fcmp fast one <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x i1> %res } diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll b/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll index abe6c13fb6c63..b71b762ee4d60 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-converts.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 ; RUN: llc -mattr=+sve < %s | FileCheck %s --check-prefixes=CHECK,NOBF16 -; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,BF16 -; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,BF16 +; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,BF16,BF16-NONSTREAMING +; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,BF16,BF16-STREAMING target triple = "aarch64-unknown-linux-gnu" @@ -28,13 +28,29 @@ define <vscale x 4 x float> @fpext_nxv4bf16_to_nxv4f32(<vscale x 4 x bfloat> %a) } define <vscale x 8 x float> @fpext_nxv8bf16_to_nxv8f32(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fpext_nxv8bf16_to_nxv8f32: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: zip1 z2.h, z1.h, z0.h -; CHECK-NEXT: zip2 z1.h, z1.h, z0.h -; CHECK-NEXT: mov z0.d, z2.d -; CHECK-NEXT: ret +; NOBF16-LABEL: fpext_nxv8bf16_to_nxv8f32: +; NOBF16: // %bb.0: +; NOBF16-NEXT: movi v1.2d, #0000000000000000 +; NOBF16-NEXT: zip1 z2.h, z1.h, z0.h +; NOBF16-NEXT: zip2 z1.h, z1.h, z0.h +; NOBF16-NEXT: mov z0.d, z2.d +; NOBF16-NEXT: ret +; +; BF16-NONSTREAMING-LABEL: fpext_nxv8bf16_to_nxv8f32: +; BF16-NONSTREAMING: // %bb.0: +; BF16-NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; BF16-NONSTREAMING-NEXT: zip1 z2.h, z1.h, z0.h +; BF16-NONSTREAMING-NEXT: zip2 z1.h, z1.h, z0.h +; BF16-NONSTREAMING-NEXT: mov z0.d, z2.d +; BF16-NONSTREAMING-NEXT: ret +; +; BF16-STREAMING-LABEL: fpext_nxv8bf16_to_nxv8f32: +; BF16-STREAMING: // %bb.0: +; BF16-STREAMING-NEXT: mov z1.h, #0 // =0x0 +; BF16-STREAMING-NEXT: zip1 z2.h, z1.h, z0.h +; BF16-STREAMING-NEXT: zip2 z1.h, z1.h, z0.h +; BF16-STREAMING-NEXT: mov z0.d, z2.d +; BF16-STREAMING-NEXT: ret %res = fpext <vscale x 8 x bfloat> %a to <vscale x 8 x float> ret <vscale x 8 x float> %res } diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll b/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll index 3e7f8424c1334..0d4be4a2fcfd5 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll @@ -123,63 +123,109 @@ define <vscale x 4 x i64> @fptosi_nxv4bf16_to_nxv4i64(<vscale x 4 x bfloat> %a) } define <vscale x 8 x i1> @fptosi_nxv8bf16_to_nxv8i1(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i1: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s -; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s -; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h -; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 -; CHECK-NEXT: ret +; SVE-LABEL: fptosi_nxv8bf16_to_nxv8i1: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; SVE-NEXT: ptrue p0.h +; SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; SVE-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptosi_nxv8bf16_to_nxv8i1: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; STREAMING-SVE-NEXT: ptrue p0.h +; STREAMING-SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; STREAMING-SVE-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; STREAMING-SVE-NEXT: ret %res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i1> ret <vscale x 8 x i1> %res } define <vscale x 8 x i8> @fptosi_nxv8bf16_to_nxv8i8(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i8: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s -; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h -; CHECK-NEXT: ret +; SVE-LABEL: fptosi_nxv8bf16_to_nxv8i8: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptosi_nxv8bf16_to_nxv8i8: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; STREAMING-SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; STREAMING-SVE-NEXT: ret %res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i8> ret <vscale x 8 x i8> %res } define <vscale x 8 x i16> @fptosi_nxv8bf16_to_nxv8i16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s -; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h -; CHECK-NEXT: ret +; SVE-LABEL: fptosi_nxv8bf16_to_nxv8i16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptosi_nxv8bf16_to_nxv8i16: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; STREAMING-SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; STREAMING-SVE-NEXT: ret %res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i16> ret <vscale x 8 x i16> %res } define <vscale x 8 x i32> @fptosi_nxv8bf16_to_nxv8i32(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptosi_nxv8bf16_to_nxv8i32: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip1 z2.h, z1.h, z0.h -; CHECK-NEXT: zip2 z1.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z1.s, p0/m, z1.s -; CHECK-NEXT: movprfx z0, z2 -; CHECK-NEXT: fcvtzs z0.s, p0/m, z2.s -; CHECK-NEXT: ret +; SVE-LABEL: fptosi_nxv8bf16_to_nxv8i32: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip1 z2.h, z1.h, z0.h +; SVE-NEXT: zip2 z1.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z1.s, p0/m, z1.s +; SVE-NEXT: movprfx z0, z2 +; SVE-NEXT: fcvtzs z0.s, p0/m, z2.s +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptosi_nxv8bf16_to_nxv8i32: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip1 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip2 z1.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z1.s, p0/m, z1.s +; STREAMING-SVE-NEXT: movprfx z0, z2 +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z2.s +; STREAMING-SVE-NEXT: ret %res = fptosi <vscale x 8 x bfloat> %a to <vscale x 8 x i32> ret <vscale x 8 x i32> %res } @@ -331,63 +377,109 @@ define <vscale x 4 x i64> @fptoui_nxv4bf16_to_nxv4i64(<vscale x 4 x bfloat> %a) } define <vscale x 8 x i1> @fptoui_nxv8bf16_to_nxv8i1(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i1: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s -; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s -; CHECK-NEXT: ptrue p0.h -; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h -; CHECK-NEXT: cmpne p0.h, p0/z, z0.h, #0 -; CHECK-NEXT: ret +; SVE-LABEL: fptoui_nxv8bf16_to_nxv8i1: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; SVE-NEXT: ptrue p0.h +; SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; SVE-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptoui_nxv8bf16_to_nxv8i1: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; STREAMING-SVE-NEXT: ptrue p0.h +; STREAMING-SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; STREAMING-SVE-NEXT: cmpne p0.h, p0/z, z0.h, #0 +; STREAMING-SVE-NEXT: ret %res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i1> ret <vscale x 8 x i1> %res } define <vscale x 8 x i8> @fptoui_nxv8bf16_to_nxv8i8(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i8: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s -; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h -; CHECK-NEXT: ret +; SVE-LABEL: fptoui_nxv8bf16_to_nxv8i8: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptoui_nxv8bf16_to_nxv8i8: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; STREAMING-SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; STREAMING-SVE-NEXT: ret %res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i8> ret <vscale x 8 x i8> %res } define <vscale x 8 x i16> @fptoui_nxv8bf16_to_nxv8i16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fcvtzs z2.s, p0/m, z2.s -; CHECK-NEXT: fcvtzs z0.s, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z2.h -; CHECK-NEXT: ret +; SVE-LABEL: fptoui_nxv8bf16_to_nxv8i16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptoui_nxv8bf16_to_nxv8i16: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzs z2.s, p0/m, z2.s +; STREAMING-SVE-NEXT: fcvtzs z0.s, p0/m, z0.s +; STREAMING-SVE-NEXT: uzp1 z0.h, z0.h, z2.h +; STREAMING-SVE-NEXT: ret %res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i16> ret <vscale x 8 x i16> %res } define <vscale x 8 x i32> @fptoui_nxv8bf16_to_nxv8i32(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fptoui_nxv8bf16_to_nxv8i32: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip1 z2.h, z1.h, z0.h -; CHECK-NEXT: zip2 z1.h, z1.h, z0.h -; CHECK-NEXT: fcvtzu z1.s, p0/m, z1.s -; CHECK-NEXT: movprfx z0, z2 -; CHECK-NEXT: fcvtzu z0.s, p0/m, z2.s -; CHECK-NEXT: ret +; SVE-LABEL: fptoui_nxv8bf16_to_nxv8i32: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip1 z2.h, z1.h, z0.h +; SVE-NEXT: zip2 z1.h, z1.h, z0.h +; SVE-NEXT: fcvtzu z1.s, p0/m, z1.s +; SVE-NEXT: movprfx z0, z2 +; SVE-NEXT: fcvtzu z0.s, p0/m, z2.s +; SVE-NEXT: ret +; +; STREAMING-SVE-LABEL: fptoui_nxv8bf16_to_nxv8i32: +; STREAMING-SVE: // %bb.0: +; STREAMING-SVE-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-SVE-NEXT: ptrue p0.s +; STREAMING-SVE-NEXT: zip1 z2.h, z1.h, z0.h +; STREAMING-SVE-NEXT: zip2 z1.h, z1.h, z0.h +; STREAMING-SVE-NEXT: fcvtzu z1.s, p0/m, z1.s +; STREAMING-SVE-NEXT: movprfx z0, z2 +; STREAMING-SVE-NEXT: fcvtzu z0.s, p0/m, z2.s +; STREAMING-SVE-NEXT: ret %res = fptoui <vscale x 8 x bfloat> %a to <vscale x 8 x i32> ret <vscale x 8 x i32> %res } diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll b/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll index a888238503e23..6ab8f8e57dfcf 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll @@ -31,16 +31,27 @@ define bfloat @faddv_nxv4bf16(<vscale x 4 x bfloat> %a) { } define bfloat @faddv_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: faddv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fadd z0.s, z0.s, z2.s -; CHECK-NEXT: faddv s0, p0, z0.s -; CHECK-NEXT: bfcvt h0, s0 -; CHECK-NEXT: ret +; SVE-LABEL: faddv_nxv8bf16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fadd z0.s, z0.s, z2.s +; SVE-NEXT: faddv s0, p0, z0.s +; SVE-NEXT: bfcvt h0, s0 +; SVE-NEXT: ret +; +; SME-LABEL: faddv_nxv8bf16: +; SME: // %bb.0: +; SME-NEXT: mov z1.h, #0 // =0x0 +; SME-NEXT: ptrue p0.s +; SME-NEXT: zip2 z2.h, z1.h, z0.h +; SME-NEXT: zip1 z0.h, z1.h, z0.h +; SME-NEXT: fadd z0.s, z0.s, z2.s +; SME-NEXT: faddv s0, p0, z0.s +; SME-NEXT: bfcvt h0, s0 +; SME-NEXT: ret %res = call fast bfloat @llvm.vector.reduce.fadd.nxv8bf16(bfloat zeroinitializer, <vscale x 8 x bfloat> %a) ret bfloat %res } @@ -72,16 +83,27 @@ define bfloat @fmaxv_nxv4bf16(<vscale x 4 x bfloat> %a) { } define bfloat @fmaxv_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fmaxv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fmaxnm z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: fmaxnmv s0, p0, z0.s -; CHECK-NEXT: bfcvt h0, s0 -; CHECK-NEXT: ret +; SVE-LABEL: fmaxv_nxv8bf16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fmaxnm z0.s, p0/m, z0.s, z2.s +; SVE-NEXT: fmaxnmv s0, p0, z0.s +; SVE-NEXT: bfcvt h0, s0 +; SVE-NEXT: ret +; +; SME-LABEL: fmaxv_nxv8bf16: +; SME: // %bb.0: +; SME-NEXT: mov z1.h, #0 // =0x0 +; SME-NEXT: ptrue p0.s +; SME-NEXT: zip2 z2.h, z1.h, z0.h +; SME-NEXT: zip1 z0.h, z1.h, z0.h +; SME-NEXT: fmaxnm z0.s, p0/m, z0.s, z2.s +; SME-NEXT: fmaxnmv s0, p0, z0.s +; SME-NEXT: bfcvt h0, s0 +; SME-NEXT: ret %res = call bfloat @llvm.vector.reduce.fmax.nxv8bf16(<vscale x 8 x bfloat> %a) ret bfloat %res } @@ -113,16 +135,27 @@ define bfloat @fminv_nxv4bf16(<vscale x 4 x bfloat> %a) { } define bfloat @fminv_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fminv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fminnm z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: fminnmv s0, p0, z0.s -; CHECK-NEXT: bfcvt h0, s0 -; CHECK-NEXT: ret +; SVE-LABEL: fminv_nxv8bf16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fminnm z0.s, p0/m, z0.s, z2.s +; SVE-NEXT: fminnmv s0, p0, z0.s +; SVE-NEXT: bfcvt h0, s0 +; SVE-NEXT: ret +; +; SME-LABEL: fminv_nxv8bf16: +; SME: // %bb.0: +; SME-NEXT: mov z1.h, #0 // =0x0 +; SME-NEXT: ptrue p0.s +; SME-NEXT: zip2 z2.h, z1.h, z0.h +; SME-NEXT: zip1 z0.h, z1.h, z0.h +; SME-NEXT: fminnm z0.s, p0/m, z0.s, z2.s +; SME-NEXT: fminnmv s0, p0, z0.s +; SME-NEXT: bfcvt h0, s0 +; SME-NEXT: ret %res = call bfloat @llvm.vector.reduce.fmin.nxv8bf16(<vscale x 8 x bfloat> %a) ret bfloat %res } @@ -154,16 +187,27 @@ define bfloat @fmaximumv_nxv4bf16(<vscale x 4 x bfloat> %a) { } define bfloat @fmaximumv_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fmaximumv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fmax z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: fmaxv s0, p0, z0.s -; CHECK-NEXT: bfcvt h0, s0 -; CHECK-NEXT: ret +; SVE-LABEL: fmaximumv_nxv8bf16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fmax z0.s, p0/m, z0.s, z2.s +; SVE-NEXT: fmaxv s0, p0, z0.s +; SVE-NEXT: bfcvt h0, s0 +; SVE-NEXT: ret +; +; SME-LABEL: fmaximumv_nxv8bf16: +; SME: // %bb.0: +; SME-NEXT: mov z1.h, #0 // =0x0 +; SME-NEXT: ptrue p0.s +; SME-NEXT: zip2 z2.h, z1.h, z0.h +; SME-NEXT: zip1 z0.h, z1.h, z0.h +; SME-NEXT: fmax z0.s, p0/m, z0.s, z2.s +; SME-NEXT: fmaxv s0, p0, z0.s +; SME-NEXT: bfcvt h0, s0 +; SME-NEXT: ret %res = call bfloat @llvm.vector.reduce.fmaximum.nxv8bf16(<vscale x 8 x bfloat> %a) ret bfloat %res } @@ -195,16 +239,27 @@ define bfloat @fminimumv_nxv4bf16(<vscale x 4 x bfloat> %a) { } define bfloat @fminimumv_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: fminimumv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: fmin z0.s, p0/m, z0.s, z2.s -; CHECK-NEXT: fminv s0, p0, z0.s -; CHECK-NEXT: bfcvt h0, s0 -; CHECK-NEXT: ret +; SVE-LABEL: fminimumv_nxv8bf16: +; SVE: // %bb.0: +; SVE-NEXT: movi v1.2d, #0000000000000000 +; SVE-NEXT: ptrue p0.s +; SVE-NEXT: zip2 z2.h, z1.h, z0.h +; SVE-NEXT: zip1 z0.h, z1.h, z0.h +; SVE-NEXT: fmin z0.s, p0/m, z0.s, z2.s +; SVE-NEXT: fminv s0, p0, z0.s +; SVE-NEXT: bfcvt h0, s0 +; SVE-NEXT: ret +; +; SME-LABEL: fminimumv_nxv8bf16: +; SME: // %bb.0: +; SME-NEXT: mov z1.h, #0 // =0x0 +; SME-NEXT: ptrue p0.s +; SME-NEXT: zip2 z2.h, z1.h, z0.h +; SME-NEXT: zip1 z0.h, z1.h, z0.h +; SME-NEXT: fmin z0.s, p0/m, z0.s, z2.s +; SME-NEXT: fminv s0, p0, z0.s +; SME-NEXT: bfcvt h0, s0 +; SME-NEXT: ret %res = call bfloat @llvm.vector.reduce.fminimum.nxv8bf16(<vscale x 8 x bfloat> %a) ret bfloat %res } diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll b/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll index 80b54ff31aaf0..51b8929c1f854 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub -; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s -; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s +; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING +; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING target triple = "aarch64-unknown-linux-gnu" @@ -33,18 +33,31 @@ define <vscale x 4 x bfloat> @frintp_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frintp_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frintp_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frintp z2.s, p0/m, z2.s -; CHECK-NEXT: frintp z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frintp_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frintp z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frintp z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frintp_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frintp z2.s, p0/m, z2.s +; STREAMING-NEXT: frintp z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.ceil.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -78,18 +91,31 @@ define <vscale x 4 x bfloat> @frintm_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frintm_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frintm_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frintm z2.s, p0/m, z2.s -; CHECK-NEXT: frintm z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frintm_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frintm z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frintm z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frintm_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frintm z2.s, p0/m, z2.s +; STREAMING-NEXT: frintm z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.floor.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -123,18 +149,31 @@ define <vscale x 4 x bfloat> @frinti_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frinti_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frinti_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frinti z2.s, p0/m, z2.s -; CHECK-NEXT: frinti z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frinti_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frinti z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frinti z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frinti_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frinti z2.s, p0/m, z2.s +; STREAMING-NEXT: frinti z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.nearbyint.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -168,18 +207,31 @@ define <vscale x 4 x bfloat> @frintx_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frintx_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frintx_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frintx z2.s, p0/m, z2.s -; CHECK-NEXT: frintx z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frintx_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frintx z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frintx z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frintx_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frintx z2.s, p0/m, z2.s +; STREAMING-NEXT: frintx z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.rint.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -213,18 +265,31 @@ define <vscale x 4 x bfloat> @frinta_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frinta_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frinta_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frinta z2.s, p0/m, z2.s -; CHECK-NEXT: frinta z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frinta_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frinta z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frinta z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frinta_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frinta z2.s, p0/m, z2.s +; STREAMING-NEXT: frinta z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.round.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -258,18 +323,31 @@ define <vscale x 4 x bfloat> @frintn_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frintn_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frintn_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frintn z2.s, p0/m, z2.s -; CHECK-NEXT: frintn z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frintn_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frintn z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frintn z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frintn_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frintn z2.s, p0/m, z2.s +; STREAMING-NEXT: frintn z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.roundeven.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } @@ -303,18 +381,31 @@ define <vscale x 4 x bfloat> @frintz_nxv4bf16(<vscale x 4 x bfloat> %a) { } define <vscale x 8 x bfloat> @frintz_nxv8bf16(<vscale x 8 x bfloat> %a) { -; CHECK-LABEL: frintz_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z2.h, z1.h, z0.h -; CHECK-NEXT: zip1 z0.h, z1.h, z0.h -; CHECK-NEXT: frintz z2.s, p0/m, z2.s -; CHECK-NEXT: frintz z0.s, p0/m, z0.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z2.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: frintz_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v1.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; NONSTREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; NONSTREAMING-NEXT: frintz z2.s, p0/m, z2.s +; NONSTREAMING-NEXT: frintz z0.s, p0/m, z0.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: frintz_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z1.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z2.h, z1.h, z0.h +; STREAMING-NEXT: zip1 z0.h, z1.h, z0.h +; STREAMING-NEXT: frintz z2.s, p0/m, z2.s +; STREAMING-NEXT: frintz z0.s, p0/m, z0.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.trunc.nxv8bf16(<vscale x 8 x bfloat> %a) ret <vscale x 8 x bfloat> %res } _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
