llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT--> @llvm/pr-subscribers-backend-aarch64 Author: llvmbot <details> <summary>Changes</summary> Backport 9679fbfea993cb2e815fe0fa5eefde310f917349 Requested by: @<!-- -->paulwalker-arm --- Patch is 95.16 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/222588.diff 7 Files Affected: - (modified) llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td (+11-2) - (modified) llvm/test/CodeGen/AArch64/sve-bf16-arith.ll (+265-130) - (modified) llvm/test/CodeGen/AArch64/sve-bf16-compares.ll (+538-260) - (modified) llvm/test/CodeGen/AArch64/sve-bf16-converts.ll (+25-9) - (modified) llvm/test/CodeGen/AArch64/sve-bf16-int-converts.ll (+176-84) - (modified) llvm/test/CodeGen/AArch64/sve-bf16-reductions.ll (+105-50) - (modified) llvm/test/CodeGen/AArch64/sve-bf16-rounding.ll (+177-86) ``````````diff diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 26b0a38f6bae0..0d8fac0d95706 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -2653,9 +2653,9 @@ let Predicates = [HasSVE_or_SME] in { // A common sequence that occurs when promoting nxv8bf16 operations. def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 0)), undef)), - (ZIP1_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; + (ZIP1_ZZZ_H (DUP_ZI_H 0, 0), $Zs)>; def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 4)), undef)), - (ZIP2_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; + (ZIP2_ZZZ_H (DUP_ZI_H 0, 0), $Zs)>; // Signed integer -> Floating-point def : Pat<(nxv2f16 (AArch64scvtf_mt (nxv2i1 (SVEAllActive):$Pg), @@ -2715,6 +2715,15 @@ let Predicates = [HasSVE_or_SME] in { defm FSQRT_ZPmZ : sve_fp_2op_p_zd_HSD<0b01101, "fsqrt", AArch64fsqrt_mt>; } // End HasSVE_or_SME +let Predicates = [HasSVE, HasNEON], AddedComplexity = 1 in { + // A common sequence that occurs when promoting nxv8bf16 operations. + // NOTE: MOVIv2d_ns is not available in streaming-mode. + def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 0)), undef)), + (ZIP1_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; + def : Pat<(nxv4f32 (AArch64fcvte_mt (SVEAnyPredicate), (extract_subvector nxv8bf16:$Zs, (i64 4)), undef)), + (ZIP2_ZZZ_H (SUBREG_TO_REG (MOVIv2d_ns (i32 0)), zsub), $Zs)>; +} + let Predicates = [HasBF16, HasSVE_or_SME] in { defm BFDOT_ZZZ : sve_float_dot<0b1, 0b0, ZPR32, ZPR16, "bfdot", nxv8bf16, int_aarch64_sve_bfdot>; defm BFDOT_ZZI : sve_float_dot_indexed<0b1, 0b00, ZPR16, ZPR3b16, "bfdot", nxv8bf16, int_aarch64_sve_bfdot_lane_v2>; diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll index c916f81edf669..6337fa8ee7fe8 100644 --- a/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll +++ b/llvm/test/CodeGen/AArch64/sve-bf16-arith.ll @@ -1,8 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5 -; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-NONSTREAMING -; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,B16B16 -; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,NOB16B16,NOB16B16-STREAMING -; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,B16B16 +; RUN: llc -mattr=+sve,+bf16 < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING,NOB16B16,NOB16B16-NONSTREAMING +; RUN: llc -mattr=+sve,+bf16,+sve-b16b16 < %s | FileCheck %s --check-prefixes=CHECK,NONSTREAMING,B16B16 +; RUN: llc -mattr=+sme,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING,NOB16B16,NOB16B16-STREAMING +; RUN: llc -mattr=+sme2,+sve-b16b16 -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,STREAMING,B16B16 target triple = "aarch64-unknown-linux-gnu" @@ -83,25 +83,40 @@ define <vscale x 4 x bfloat> @fadd_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fadd_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fadd_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fadd z2.s, z4.s, z3.s -; NOB16B16-NEXT: fadd z0.s, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fadd_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fadd z2.s, z4.s, z3.s +; NOB16B16-NONSTREAMING-NEXT: fadd z0.s, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fadd_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: bfadd z0.h, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fadd_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fadd z2.s, z4.s, z3.s +; NOB16B16-STREAMING-NEXT: fadd z0.s, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = fadd <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x bfloat> %res } @@ -137,20 +152,35 @@ define <vscale x 4 x bfloat> @fdiv_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fdiv_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; CHECK-LABEL: fdiv_nxv8bf16: -; CHECK: // %bb.0: -; CHECK-NEXT: movi v2.2d, #0000000000000000 -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: zip2 z3.h, z2.h, z1.h -; CHECK-NEXT: zip2 z4.h, z2.h, z0.h -; CHECK-NEXT: zip1 z1.h, z2.h, z1.h -; CHECK-NEXT: zip1 z0.h, z2.h, z0.h -; CHECK-NEXT: fdivr z3.s, p0/m, z3.s, z4.s -; CHECK-NEXT: fdiv z0.s, p0/m, z0.s, z1.s -; CHECK-NEXT: bfcvt z1.h, p0/m, z3.s -; CHECK-NEXT: bfcvt z0.h, p0/m, z0.s -; CHECK-NEXT: uzp1 z0.h, z0.h, z1.h -; CHECK-NEXT: ret +; NONSTREAMING-LABEL: fdiv_nxv8bf16: +; NONSTREAMING: // %bb.0: +; NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NONSTREAMING-NEXT: ptrue p0.s +; NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NONSTREAMING-NEXT: fdivr z3.s, p0/m, z3.s, z4.s +; NONSTREAMING-NEXT: fdiv z0.s, p0/m, z0.s, z1.s +; NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NONSTREAMING-NEXT: ret +; +; STREAMING-LABEL: fdiv_nxv8bf16: +; STREAMING: // %bb.0: +; STREAMING-NEXT: mov z2.h, #0 // =0x0 +; STREAMING-NEXT: ptrue p0.s +; STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; STREAMING-NEXT: fdivr z3.s, p0/m, z3.s, z4.s +; STREAMING-NEXT: fdiv z0.s, p0/m, z0.s, z1.s +; STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; STREAMING-NEXT: ret %res = fdiv <vscale x 8 x bfloat> %a, %b ret <vscale x 8 x bfloat> %res } @@ -198,26 +228,41 @@ define <vscale x 4 x bfloat> @fmax_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fmax_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fmax_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fmax z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fmax z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmax_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fmax z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fmax z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmax_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmax z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmax_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fmax z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fmax z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.maximum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -265,26 +310,41 @@ define <vscale x 4 x bfloat> @fmaxnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale } define <vscale x 8 x bfloat> @fmaxnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fmaxnm_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmaxnm_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmaxnm_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmaxnm z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmaxnm_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fmaxnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fmaxnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.maxnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -332,26 +392,41 @@ define <vscale x 4 x bfloat> @fmin_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fmin_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fmin_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fmin z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fmin z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmin_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fmin z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fmin z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmin_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmin z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmin_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fmin z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fmin z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.minimum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -399,26 +474,41 @@ define <vscale x 4 x bfloat> @fminnm_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale } define <vscale x 8 x bfloat> @fminnm_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) { -; NOB16B16-LABEL: fminnm_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v2.2d, #0000000000000000 -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z3.h, z2.h, z1.h -; NOB16B16-NEXT: zip2 z4.h, z2.h, z0.h -; NOB16B16-NEXT: zip1 z1.h, z2.h, z1.h -; NOB16B16-NEXT: zip1 z0.h, z2.h, z0.h -; NOB16B16-NEXT: fminnm z3.s, p0/m, z3.s, z4.s -; NOB16B16-NEXT: fminnm z0.s, p0/m, z0.s, z1.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z3.s -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z0.s -; NOB16B16-NEXT: uzp1 z0.h, z0.h, z1.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fminnm_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v2.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: fminnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-NONSTREAMING-NEXT: fminnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fminnm_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfminnm z0.h, p0/m, z0.h, z1.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fminnm_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z2.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z3.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip2 z4.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: zip1 z1.h, z2.h, z1.h +; NOB16B16-STREAMING-NEXT: zip1 z0.h, z2.h, z0.h +; NOB16B16-STREAMING-NEXT: fminnm z3.s, p0/m, z3.s, z4.s +; NOB16B16-STREAMING-NEXT: fminnm z0.s, p0/m, z0.s, z1.s +; NOB16B16-STREAMING-NEXT: bfcvt z1.h, p0/m, z3.s +; NOB16B16-STREAMING-NEXT: bfcvt z0.h, p0/m, z0.s +; NOB16B16-STREAMING-NEXT: uzp1 z0.h, z0.h, z1.h +; NOB16B16-STREAMING-NEXT: ret %res = call <vscale x 8 x bfloat> @llvm.minnum.nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) ret <vscale x 8 x bfloat> %res } @@ -466,28 +556,45 @@ define <vscale x 4 x bfloat> @fmla_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x } define <vscale x 8 x bfloat> @fmla_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b, <vscale x 8 x bfloat> %c) { -; NOB16B16-LABEL: fmla_nxv8bf16: -; NOB16B16: // %bb.0: -; NOB16B16-NEXT: movi v3.2d, #0000000000000000 -; NOB16B16-NEXT: uunpkhi z4.s, z1.h -; NOB16B16-NEXT: uunpkhi z5.s, z0.h -; NOB16B16-NEXT: uunpklo z1.s, z1.h -; NOB16B16-NEXT: uunpklo z0.s, z0.h -; NOB16B16-NEXT: ptrue p0.s -; NOB16B16-NEXT: zip2 z6.h, z3.h, z2.h -; NOB16B16-NEXT: zip1 z2.h, z3.h, z2.h -; NOB16B16-NEXT: bfmlalb z6.s, z5.h, z4.h -; NOB16B16-NEXT: bfmlalb z2.s, z0.h, z1.h -; NOB16B16-NEXT: bfcvt z0.h, p0/m, z6.s -; NOB16B16-NEXT: bfcvt z1.h, p0/m, z2.s -; NOB16B16-NEXT: uzp1 z0.h, z1.h, z0.h -; NOB16B16-NEXT: ret +; NOB16B16-NONSTREAMING-LABEL: fmla_nxv8bf16: +; NOB16B16-NONSTREAMING: // %bb.0: +; NOB16B16-NONSTREAMING-NEXT: movi v3.2d, #0000000000000000 +; NOB16B16-NONSTREAMING-NEXT: uunpkhi z4.s, z1.h +; NOB16B16-NONSTREAMING-NEXT: uunpkhi z5.s, z0.h +; NOB16B16-NONSTREAMING-NEXT: uunpklo z1.s, z1.h +; NOB16B16-NONSTREAMING-NEXT: uunpklo z0.s, z0.h +; NOB16B16-NONSTREAMING-NEXT: ptrue p0.s +; NOB16B16-NONSTREAMING-NEXT: zip2 z6.h, z3.h, z2.h +; NOB16B16-NONSTREAMING-NEXT: zip1 z2.h, z3.h, z2.h +; NOB16B16-NONSTREAMING-NEXT: bfmlalb z6.s, z5.h, z4.h +; NOB16B16-NONSTREAMING-NEXT: bfmlalb z2.s, z0.h, z1.h +; NOB16B16-NONSTREAMING-NEXT: bfcvt z0.h, p0/m, z6.s +; NOB16B16-NONSTREAMING-NEXT: bfcvt z1.h, p0/m, z2.s +; NOB16B16-NONSTREAMING-NEXT: uzp1 z0.h, z1.h, z0.h +; NOB16B16-NONSTREAMING-NEXT: ret ; ; B16B16-LABEL: fmla_nxv8bf16: ; B16B16: // %bb.0: ; B16B16-NEXT: ptrue p0.h ; B16B16-NEXT: bfmla z0.h, p0/m, z1.h, z2.h ; B16B16-NEXT: ret +; +; NOB16B16-STREAMING-LABEL: fmla_nxv8bf16: +; NOB16B16-STREAMING: // %bb.0: +; NOB16B16-STREAMING-NEXT: mov z3.h, #0 // =0x0 +; NOB16B16-STREAMING-NEXT: uunpkhi z4.s, z1.h +; NOB16B16-STREAMING-NEXT: uunpkhi z5.s, z0.h +; NOB16B16-STREAMING-NEXT: uunpklo z1.s, z1.h +; NOB16B16-STREAMING-NEXT: uunpklo z0.s, z0.h +; NOB16B16-STREAMING-NEXT: ptrue p0.s +; NOB16B16-STREAMING-NEXT: zip2 z6.h, z3.h, z2.h +; NOB16B16-STREAMING-NEXT: zip1 z2.h, z3.h, z2.h +; NOB16B16-STREAMING-NEXT: bfmlalb z6.s, z5.h, z4.h +; NOB16B16-S... [truncated] `````````` </details> https://github.com/llvm/llvm-project/pull/222588 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
