https://github.com/llvmbot created https://github.com/llvm/llvm-project/pull/210264
Backport 50f2b9443d23eba757c3c0f42eef71e0f9721b15 Requested by: @davemgreen >From 02acfede3338d32d93d2e0a3ad29667cff141c26 Mon Sep 17 00:00:00 2001 From: David Green <[email protected]> Date: Wed, 15 Jul 2026 21:19:55 +0100 Subject: [PATCH] [AArch64] Lower fixed width bf16 fpround (#209411) Similar to #209194, this was previously hitting an error with an illegal FP_ROUND_MERGE_PASSTHRU. This time we lower the fptrunc to a scalable fptrunc and let it legalize naturally. On SVE systems a f64->bf16 fptrunc will fail to legalize, but lowers successfully with sve2 with fcvtn. A store(fptrunc) -> truncstore combine was disabled too as it fails with bf16. (cherry picked from commit 50f2b9443d23eba757c3c0f42eef71e0f9721b15) --- .../Target/AArch64/AArch64ISelLowering.cpp | 21 +- ....ll => sve-fixed-length-fp-extend-bf16.ll} | 0 .../AArch64/sve-fixed-length-fp-trunc-bf16.ll | 840 ++++++++++++++++++ 3 files changed, 851 insertions(+), 10 deletions(-) rename llvm/test/CodeGen/AArch64/{sve-fixed-length-fp-extend-trunc-bf16.ll => sve-fixed-length-fp-extend-bf16.ll} (100%) create mode 100644 llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index d6f2633297e51..cd516aae0365a 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -27379,11 +27379,6 @@ static SDValue performSTORECombine(SDNode *N, if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget)) return Res; - auto hasValidElementTypeForFPTruncStore = [](EVT VT) { - EVT EltVT = VT.getVectorElementType(); - return EltVT == MVT::f32 || EltVT == MVT::f64; - }; - // Cast ptr32 and ptr64 pointers to the default address space before a store. unsigned AddrSpace = ST->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || @@ -27400,16 +27395,24 @@ static SDValue performSTORECombine(SDNode *N, if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget)) return Res; + auto hasValidElementTypeForFPTruncStore = [](EVT DstVT, EVT SrcVT) { + if (DstVT.getScalarType() == MVT::bf16) + return false; + EVT EltVT = SrcVT.getVectorElementType(); + return EltVT == MVT::f32 || EltVT == MVT::f64; + }; + // If this is an FP_ROUND followed by a store, fold this into a truncating // store. We can do this even if this is already a truncstore. // We purposefully don't care about legality of the nodes here as we know // they can be split down into something legal. if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND && - Value.getNode()->hasOneUse() && ST->isUnindexed() && + Value->hasOneUse() && ST->isUnindexed() && Subtarget->useSVEForFixedLengthVectors() && ValueVT.isFixedLengthVector() && ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() && - hasValidElementTypeForFPTruncStore(Value.getOperand(0).getValueType())) + hasValidElementTypeForFPTruncStore(Value.getValueType(), + Value.getOperand(0).getValueType())) return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT, ST->getMemOperand()); @@ -34028,11 +34031,9 @@ AArch64TargetLowering::LowerFixedLengthFPRoundToSVE(SDValue Op, EVT ContainerSrcVT = getContainerForFixedLengthVector(DAG, SrcVT); EVT RoundVT = ContainerSrcVT.changeVectorElementType( *DAG.getContext(), VT.getVectorElementType()); - SDValue Pg = getPredicateForVector(DAG, DL, RoundVT); Val = convertToScalableVector(DAG, ContainerSrcVT, Val); - Val = DAG.getNode(AArch64ISD::FP_ROUND_MERGE_PASSTHRU, DL, RoundVT, Pg, Val, - Op.getOperand(1), DAG.getPOISON(RoundVT)); + Val = DAG.getNode(Op.getOpcode(), DL, RoundVT, Val, Op.getOperand(1)); Val = getSVESafeBitCast(ContainerSrcVT.changeTypeToInteger(), Val, DAG); Val = convertFromScalableVector(DAG, SrcVT.changeTypeToInteger(), Val); diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll similarity index 100% rename from llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-trunc-bf16.ll rename to llvm/test/CodeGen/AArch64/sve-fixed-length-fp-extend-bf16.ll diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll new file mode 100644 index 0000000000000..6d988dadf64e4 --- /dev/null +++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-fp-trunc-bf16.ll @@ -0,0 +1,840 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_256 +; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_512 +; RUN: llc -mtriple=aarch64 -mattr=+sve2 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BASE,CHECK_BASE_GE_2048 +; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=256 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_256 +; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=512 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_512 +; RUN: llc -mtriple=aarch64 -mattr=+sve2,+bf16 -aarch64-sve-vector-bits-min=2048 < %s | FileCheck %s -check-prefixes=CHECK,CHECK-BF16,CHECK_BF16_GE_2048 + +define void @fcvt_v2f32_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 { +; CHECK-BASE-LABEL: fcvt_v2f32_v2bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ldr d1, [x0] +; CHECK-BASE-NEXT: movi v0.4s, #127, msl #8 +; CHECK-BASE-NEXT: ushr v2.4s, v1.4s, #16 +; CHECK-BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s +; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-BASE-NEXT: orr v1.4s, #64, lsl #16 +; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1 +; CHECK-BASE-NEXT: add v0.4s, v2.4s, v0.4s +; CHECK-BASE-NEXT: bif v0.16b, v1.16b, v3.16b +; CHECK-BASE-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-BASE-NEXT: str s0, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v2f32_v2bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ldr d0, [x0] +; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s +; CHECK-BF16-NEXT: str s0, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <2 x float>, ptr %a + %res = fptrunc <2 x float> %op1 to <2 x bfloat> + store <2 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v4f32_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 { +; CHECK-BASE-LABEL: fcvt_v4f32_v4bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ldr q1, [x0] +; CHECK-BASE-NEXT: movi v0.4s, #127, msl #8 +; CHECK-BASE-NEXT: ushr v2.4s, v1.4s, #16 +; CHECK-BASE-NEXT: fcmeq v3.4s, v1.4s, v1.4s +; CHECK-BASE-NEXT: add v0.4s, v1.4s, v0.4s +; CHECK-BASE-NEXT: orr v1.4s, #64, lsl #16 +; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1 +; CHECK-BASE-NEXT: add v0.4s, v2.4s, v0.4s +; CHECK-BASE-NEXT: bif v0.16b, v1.16b, v3.16b +; CHECK-BASE-NEXT: shrn v0.4h, v0.4s, #16 +; CHECK-BASE-NEXT: str d0, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v4f32_v4bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ldr q0, [x0] +; CHECK-BF16-NEXT: bfcvtn v0.4h, v0.4s +; CHECK-BF16-NEXT: str d0, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <4 x float>, ptr %a + %res = fptrunc <4 x float> %op1 to <4 x bfloat> + store <4 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v8f32_v8bf16(ptr %a, ptr %b) vscale_range(2,0) #0 { +; CHECK-BASE-LABEL: fcvt_v8f32_v8bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.s, vl8 +; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-BASE-NEXT: ptrue p0.s +; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16 +; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1 +; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s +; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: str q0, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v8f32_v8bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.s, vl8 +; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-BF16-NEXT: ptrue p0.s +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: str q0, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <8 x float>, ptr %a + %res = fptrunc <8 x float> %op1 to <8 x bfloat> + store <8 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v16f32_v16bf16(ptr %a, ptr %b) #0 { +; CHECK_BASE_GE_256-LABEL: fcvt_v16f32_v16bf16: +; CHECK_BASE_GE_256: // %bb.0: +; CHECK_BASE_GE_256-NEXT: ptrue p0.s, vl8 +; CHECK_BASE_GE_256-NEXT: mov x8, #8 // =0x8 +; CHECK_BASE_GE_256-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK_BASE_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] +; CHECK_BASE_GE_256-NEXT: ld1w { z2.s }, p0/z, [x0] +; CHECK_BASE_GE_256-NEXT: ptrue p0.s +; CHECK_BASE_GE_256-NEXT: lsr z3.s, z0.s, #16 +; CHECK_BASE_GE_256-NEXT: lsr z5.s, z2.s, #16 +; CHECK_BASE_GE_256-NEXT: add z4.s, z0.s, z1.s +; CHECK_BASE_GE_256-NEXT: add z1.s, z2.s, z1.s +; CHECK_BASE_GE_256-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK_BASE_GE_256-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK_BASE_GE_256-NEXT: and z3.s, z3.s, #0x1 +; CHECK_BASE_GE_256-NEXT: and z5.s, z5.s, #0x1 +; CHECK_BASE_GE_256-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s +; CHECK_BASE_GE_256-NEXT: orr z2.s, z2.s, #0x400000 +; CHECK_BASE_GE_256-NEXT: add z3.s, z3.s, z4.s +; CHECK_BASE_GE_256-NEXT: add z1.s, z5.s, z1.s +; CHECK_BASE_GE_256-NEXT: sel z0.s, p1, z0.s, z3.s +; CHECK_BASE_GE_256-NEXT: mov z1.s, p0/m, z2.s +; CHECK_BASE_GE_256-NEXT: lsr z0.s, z0.s, #16 +; CHECK_BASE_GE_256-NEXT: lsr z1.s, z1.s, #16 +; CHECK_BASE_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BASE_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BASE_GE_256-NEXT: stp q1, q0, [x1] +; CHECK_BASE_GE_256-NEXT: ret +; +; CHECK_BASE_GE_512-LABEL: fcvt_v16f32_v16bf16: +; CHECK_BASE_GE_512: // %bb.0: +; CHECK_BASE_GE_512-NEXT: ptrue p0.s, vl16 +; CHECK_BASE_GE_512-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK_BASE_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK_BASE_GE_512-NEXT: ptrue p0.s +; CHECK_BASE_GE_512-NEXT: movprfx z2, z0 +; CHECK_BASE_GE_512-NEXT: ext z2.b, z2.b, z0.b, #32 +; CHECK_BASE_GE_512-NEXT: lsr z3.s, z0.s, #16 +; CHECK_BASE_GE_512-NEXT: add z5.s, z0.s, z1.s +; CHECK_BASE_GE_512-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK_BASE_GE_512-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK_BASE_GE_512-NEXT: lsr z4.s, z2.s, #16 +; CHECK_BASE_GE_512-NEXT: and z3.s, z3.s, #0x1 +; CHECK_BASE_GE_512-NEXT: add z1.s, z2.s, z1.s +; CHECK_BASE_GE_512-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s +; CHECK_BASE_GE_512-NEXT: orr z2.s, z2.s, #0x400000 +; CHECK_BASE_GE_512-NEXT: and z4.s, z4.s, #0x1 +; CHECK_BASE_GE_512-NEXT: add z3.s, z3.s, z5.s +; CHECK_BASE_GE_512-NEXT: add z1.s, z4.s, z1.s +; CHECK_BASE_GE_512-NEXT: sel z0.s, p1, z0.s, z3.s +; CHECK_BASE_GE_512-NEXT: mov z1.s, p0/m, z2.s +; CHECK_BASE_GE_512-NEXT: lsr z0.s, z0.s, #16 +; CHECK_BASE_GE_512-NEXT: lsr z1.s, z1.s, #16 +; CHECK_BASE_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BASE_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BASE_GE_512-NEXT: stp q0, q1, [x1] +; CHECK_BASE_GE_512-NEXT: ret +; +; CHECK_BASE_GE_2048-LABEL: fcvt_v16f32_v16bf16: +; CHECK_BASE_GE_2048: // %bb.0: +; CHECK_BASE_GE_2048-NEXT: ptrue p0.s, vl16 +; CHECK_BASE_GE_2048-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK_BASE_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK_BASE_GE_2048-NEXT: ptrue p0.s +; CHECK_BASE_GE_2048-NEXT: movprfx z2, z0 +; CHECK_BASE_GE_2048-NEXT: ext z2.b, z2.b, z0.b, #32 +; CHECK_BASE_GE_2048-NEXT: lsr z3.s, z0.s, #16 +; CHECK_BASE_GE_2048-NEXT: add z5.s, z0.s, z1.s +; CHECK_BASE_GE_2048-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK_BASE_GE_2048-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK_BASE_GE_2048-NEXT: lsr z4.s, z2.s, #16 +; CHECK_BASE_GE_2048-NEXT: and z3.s, z3.s, #0x1 +; CHECK_BASE_GE_2048-NEXT: add z1.s, z2.s, z1.s +; CHECK_BASE_GE_2048-NEXT: fcmuo p0.s, p0/z, z2.s, z2.s +; CHECK_BASE_GE_2048-NEXT: orr z2.s, z2.s, #0x400000 +; CHECK_BASE_GE_2048-NEXT: and z4.s, z4.s, #0x1 +; CHECK_BASE_GE_2048-NEXT: add z3.s, z3.s, z5.s +; CHECK_BASE_GE_2048-NEXT: add z1.s, z4.s, z1.s +; CHECK_BASE_GE_2048-NEXT: sel z0.s, p1, z0.s, z3.s +; CHECK_BASE_GE_2048-NEXT: mov z1.s, p0/m, z2.s +; CHECK_BASE_GE_2048-NEXT: lsr z0.s, z0.s, #16 +; CHECK_BASE_GE_2048-NEXT: lsr z1.s, z1.s, #16 +; CHECK_BASE_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BASE_GE_2048-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BASE_GE_2048-NEXT: stp q0, q1, [x1] +; CHECK_BASE_GE_2048-NEXT: ret +; +; CHECK_BF16_GE_256-LABEL: fcvt_v16f32_v16bf16: +; CHECK_BF16_GE_256: // %bb.0: +; CHECK_BF16_GE_256-NEXT: ptrue p0.s, vl8 +; CHECK_BF16_GE_256-NEXT: mov x8, #8 // =0x8 +; CHECK_BF16_GE_256-NEXT: ptrue p1.s +; CHECK_BF16_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2] +; CHECK_BF16_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0] +; CHECK_BF16_GE_256-NEXT: bfcvt z0.h, p1/m, z0.s +; CHECK_BF16_GE_256-NEXT: bfcvt z1.h, p1/m, z1.s +; CHECK_BF16_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BF16_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BF16_GE_256-NEXT: stp q1, q0, [x1] +; CHECK_BF16_GE_256-NEXT: ret +; +; CHECK_BF16_GE_512-LABEL: fcvt_v16f32_v16bf16: +; CHECK_BF16_GE_512: // %bb.0: +; CHECK_BF16_GE_512-NEXT: ptrue p0.s, vl16 +; CHECK_BF16_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK_BF16_GE_512-NEXT: ptrue p0.s +; CHECK_BF16_GE_512-NEXT: movprfx z1, z0 +; CHECK_BF16_GE_512-NEXT: ext z1.b, z1.b, z0.b, #32 +; CHECK_BF16_GE_512-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK_BF16_GE_512-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK_BF16_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BF16_GE_512-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BF16_GE_512-NEXT: stp q0, q1, [x1] +; CHECK_BF16_GE_512-NEXT: ret +; +; CHECK_BF16_GE_2048-LABEL: fcvt_v16f32_v16bf16: +; CHECK_BF16_GE_2048: // %bb.0: +; CHECK_BF16_GE_2048-NEXT: ptrue p0.s, vl16 +; CHECK_BF16_GE_2048-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK_BF16_GE_2048-NEXT: ptrue p0.s +; CHECK_BF16_GE_2048-NEXT: movprfx z1, z0 +; CHECK_BF16_GE_2048-NEXT: ext z1.b, z1.b, z0.b, #32 +; CHECK_BF16_GE_2048-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK_BF16_GE_2048-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK_BF16_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BF16_GE_2048-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BF16_GE_2048-NEXT: stp q0, q1, [x1] +; CHECK_BF16_GE_2048-NEXT: ret + %op1 = load <16 x float>, ptr %a + %res = fptrunc <16 x float> %op1 to <16 x bfloat> + store <16 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v32f32_v32bf16(ptr %a, ptr %b) vscale_range(8,0) #0 { +; CHECK-BASE-LABEL: fcvt_v32f32_v32bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.s, vl32 +; CHECK-BASE-NEXT: mov z3.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-BASE-NEXT: ptrue p0.s +; CHECK-BASE-NEXT: movprfx z1, z0 +; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #64 +; CHECK-BASE-NEXT: movprfx z2, z0 +; CHECK-BASE-NEXT: ext z2.b, z2.b, z0.b, #32 +; CHECK-BASE-NEXT: lsr z4.s, z0.s, #16 +; CHECK-BASE-NEXT: add z16.s, z0.s, z3.s +; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: movprfx z7, z1 +; CHECK-BASE-NEXT: ext z7.b, z7.b, z1.b, #32 +; CHECK-BASE-NEXT: lsr z5.s, z2.s, #16 +; CHECK-BASE-NEXT: lsr z6.s, z1.s, #16 +; CHECK-BASE-NEXT: add z17.s, z2.s, z3.s +; CHECK-BASE-NEXT: add z18.s, z1.s, z3.s +; CHECK-BASE-NEXT: and z4.s, z4.s, #0x1 +; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s +; CHECK-BASE-NEXT: orr z2.s, z2.s, #0x400000 +; CHECK-BASE-NEXT: lsr z19.s, z7.s, #16 +; CHECK-BASE-NEXT: and z5.s, z5.s, #0x1 +; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1 +; CHECK-BASE-NEXT: add z3.s, z7.s, z3.s +; CHECK-BASE-NEXT: add z4.s, z4.s, z16.s +; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z1.s, z1.s +; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000 +; CHECK-BASE-NEXT: and z19.s, z19.s, #0x1 +; CHECK-BASE-NEXT: add z5.s, z5.s, z17.s +; CHECK-BASE-NEXT: add z6.s, z6.s, z18.s +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z7.s, z7.s +; CHECK-BASE-NEXT: orr z7.s, z7.s, #0x400000 +; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z4.s +; CHECK-BASE-NEXT: add z3.s, z19.s, z3.s +; CHECK-BASE-NEXT: sel z2.s, p2, z2.s, z5.s +; CHECK-BASE-NEXT: sel z1.s, p3, z1.s, z6.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: mov z3.s, p0/m, z7.s +; CHECK-BASE-NEXT: lsr z2.s, z2.s, #16 +; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16 +; CHECK-BASE-NEXT: uzp1 z2.h, z2.h, z2.h +; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h +; CHECK-BASE-NEXT: stp q0, q2, [x1] +; CHECK-BASE-NEXT: stp q1, q3, [x1, #32] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v32f32_v32bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.s, vl32 +; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-BF16-NEXT: ptrue p0.s +; CHECK-BF16-NEXT: movprfx z1, z0 +; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #64 +; CHECK-BF16-NEXT: movprfx z2, z0 +; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #32 +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: movprfx z3, z1 +; CHECK-BF16-NEXT: ext z3.b, z3.b, z1.b, #32 +; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s +; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s +; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h +; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h +; CHECK-BF16-NEXT: stp q0, q2, [x1] +; CHECK-BF16-NEXT: stp q1, q3, [x1, #32] +; CHECK-BF16-NEXT: ret + %op1 = load <32 x float>, ptr %a + %res = fptrunc <32 x float> %op1 to <32 x bfloat> + store <32 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v64f32_v64bf16(ptr %a, ptr %b) vscale_range(16,0) #0 { +; CHECK-BASE-LABEL: fcvt_v64f32_v64bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.s, vl64 +; CHECK-BASE-NEXT: mov z2.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-BASE-NEXT: ptrue p0.s +; CHECK-BASE-NEXT: movprfx z1, z0 +; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #128 +; CHECK-BASE-NEXT: movprfx z3, z0 +; CHECK-BASE-NEXT: ext z3.b, z3.b, z0.b, #32 +; CHECK-BASE-NEXT: movprfx z4, z0 +; CHECK-BASE-NEXT: ext z4.b, z4.b, z0.b, #64 +; CHECK-BASE-NEXT: lsr z16.s, z0.s, #16 +; CHECK-BASE-NEXT: fcmuo p5.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: lsr z22.s, z1.s, #16 +; CHECK-BASE-NEXT: movprfx z5, z1 +; CHECK-BASE-NEXT: ext z5.b, z5.b, z1.b, #32 +; CHECK-BASE-NEXT: lsr z6.s, z3.s, #16 +; CHECK-BASE-NEXT: lsr z7.s, z4.s, #16 +; CHECK-BASE-NEXT: movprfx z17, z1 +; CHECK-BASE-NEXT: ext z17.b, z17.b, z1.b, #64 +; CHECK-BASE-NEXT: add z23.s, z1.s, z2.s +; CHECK-BASE-NEXT: movprfx z18, z4 +; CHECK-BASE-NEXT: ext z18.b, z18.b, z4.b, #32 +; CHECK-BASE-NEXT: add z19.s, z3.s, z2.s +; CHECK-BASE-NEXT: add z21.s, z4.s, z2.s +; CHECK-BASE-NEXT: and z22.s, z22.s, #0x1 +; CHECK-BASE-NEXT: lsr z20.s, z5.s, #16 +; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1 +; CHECK-BASE-NEXT: and z7.s, z7.s, #0x1 +; CHECK-BASE-NEXT: movprfx z24, z17 +; CHECK-BASE-NEXT: ext z24.b, z24.b, z17.b, #32 +; CHECK-BASE-NEXT: and z16.s, z16.s, #0x1 +; CHECK-BASE-NEXT: add z26.s, z5.s, z2.s +; CHECK-BASE-NEXT: lsr z25.s, z18.s, #16 +; CHECK-BASE-NEXT: add z22.s, z22.s, z23.s +; CHECK-BASE-NEXT: add z23.s, z0.s, z2.s +; CHECK-BASE-NEXT: and z20.s, z20.s, #0x1 +; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z3.s, z3.s +; CHECK-BASE-NEXT: add z6.s, z6.s, z19.s +; CHECK-BASE-NEXT: add z7.s, z7.s, z21.s +; CHECK-BASE-NEXT: lsr z21.s, z17.s, #16 +; CHECK-BASE-NEXT: orr z3.s, z3.s, #0x400000 +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: add z16.s, z16.s, z23.s +; CHECK-BASE-NEXT: lsr z23.s, z24.s, #16 +; CHECK-BASE-NEXT: add z20.s, z20.s, z26.s +; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z5.s, z5.s +; CHECK-BASE-NEXT: orr z5.s, z5.s, #0x400000 +; CHECK-BASE-NEXT: add z19.s, z18.s, z2.s +; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z4.s, z4.s +; CHECK-BASE-NEXT: add z26.s, z17.s, z2.s +; CHECK-BASE-NEXT: and z25.s, z25.s, #0x1 +; CHECK-BASE-NEXT: fcmuo p4.s, p0/z, z1.s, z1.s +; CHECK-BASE-NEXT: and z21.s, z21.s, #0x1 +; CHECK-BASE-NEXT: orr z4.s, z4.s, #0x400000 +; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000 +; CHECK-BASE-NEXT: sel z3.s, p2, z3.s, z6.s +; CHECK-BASE-NEXT: sel z0.s, p5, z0.s, z16.s +; CHECK-BASE-NEXT: and z23.s, z23.s, #0x1 +; CHECK-BASE-NEXT: add z2.s, z24.s, z2.s +; CHECK-BASE-NEXT: sel z5.s, p1, z5.s, z20.s +; CHECK-BASE-NEXT: sel z4.s, p3, z4.s, z7.s +; CHECK-BASE-NEXT: add z6.s, z25.s, z19.s +; CHECK-BASE-NEXT: add z7.s, z21.s, z26.s +; CHECK-BASE-NEXT: sel z1.s, p4, z1.s, z22.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16 +; CHECK-BASE-NEXT: add z2.s, z23.s, z2.s +; CHECK-BASE-NEXT: lsr z5.s, z5.s, #16 +; CHECK-BASE-NEXT: lsr z4.s, z4.s, #16 +; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h +; CHECK-BASE-NEXT: uzp1 z5.h, z5.h, z5.h +; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z18.s, z18.s +; CHECK-BASE-NEXT: orr z18.s, z18.s, #0x400000 +; CHECK-BASE-NEXT: stp q0, q3, [x1] +; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z17.s, z17.s +; CHECK-BASE-NEXT: orr z17.s, z17.s, #0x400000 +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z24.s, z24.s +; CHECK-BASE-NEXT: orr z24.s, z24.s, #0x400000 +; CHECK-BASE-NEXT: stp q1, q5, [x1, #64] +; CHECK-BASE-NEXT: mov z6.s, p1/m, z18.s +; CHECK-BASE-NEXT: mov z7.s, p2/m, z17.s +; CHECK-BASE-NEXT: mov z2.s, p0/m, z24.s +; CHECK-BASE-NEXT: lsr z6.s, z6.s, #16 +; CHECK-BASE-NEXT: lsr z7.s, z7.s, #16 +; CHECK-BASE-NEXT: lsr z0.s, z2.s, #16 +; CHECK-BASE-NEXT: uzp1 z2.h, z4.h, z4.h +; CHECK-BASE-NEXT: uzp1 z3.h, z6.h, z6.h +; CHECK-BASE-NEXT: uzp1 z1.h, z7.h, z7.h +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: stp q2, q3, [x1, #32] +; CHECK-BASE-NEXT: stp q1, q0, [x1, #96] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v64f32_v64bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.s, vl64 +; CHECK-BF16-NEXT: ld1w { z0.s }, p0/z, [x0] +; CHECK-BF16-NEXT: ptrue p0.s +; CHECK-BF16-NEXT: movprfx z3, z0 +; CHECK-BF16-NEXT: ext z3.b, z3.b, z0.b, #32 +; CHECK-BF16-NEXT: movprfx z1, z0 +; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #128 +; CHECK-BF16-NEXT: movprfx z2, z0 +; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #64 +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s +; CHECK-BF16-NEXT: movprfx z4, z1 +; CHECK-BF16-NEXT: ext z4.b, z4.b, z1.b, #32 +; CHECK-BF16-NEXT: movprfx z5, z1 +; CHECK-BF16-NEXT: ext z5.b, z5.b, z1.b, #64 +; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK-BF16-NEXT: movprfx z6, z2 +; CHECK-BF16-NEXT: ext z6.b, z6.b, z2.b, #32 +; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: bfcvt z4.h, p0/m, z4.s +; CHECK-BF16-NEXT: movprfx z7, z5 +; CHECK-BF16-NEXT: ext z7.b, z7.b, z5.b, #32 +; CHECK-BF16-NEXT: bfcvt z5.h, p0/m, z5.s +; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h +; CHECK-BF16-NEXT: bfcvt z6.h, p0/m, z6.s +; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h +; CHECK-BF16-NEXT: stp q0, q3, [x1] +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z7.s +; CHECK-BF16-NEXT: uzp1 z4.h, z4.h, z4.h +; CHECK-BF16-NEXT: uzp1 z3.h, z5.h, z5.h +; CHECK-BF16-NEXT: stp q1, q4, [x1, #64] +; CHECK-BF16-NEXT: uzp1 z1.h, z6.h, z6.h +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: stp q2, q1, [x1, #32] +; CHECK-BF16-NEXT: stp q3, q0, [x1, #96] +; CHECK-BF16-NEXT: ret + %op1 = load <64 x float>, ptr %a + %res = fptrunc <64 x float> %op1 to <64 x bfloat> + store <64 x bfloat> %res, ptr %b + ret void +} + +; FCVT D -> H + +define void @fcvt_v1f64_v1bf16(ptr %a, ptr %b) vscale_range(2,0) #0 { +; CHECK-BASE-LABEL: fcvt_v1f64_v1bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ldr d0, [x0] +; CHECK-BASE-NEXT: mov w8, #32767 // =0x7fff +; CHECK-BASE-NEXT: fcvtxn s0, d0 +; CHECK-BASE-NEXT: fmov w9, s0 +; CHECK-BASE-NEXT: ubfx w10, w9, #16, #1 +; CHECK-BASE-NEXT: add w8, w9, w8 +; CHECK-BASE-NEXT: add w8, w10, w8 +; CHECK-BASE-NEXT: lsr w8, w8, #16 +; CHECK-BASE-NEXT: fmov s0, w8 +; CHECK-BASE-NEXT: str h0, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v1f64_v1bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ldr d0, [x0] +; CHECK-BF16-NEXT: fcvtxn s0, d0 +; CHECK-BF16-NEXT: bfcvt h0, s0 +; CHECK-BF16-NEXT: str h0, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <1 x double>, ptr %a + %res = fptrunc <1 x double> %op1 to <1 x bfloat> + store <1 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v2f64_v2bf16(ptr %a, ptr %b) vscale_range(2,0) #0 { +; CHECK-BASE-LABEL: fcvt_v2f64_v2bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.d +; CHECK-BASE-NEXT: ldr q0, [x0] +; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16 +; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1 +; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s +; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: str s0, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v2f64_v2bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.d +; CHECK-BF16-NEXT: ldr q0, [x0] +; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: str s0, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <2 x double>, ptr %a + %res = fptrunc <2 x double> %op1 to <2 x bfloat> + store <2 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v4f64_v4bf16(ptr %a, ptr %b) vscale_range(2,0) #0 { +; CHECK-BASE-LABEL: fcvt_v4f64_v4bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.d, vl4 +; CHECK-BASE-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-BASE-NEXT: ptrue p0.d +; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BASE-NEXT: lsr z2.s, z0.s, #16 +; CHECK-BASE-NEXT: add z1.s, z0.s, z1.s +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: and z2.s, z2.s, #0x1 +; CHECK-BASE-NEXT: add z1.s, z2.s, z1.s +; CHECK-BASE-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: str d0, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v4f64_v4bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.d, vl4 +; CHECK-BF16-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-BF16-NEXT: ptrue p0.d +; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: str d0, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <4 x double>, ptr %a + %res = fptrunc <4 x double> %op1 to <4 x bfloat> + store <4 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v8f64_v8bf16(ptr %a, ptr %b) #0 { +; CHECK_BASE_GE_256-LABEL: fcvt_v8f64_v8bf16: +; CHECK_BASE_GE_256: // %bb.0: +; CHECK_BASE_GE_256-NEXT: ptrue p1.d, vl4 +; CHECK_BASE_GE_256-NEXT: mov x8, #4 // =0x4 +; CHECK_BASE_GE_256-NEXT: mov z2.s, #32767 // =0x7fff +; CHECK_BASE_GE_256-NEXT: ptrue p0.d +; CHECK_BASE_GE_256-NEXT: ld1d { z0.d }, p1/z, [x0, x8, lsl #3] +; CHECK_BASE_GE_256-NEXT: ld1d { z1.d }, p1/z, [x0] +; CHECK_BASE_GE_256-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK_BASE_GE_256-NEXT: fcvtx z1.s, p0/m, z1.d +; CHECK_BASE_GE_256-NEXT: lsr z3.s, z0.s, #16 +; CHECK_BASE_GE_256-NEXT: lsr z5.s, z1.s, #16 +; CHECK_BASE_GE_256-NEXT: add z4.s, z0.s, z2.s +; CHECK_BASE_GE_256-NEXT: add z2.s, z1.s, z2.s +; CHECK_BASE_GE_256-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK_BASE_GE_256-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK_BASE_GE_256-NEXT: and z3.s, z3.s, #0x1 +; CHECK_BASE_GE_256-NEXT: and z5.s, z5.s, #0x1 +; CHECK_BASE_GE_256-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s +; CHECK_BASE_GE_256-NEXT: orr z1.s, z1.s, #0x400000 +; CHECK_BASE_GE_256-NEXT: add z3.s, z3.s, z4.s +; CHECK_BASE_GE_256-NEXT: add z2.s, z5.s, z2.s +; CHECK_BASE_GE_256-NEXT: sel z0.s, p1, z0.s, z3.s +; CHECK_BASE_GE_256-NEXT: sel z1.s, p0, z1.s, z2.s +; CHECK_BASE_GE_256-NEXT: lsr z0.s, z0.s, #16 +; CHECK_BASE_GE_256-NEXT: lsr z1.s, z1.s, #16 +; CHECK_BASE_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BASE_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK_BASE_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BASE_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BASE_GE_256-NEXT: mov v1.d[1], v0.d[0] +; CHECK_BASE_GE_256-NEXT: str q1, [x1] +; CHECK_BASE_GE_256-NEXT: ret +; +; CHECK_BASE_GE_512-LABEL: fcvt_v8f64_v8bf16: +; CHECK_BASE_GE_512: // %bb.0: +; CHECK_BASE_GE_512-NEXT: ptrue p0.d, vl8 +; CHECK_BASE_GE_512-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK_BASE_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK_BASE_GE_512-NEXT: ptrue p0.d +; CHECK_BASE_GE_512-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK_BASE_GE_512-NEXT: lsr z2.s, z0.s, #16 +; CHECK_BASE_GE_512-NEXT: add z1.s, z0.s, z1.s +; CHECK_BASE_GE_512-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s +; CHECK_BASE_GE_512-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK_BASE_GE_512-NEXT: and z2.s, z2.s, #0x1 +; CHECK_BASE_GE_512-NEXT: add z1.s, z2.s, z1.s +; CHECK_BASE_GE_512-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK_BASE_GE_512-NEXT: lsr z0.s, z0.s, #16 +; CHECK_BASE_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BASE_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BASE_GE_512-NEXT: str q0, [x1] +; CHECK_BASE_GE_512-NEXT: ret +; +; CHECK_BASE_GE_2048-LABEL: fcvt_v8f64_v8bf16: +; CHECK_BASE_GE_2048: // %bb.0: +; CHECK_BASE_GE_2048-NEXT: ptrue p0.d, vl8 +; CHECK_BASE_GE_2048-NEXT: mov z1.s, #32767 // =0x7fff +; CHECK_BASE_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK_BASE_GE_2048-NEXT: ptrue p0.d +; CHECK_BASE_GE_2048-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK_BASE_GE_2048-NEXT: lsr z2.s, z0.s, #16 +; CHECK_BASE_GE_2048-NEXT: add z1.s, z0.s, z1.s +; CHECK_BASE_GE_2048-NEXT: fcmuo p0.s, p0/z, z0.s, z0.s +; CHECK_BASE_GE_2048-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK_BASE_GE_2048-NEXT: and z2.s, z2.s, #0x1 +; CHECK_BASE_GE_2048-NEXT: add z1.s, z2.s, z1.s +; CHECK_BASE_GE_2048-NEXT: sel z0.s, p0, z0.s, z1.s +; CHECK_BASE_GE_2048-NEXT: lsr z0.s, z0.s, #16 +; CHECK_BASE_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BASE_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BASE_GE_2048-NEXT: str q0, [x1] +; CHECK_BASE_GE_2048-NEXT: ret +; +; CHECK_BF16_GE_256-LABEL: fcvt_v8f64_v8bf16: +; CHECK_BF16_GE_256: // %bb.0: +; CHECK_BF16_GE_256-NEXT: ptrue p0.d, vl4 +; CHECK_BF16_GE_256-NEXT: mov x8, #4 // =0x4 +; CHECK_BF16_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3] +; CHECK_BF16_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0] +; CHECK_BF16_GE_256-NEXT: ptrue p0.d +; CHECK_BF16_GE_256-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK_BF16_GE_256-NEXT: fcvtx z1.s, p0/m, z1.d +; CHECK_BF16_GE_256-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK_BF16_GE_256-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK_BF16_GE_256-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BF16_GE_256-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK_BF16_GE_256-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BF16_GE_256-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK_BF16_GE_256-NEXT: mov v1.d[1], v0.d[0] +; CHECK_BF16_GE_256-NEXT: str q1, [x1] +; CHECK_BF16_GE_256-NEXT: ret +; +; CHECK_BF16_GE_512-LABEL: fcvt_v8f64_v8bf16: +; CHECK_BF16_GE_512: // %bb.0: +; CHECK_BF16_GE_512-NEXT: ptrue p0.d, vl8 +; CHECK_BF16_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK_BF16_GE_512-NEXT: ptrue p0.d +; CHECK_BF16_GE_512-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK_BF16_GE_512-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK_BF16_GE_512-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BF16_GE_512-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BF16_GE_512-NEXT: str q0, [x1] +; CHECK_BF16_GE_512-NEXT: ret +; +; CHECK_BF16_GE_2048-LABEL: fcvt_v8f64_v8bf16: +; CHECK_BF16_GE_2048: // %bb.0: +; CHECK_BF16_GE_2048-NEXT: ptrue p0.d, vl8 +; CHECK_BF16_GE_2048-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK_BF16_GE_2048-NEXT: ptrue p0.d +; CHECK_BF16_GE_2048-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK_BF16_GE_2048-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK_BF16_GE_2048-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK_BF16_GE_2048-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK_BF16_GE_2048-NEXT: str q0, [x1] +; CHECK_BF16_GE_2048-NEXT: ret + %op1 = load <8 x double>, ptr %a + %res = fptrunc <8 x double> %op1 to <8 x bfloat> + store <8 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v16f64_v16bf16(ptr %a, ptr %b) vscale_range(8,0) #0 { +; CHECK-BASE-LABEL: fcvt_v16f64_v16bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.d, vl16 +; CHECK-BASE-NEXT: mov z2.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-BASE-NEXT: ptrue p0.d +; CHECK-BASE-NEXT: movprfx z1, z0 +; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #64 +; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BASE-NEXT: fcvtx z1.s, p0/m, z1.d +; CHECK-BASE-NEXT: lsr z3.s, z0.s, #16 +; CHECK-BASE-NEXT: add z5.s, z0.s, z2.s +; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: lsr z4.s, z1.s, #16 +; CHECK-BASE-NEXT: and z3.s, z3.s, #0x1 +; CHECK-BASE-NEXT: add z2.s, z1.s, z2.s +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s +; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000 +; CHECK-BASE-NEXT: and z4.s, z4.s, #0x1 +; CHECK-BASE-NEXT: add z3.s, z3.s, z5.s +; CHECK-BASE-NEXT: add z2.s, z4.s, z2.s +; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z3.s +; CHECK-BASE-NEXT: sel z1.s, p0, z1.s, z2.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BASE-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BASE-NEXT: stp q0, q1, [x1] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v16f64_v16bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.d, vl16 +; CHECK-BF16-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-BF16-NEXT: ptrue p0.d +; CHECK-BF16-NEXT: movprfx z1, z0 +; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #64 +; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BF16-NEXT: fcvtx z1.s, p0/m, z1.d +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BF16-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BF16-NEXT: stp q0, q1, [x1] +; CHECK-BF16-NEXT: ret + %op1 = load <16 x double>, ptr %a + %res = fptrunc <16 x double> %op1 to <16 x bfloat> + store <16 x bfloat> %res, ptr %b + ret void +} + +define void @fcvt_v32f64_v32bf16(ptr %a, ptr %b) vscale_range(16,0) #0 { +; CHECK-BASE-LABEL: fcvt_v32f64_v32bf16: +; CHECK-BASE: // %bb.0: +; CHECK-BASE-NEXT: ptrue p0.d, vl32 +; CHECK-BASE-NEXT: mov z4.s, #32767 // =0x7fff +; CHECK-BASE-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-BASE-NEXT: ptrue p0.d +; CHECK-BASE-NEXT: movprfx z1, z0 +; CHECK-BASE-NEXT: ext z1.b, z1.b, z0.b, #128 +; CHECK-BASE-NEXT: movprfx z2, z0 +; CHECK-BASE-NEXT: ext z2.b, z2.b, z0.b, #64 +; CHECK-BASE-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BASE-NEXT: fcvtx z3.s, p0/m, z1.d +; CHECK-BASE-NEXT: ext z1.b, z1.b, z1.b, #64 +; CHECK-BASE-NEXT: fcvtx z2.s, p0/m, z2.d +; CHECK-BASE-NEXT: lsr z5.s, z0.s, #16 +; CHECK-BASE-NEXT: add z16.s, z0.s, z4.s +; CHECK-BASE-NEXT: fcmuo p1.s, p0/z, z0.s, z0.s +; CHECK-BASE-NEXT: orr z0.s, z0.s, #0x400000 +; CHECK-BASE-NEXT: fcvtx z1.s, p0/m, z1.d +; CHECK-BASE-NEXT: lsr z6.s, z2.s, #16 +; CHECK-BASE-NEXT: lsr z7.s, z3.s, #16 +; CHECK-BASE-NEXT: add z17.s, z2.s, z4.s +; CHECK-BASE-NEXT: add z18.s, z3.s, z4.s +; CHECK-BASE-NEXT: and z5.s, z5.s, #0x1 +; CHECK-BASE-NEXT: fcmuo p2.s, p0/z, z2.s, z2.s +; CHECK-BASE-NEXT: orr z2.s, z2.s, #0x400000 +; CHECK-BASE-NEXT: lsr z19.s, z1.s, #16 +; CHECK-BASE-NEXT: and z6.s, z6.s, #0x1 +; CHECK-BASE-NEXT: and z7.s, z7.s, #0x1 +; CHECK-BASE-NEXT: add z4.s, z1.s, z4.s +; CHECK-BASE-NEXT: add z5.s, z5.s, z16.s +; CHECK-BASE-NEXT: fcmuo p3.s, p0/z, z3.s, z3.s +; CHECK-BASE-NEXT: orr z3.s, z3.s, #0x400000 +; CHECK-BASE-NEXT: and z19.s, z19.s, #0x1 +; CHECK-BASE-NEXT: add z6.s, z6.s, z17.s +; CHECK-BASE-NEXT: add z7.s, z7.s, z18.s +; CHECK-BASE-NEXT: fcmuo p0.s, p0/z, z1.s, z1.s +; CHECK-BASE-NEXT: orr z1.s, z1.s, #0x400000 +; CHECK-BASE-NEXT: sel z0.s, p1, z0.s, z5.s +; CHECK-BASE-NEXT: add z4.s, z19.s, z4.s +; CHECK-BASE-NEXT: sel z2.s, p2, z2.s, z6.s +; CHECK-BASE-NEXT: sel z3.s, p3, z3.s, z7.s +; CHECK-BASE-NEXT: lsr z0.s, z0.s, #16 +; CHECK-BASE-NEXT: sel z1.s, p0, z1.s, z4.s +; CHECK-BASE-NEXT: lsr z2.s, z2.s, #16 +; CHECK-BASE-NEXT: lsr z3.s, z3.s, #16 +; CHECK-BASE-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BASE-NEXT: lsr z1.s, z1.s, #16 +; CHECK-BASE-NEXT: uzp1 z2.s, z2.s, z2.s +; CHECK-BASE-NEXT: uzp1 z3.s, z3.s, z3.s +; CHECK-BASE-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BASE-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-BASE-NEXT: uzp1 z2.h, z2.h, z2.h +; CHECK-BASE-NEXT: uzp1 z3.h, z3.h, z3.h +; CHECK-BASE-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BASE-NEXT: stp q0, q2, [x1] +; CHECK-BASE-NEXT: stp q3, q1, [x1, #32] +; CHECK-BASE-NEXT: ret +; +; CHECK-BF16-LABEL: fcvt_v32f64_v32bf16: +; CHECK-BF16: // %bb.0: +; CHECK-BF16-NEXT: ptrue p0.d, vl32 +; CHECK-BF16-NEXT: ld1d { z0.d }, p0/z, [x0] +; CHECK-BF16-NEXT: ptrue p0.d +; CHECK-BF16-NEXT: movprfx z1, z0 +; CHECK-BF16-NEXT: ext z1.b, z1.b, z0.b, #128 +; CHECK-BF16-NEXT: movprfx z2, z0 +; CHECK-BF16-NEXT: ext z2.b, z2.b, z0.b, #64 +; CHECK-BF16-NEXT: fcvtx z0.s, p0/m, z0.d +; CHECK-BF16-NEXT: movprfx z3, z1 +; CHECK-BF16-NEXT: ext z3.b, z3.b, z1.b, #64 +; CHECK-BF16-NEXT: fcvtx z2.s, p0/m, z2.d +; CHECK-BF16-NEXT: fcvtx z1.s, p0/m, z1.d +; CHECK-BF16-NEXT: bfcvt z0.h, p0/m, z0.s +; CHECK-BF16-NEXT: fcvtx z3.s, p0/m, z3.d +; CHECK-BF16-NEXT: bfcvt z2.h, p0/m, z2.s +; CHECK-BF16-NEXT: bfcvt z1.h, p0/m, z1.s +; CHECK-BF16-NEXT: uzp1 z0.s, z0.s, z0.s +; CHECK-BF16-NEXT: bfcvt z3.h, p0/m, z3.s +; CHECK-BF16-NEXT: uzp1 z2.s, z2.s, z2.s +; CHECK-BF16-NEXT: uzp1 z1.s, z1.s, z1.s +; CHECK-BF16-NEXT: uzp1 z0.h, z0.h, z0.h +; CHECK-BF16-NEXT: uzp1 z3.s, z3.s, z3.s +; CHECK-BF16-NEXT: uzp1 z2.h, z2.h, z2.h +; CHECK-BF16-NEXT: uzp1 z1.h, z1.h, z1.h +; CHECK-BF16-NEXT: uzp1 z3.h, z3.h, z3.h +; CHECK-BF16-NEXT: stp q0, q2, [x1] +; CHECK-BF16-NEXT: stp q1, q3, [x1, #32] +; CHECK-BF16-NEXT: ret + %op1 = load <32 x double>, ptr %a + %res = fptrunc <32 x double> %op1 to <32 x bfloat> + store <32 x bfloat> %res, ptr %b + ret void +} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; CHECK: {{.*}} _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
