https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232527
>From 4faa90d2e55d96debc529fbc406bcd48604be490 Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Thu, 15 Oct 2026 22:23:45 +0100 Subject: [PATCH 1/2] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _m forms Replace NOT followed by merging AND with BIC for expressions such as: return svand_u32_m(pg, a, svnot_u32_x(pg, b)); Combine when NOT is the second operand and its predicate matches the AND predicate or is all-active. Use predicated BIC to preserve the AND's first operand in inactive lanes. Use unpredicated BIC when the AND predicate is all-active. Do not commute merging AND operands, since this would change inactive results. Leave pairs with incompatible predicates unchanged. This applies to all integer element widths. Update the pre-commit LLVM expectations, including cases where combining would change the result. --- .../Target/AArch64/AArch64ISelLowering.cpp | 33 ++++++++++ .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 60 ++++--------------- 2 files changed, 45 insertions(+), 48 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index f806a3223ed4eb..fb1e39aeb1b348 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25623,6 +25623,37 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } +static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { + bool IsMerging = getIntrinsicID(N) == Intrinsic::aarch64_sve_and; + SDLoc DL(N); + EVT VT = N->getValueType(0); + SDValue Pg = N->getOperand(1); + + // Only commute AND_U: a merging AND must preserve its first operand in + // inactive lanes. NOT must cover all of AND's active lanes. + for (unsigned I = 0, E = IsMerging ? 1 : 2; I != E; ++I) { + unsigned NotOperandIdx = 3 - I; + unsigned OtherOperandIdx = 2 + I; + SDValue Not = N->getOperand(NotOperandIdx); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || + (Not.getOperand(2) != Pg && + !isAllActivePredicate(DAG, Not.getOperand(2)))) + continue; + + SDValue A = N->getOperand(OtherOperandIdx); + SDValue B = Not.getOperand(3); + if (!IsMerging || isAllActivePredicate(DAG, Pg)) + return DAG.getNode(AArch64ISD::BIC, DL, VT, A, B); + + return DAG.getNode( + ISD::INTRINSIC_WO_CHAIN, DL, VT, + DAG.getConstant(Intrinsic::aarch64_sve_bic, DL, MVT::i32), Pg, A, B); + } + if (IsMerging) + return SDValue(); + return DAG.getNode(ISD::AND, DL, VT, N->getOperand(2), N->getOperand(3)); +} + static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -25841,6 +25872,8 @@ static SDValue performIntrinsicCombine(SDNode *N, N->getOperand(3)); case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); + case Intrinsic::aarch64_sve_and: + return performSVEAndNotIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_and_u: return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index 13b4baa8c05549..1f58c974d8380a 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -148,8 +148,7 @@ define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.s, p0/m, z0.s, z2.s +; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) @@ -159,8 +158,7 @@ define <vscale x 4 x i32> @merging_and_rhs(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b, <vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.b, p0/m, z1.b -; CHECK-NEXT: and z0.b, p0/m, z0.b, z2.b +; CHECK-NEXT: bic z0.b, p0/m, z0.b, z1.b ; CHECK-NEXT: ret %not = call <vscale x 16 x i8> @llvm.aarch64.sve.not.nxv16i8(<vscale x 16 x i8> %inactive, <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b) %and = call <vscale x 16 x i8> @llvm.aarch64.sve.and.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %not) @@ -170,8 +168,7 @@ define <vscale x 16 x i8> @merging_and_rhs_i8(<vscale x 16 x i8> %a, <vscale x 1 define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b, <vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs_i16: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.h, p0/m, z1.h -; CHECK-NEXT: and z0.h, p0/m, z0.h, z2.h +; CHECK-NEXT: bic z0.h, p0/m, z0.h, z1.h ; CHECK-NEXT: ret %not = call <vscale x 8 x i16> @llvm.aarch64.sve.not.nxv8i16(<vscale x 8 x i16> %inactive, <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b) %and = call <vscale x 8 x i16> @llvm.aarch64.sve.and.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %not) @@ -181,8 +178,7 @@ define <vscale x 8 x i16> @merging_and_rhs_i16(<vscale x 8 x i16> %a, <vscale x define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b, <vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg) { ; CHECK-LABEL: merging_and_rhs_i64: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.d, p0/m, z1.d -; CHECK-NEXT: and z0.d, p0/m, z0.d, z2.d +; CHECK-NEXT: bic z0.d, p0/m, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 2 x i64> @llvm.aarch64.sve.not.nxv2i64(<vscale x 2 x i64> %inactive, <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b) %and = call <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %not) @@ -190,52 +186,20 @@ define <vscale x 2 x i64> @merging_and_rhs_i64(<vscale x 2 x i64> %a, <vscale x } define <vscale x 4 x i32> @merging_and_all_active_not(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { -; SVE-LABEL: merging_and_all_active_not: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p1.s -; SVE-NEXT: not z1.s, p1/m, z1.s -; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE-NEXT: ret -; -; SVE2P2-LABEL: merging_and_all_active_not: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p1.s -; SVE2P2-NEXT: not z1.s, p1/z, z1.s -; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE2P2-NEXT: ret -; -; SME-LABEL: merging_and_all_active_not: -; SME: // %bb.0: -; SME-NEXT: ptrue p1.s -; SME-NEXT: not z1.s, p1/m, z1.s -; SME-NEXT: and z0.s, p0/m, z0.s, z1.s -; SME-NEXT: ret +; CHECK-LABEL: merging_and_all_active_not: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.s, p0/m, z0.s, z1.s +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and } define <vscale x 4 x i32> @merging_and_all_active(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) { -; SVE-LABEL: merging_and_all_active: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE-NEXT: ret -; -; SVE2P2-LABEL: merging_and_all_active: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.s, p0/m, z0.s, z1.s -; SVE2P2-NEXT: ret -; -; SME-LABEL: merging_and_all_active: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.s, p0/m, z0.s, z1.s -; SME-NEXT: ret +; CHECK-LABEL: merging_and_all_active: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and >From a1142a7fb67eb923df8260dd8f2e819656fe1f1f Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 21 Oct 2026 16:32:26 +0100 Subject: [PATCH 2/2] fixup! [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _m forms Select merging AND-NOT operations with TableGen patterns instead of the C++ combine. Preserve the first operand in inactive lanes and use an unpredicated BIC when both predicates are all-active. --- .../Target/AArch64/AArch64ISelLowering.cpp | 33 ------------------- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 20 +++++++++++ 2 files changed, 20 insertions(+), 33 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index fb1e39aeb1b348..f806a3223ed4eb 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25623,37 +25623,6 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } -static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { - bool IsMerging = getIntrinsicID(N) == Intrinsic::aarch64_sve_and; - SDLoc DL(N); - EVT VT = N->getValueType(0); - SDValue Pg = N->getOperand(1); - - // Only commute AND_U: a merging AND must preserve its first operand in - // inactive lanes. NOT must cover all of AND's active lanes. - for (unsigned I = 0, E = IsMerging ? 1 : 2; I != E; ++I) { - unsigned NotOperandIdx = 3 - I; - unsigned OtherOperandIdx = 2 + I; - SDValue Not = N->getOperand(NotOperandIdx); - if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || - (Not.getOperand(2) != Pg && - !isAllActivePredicate(DAG, Not.getOperand(2)))) - continue; - - SDValue A = N->getOperand(OtherOperandIdx); - SDValue B = Not.getOperand(3); - if (!IsMerging || isAllActivePredicate(DAG, Pg)) - return DAG.getNode(AArch64ISD::BIC, DL, VT, A, B); - - return DAG.getNode( - ISD::INTRINSIC_WO_CHAIN, DL, VT, - DAG.getConstant(Intrinsic::aarch64_sve_bic, DL, MVT::i32), Pg, A, B); - } - if (IsMerging) - return SDValue(); - return DAG.getNode(ISD::AND, DL, VT, N->getOperand(2), N->getOperand(3)); -} - static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -25872,8 +25841,6 @@ static SDValue performIntrinsicCombine(SDNode *N, N->getOperand(3)); case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); - case Intrinsic::aarch64_sve_and: - return performSVEAndNotIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_and_u: return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 889c5c4f7eee30..b8376d6d906a3b 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -807,6 +807,26 @@ let Predicates = [HasSVE_or_SME] in { defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", AArch64eor_m1, DestructiveBinaryComm>; defm AND_ZPmZ : sve_int_bin_pred_log<0b010, "and", AArch64and_m1, DestructiveBinaryComm>; defm BIC_ZPmZ : sve_int_bin_pred_log<0b011, "bic", int_aarch64_sve_bic, DestructiveBinary>; + + multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT, + Instruction PredBic> { + // Preserve A in inactive lanes. NOT must cover every active AND lane. + def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), + (PredBic $pg, $a, $b)>; + def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (PredBic $pg, $a, $b)>; + + // With both predicates all-active, no inactive lanes need preserving. + def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (BIC_ZZZ $a, $b)>; + } + defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; + defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; + defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; + defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; } // End HasSVE_or_SME let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in { _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
