https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/232526
>From 8324795e551639fd503593b9048d97e536f650af Mon Sep 17 00:00:00 2001 From: Jonathan Thackray <[email protected]> Date: Wed, 14 Oct 2026 23:22:09 +0100 Subject: [PATCH] [AArch64][llvm] Combine SVE AND-NOT operations into BIC for _x forms Replace NOT followed by AND with BIC for expressions such as: return svand_u32_x(svptrue_b32(), a, svnot_u32_x(svptrue_b32(), b)); Combine ordinary AND with NOT when the NOT predicate is all-active or its inactive values are undefined. Also combine matching-predicate SVE AND_x-NOT pairs before lowering discards the AND predicate. AND_x ignores inactive results, so the NOT's inactive values do not need preserving. This applies to all integer element widths. Update the pre-commit LLVM expectations, including cases where combining would change the result. --- .../Target/AArch64/AArch64ISelLowering.cpp | 36 ++++- .../CodeGen/AArch64/sve-intrinsics-and-not.ll | 123 ++++-------------- 2 files changed, 56 insertions(+), 103 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index e3257af644550..bc3ded382f070 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -22123,6 +22123,19 @@ static SDValue performReinterpretCastCombine(SDNode *N) { static SDValue performSVEAndCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { SelectionDAG &DAG = DCI.DAG; + + // Fold AND of a predicated NOT into BIC when NOT's inactive lanes do not + // matter, leaving standalone NOTs available for predicated selection. + for (unsigned I = 0; I != 2; ++I) { + SDValue Not = N->getOperand(I); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not) + continue; + if (Not.getOperand(1).isUndef() || + isAllActivePredicate(DAG, Not.getOperand(2))) + return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), + N->getOperand(1 - I), Not.getOperand(3)); + } + SDValue Src = N->getOperand(0); unsigned Opc = Src->getOpcode(); @@ -25381,6 +25394,26 @@ static SDValue lowerPredCounterWhile(Intrinsic::ID IID, SDNode *N, DAG.getTargetConstant(ElementSizeInBits, DL, MVT::i64)); } +static SDValue performSVEAndNotIntrinsicCombine(SDNode *N, SelectionDAG &DAG) { + SDValue Pg = N->getOperand(1); + + // Fold matching predicates before lowering AND_U discards its predicate. + // NOT's inactive lanes cannot affect AND_U's active lanes. + for (unsigned I = 0; I != 2; ++I) { + unsigned NotOperandIdx = 2 + I; + unsigned OtherOperandIdx = 3 - I; + SDValue Not = N->getOperand(NotOperandIdx); + if (getIntrinsicID(Not.getNode()) != Intrinsic::aarch64_sve_not || + Not.getOperand(2) != Pg) + continue; + + return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), + N->getOperand(OtherOperandIdx), Not.getOperand(3)); + } + return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), + N->getOperand(3)); +} + static SDValue performIntrinsicCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { @@ -25600,8 +25633,7 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), N->getOperand(2), - N->getOperand(3)); + return performSVEAndNotIntrinsicCombine(N, DAG); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll index cde00ee9f59d6..66ecfa31e01f2 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-not.ll @@ -6,26 +6,10 @@ ; Ordinary LLVM AND with SVE NOT. define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive) { -; SVE-LABEL: and_not_ptrue_all: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_ptrue_all: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_ptrue_all: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_ptrue_all: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %pg = call <vscale x 4 x i1> @llvm.aarch64.sve.ptrue.nxv4i1(i32 31) %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %not @@ -33,49 +17,20 @@ define <vscale x 4 x i32> @and_not_ptrue_all(<vscale x 4 x i32> %a, <vscale x 4 } define <vscale x 4 x i32> @and_not_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { -; SVE-LABEL: and_not_commuted: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z1.d, z0.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_commuted: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z1.d, z0.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_commuted: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z1.d, z0.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_commuted: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %not, %a ret <vscale x 4 x i32> %and } define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i1> %pg) { -; SVE-LABEL: and_not_poison_i32: -; SVE: // %bb.0: -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_poison_i32: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_poison_i32: -; SME: // %bb.0: -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_poison_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %not ret <vscale x 4 x i32> %and @@ -85,26 +40,10 @@ define <vscale x 4 x i32> @and_not_poison_i32(<vscale x 4 x i32> %a, <vscale x 4 ; irrelevant to AND_U. An all-active NOT can also fold with any AND predicate. define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { -; SVE-LABEL: and_not_all_i32: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: and_not_all_i32: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: and_not_all_i32: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: and_not_all_i32: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> poison, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and @@ -113,8 +52,7 @@ define <vscale x 4 x i32> @and_not_all_i32(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: matching_defined: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) @@ -124,8 +62,7 @@ define <vscale x 4 x i32> @matching_defined(<vscale x 4 x i32> %a, <vscale x 4 x define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { ; CHECK-LABEL: matching_commuted: ; CHECK: // %bb.0: -; CHECK-NEXT: not z2.s, p0/m, z1.s -; CHECK-NEXT: and z0.d, z2.d, z0.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %not, <vscale x 4 x i32> %a) @@ -133,26 +70,10 @@ define <vscale x 4 x i32> @matching_commuted(<vscale x 4 x i32> %a, <vscale x 4 } define <vscale x 4 x i32> @all_not_partial_and(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b, <vscale x 4 x i32> %inactive, <vscale x 4 x i1> %pg) { -; SVE-LABEL: all_not_partial_and: -; SVE: // %bb.0: -; SVE-NEXT: ptrue p0.s -; SVE-NEXT: not z1.s, p0/m, z1.s -; SVE-NEXT: and z0.d, z0.d, z1.d -; SVE-NEXT: ret -; -; SVE2P2-LABEL: all_not_partial_and: -; SVE2P2: // %bb.0: -; SVE2P2-NEXT: ptrue p0.s -; SVE2P2-NEXT: not z1.s, p0/z, z1.s -; SVE2P2-NEXT: and z0.d, z0.d, z1.d -; SVE2P2-NEXT: ret -; -; SME-LABEL: all_not_partial_and: -; SME: // %bb.0: -; SME-NEXT: ptrue p0.s -; SME-NEXT: not z1.s, p0/m, z1.s -; SME-NEXT: and z0.d, z0.d, z1.d -; SME-NEXT: ret +; CHECK-LABEL: all_not_partial_and: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret %not = call <vscale x 4 x i32> @llvm.aarch64.sve.not.nxv4i32(<vscale x 4 x i32> %inactive, <vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %b) %and = call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %not) ret <vscale x 4 x i32> %and _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
