https://github.com/pvimal816a updated https://github.com/llvm/llvm-project/pull/238442
>From bdc35d4f52211f51f9fa8ff802331026552b90f2 Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Wed, 4 Nov 2026 13:55:44 +0000 Subject: [PATCH 1/7] [AArch64] Add patterns to combine SVE AND and XOR to BIC Add ISel patterns to rewrite the following pattern to BIC: ``` return svand_u32_x(svptrue_b32(), a, sveor_u32_x(svptrue_b32(), a, b)); ``` Cover cases where atleast one of AND and XOR node is of _u form or is a normal LLVM AND/XOR node. Also, combine sve.and.u and sve.eor(.z) when predicate for sve.and.u matches that of sve.eor(.z) or the predicate of sve.eor(.z) is an all-active predicate. Cover all SVE integer widths and cover the following symmetric case: ``` return sveor_u32_x(svptrue_b32(), a, svand_u32_x(svptrue_b32(), a, b)); ``` --- .../Target/AArch64/AArch64ISelLowering.cpp | 4 +- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 123 ++++++++++++++++-- .../CodeGen/AArch64/sve-intrinsics-and-xor.ll | 45 ++++--- .../CodeGen/AArch64/sve-intrinsics-xor-and.ll | 45 ++++--- 4 files changed, 169 insertions(+), 48 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index a51e6bb18cdd8e..19214f4469c23c 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25890,8 +25890,8 @@ static SDValue performIntrinsicCombine(SDNode *N, return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_eor_u: - return DAG.getNode(ISD::XOR, SDLoc(N), N->getValueType(0), N->getOperand(2), - N->getOperand(3)); + return DAG.getNode(AArch64ISD::EOR_U, SDLoc(N), N->getValueType(0), + N->getOperand(1), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_orr_u: return DAG.getNode(ISD::OR, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 86898c3139c69c..7c51d5a2ffd19d 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -252,6 +252,7 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [ // Predicated operations with the result of inactive lanes being unspecified. def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>; +def AArch64eor_u : SDNode<"AArch64ISD::EOR_U", SDT_AArch64Arith>; def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>; def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>; def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>; @@ -560,10 +561,17 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (and node:$op1, (xor node:$op2, (SVEAllActive))), (AArch64bic_node node:$op1, node:$op2)]>; -// Match AND_U with either data operand order, keeping the predicate unchanged. -def SVEAndU : PatFrags<(ops node:$pg, node:$a, node:$b), - [(AArch64and_u node:$pg, node:$a, node:$b), - (AArch64and_u node:$pg, node:$b, node:$a)]>; +// Match `intr` with either data operand order, keeping the predicate unchanged. +class CommPredBinPatFrags<SDPatternOperator intr> + : PatFrags<(ops node:$pg, node:$a, node:$b), + [(intr node:$pg, node:$a, node:$b), + (intr node:$pg, node:$b, node:$a)]>; +def SVEAndU : CommPredBinPatFrags<AArch64and_u>; +def SVEAndM : CommPredBinPatFrags<int_aarch64_sve_and>; +def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>; +def SVEEorU : CommPredBinPatFrags<AArch64eor_u>; +def SVEEorM : CommPredBinPatFrags<int_aarch64_sve_eor>; +def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>; def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), [(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2), @@ -799,6 +807,92 @@ let Predicates = [HasSVE_or_SME] in { defm : SVEAndNotPats<nxv4i32, nxv4i1>; defm : SVEAndNotPats<nxv2i64, nxv2i1>; + multiclass SVEAndXorPats<ValueType VT, ValueType PT> { + // Lanes inactive in EOR may be replaced by a & ~b in result. + def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)), + (BIC_ZZZ $a, $b)>; + + // Inactive lanes in AND or EOR may be replaced by a & ~b. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Similarly, for normal LLVM XOR. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (xor VT:$a, VT:$b))), + (BIC_ZZZ $a, $b)>; + + // Matching predicate ensures that the active lanes in EOR can be + // replaced by a & ~b. + def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + (VT (SVEEorM PT:$pg, VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Similarly, for _z form of EOR. + def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + (VT (SVEEorZ PT:$pg, VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AllActive predicate in EOR ensures that the active lane in the + // result is a & ~b. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (SVEEorM (PT (SVEAllActive)), VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Similarly, for _z form of EOR. + def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + (VT (SVEEorZ (PT (SVEAllActive)), VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + } + defm : SVEAndXorPats<nxv16i8, nxv16i1>; + defm : SVEAndXorPats<nxv8i16, nxv8i1>; + defm : SVEAndXorPats<nxv4i32, nxv4i1>; + defm : SVEAndXorPats<nxv2i64, nxv2i1>; + + multiclass SVEXorAndPats<ValueType VT, ValueType PT> { + // Lanes inactive in AND may be replaced by a & ~b in result. + def : Pat<(xor VT:$a, (SVEAndU (PT srcvalue), VT:$a, VT:$b)), + (BIC_ZZZ $a, $b)>; + + // Inactive lanes in EOR or AND may be replaced by a & ~b. + def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, + (VT (SVEAndU (PT srcvalue), VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Similarly, for normal LLVM AND. + def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, (and VT:$a, VT:$b))), + (BIC_ZZZ $a, $b)>; + + // Matching predicate ensures that the active lanes in EOR can be + // replaced by a & ~b. + def : Pat<(VT (SVEEorU PT:$pg, VT:$a, + (VT (SVEAndM PT:$pg, VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Similarly, for _z form of AND. + def : Pat<(VT (SVEEorU PT:$pg, VT:$a, + (VT (SVEAndZ PT:$pg, VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // AllActive predicate in AND node ensures that the active lane + // in the result may be replaced by a & ~b. + def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, + (VT (SVEAndM (PT (SVEAllActive)), VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Similarly, for _z form of AND. + def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, + (VT (SVEAndZ (PT (SVEAllActive)), VT:$a, VT:$b)))), + (BIC_ZZZ $a, $b)>; + + // Fallback. + def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)), + (EOR_ZZZ $a, $b)>; + } + defm : SVEXorAndPats<nxv16i8, nxv16i1>; + defm : SVEXorAndPats<nxv8i16, nxv8i1>; + defm : SVEXorAndPats<nxv4i32, nxv4i1>; + defm : SVEXorAndPats<nxv2i64, nxv2i1>; + defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>; defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">; defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>; @@ -888,16 +982,21 @@ let Predicates = [HasSVE_or_SME] in { defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>; defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>; - // Allow AND_U nodes to select AND-immediate instructions. - class SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, - ValueType IT, ComplexPattern CP> - : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), + // Allow AND_U and EOR_U nodes to select AND-immediate/EOR-immediate + // instructions. + multiclass SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, + ValueType IT, ComplexPattern CP> { + def : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), (VT (splat_vector (IT (CP i64:$imm)))))), (AND_ZI_PSEUDO $a, i64:$imm)>; - def : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; - def : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; - def : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; - def : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; + def : Pat<(VT (SVEEorU (PT (SVEAnyPredicate)), (VT ZPR:$a), + (VT (splat_vector (IT (CP i64:$imm)))))), + (EOR_ZI_PSEUDO $a, i64:$imm)>; + } + defm : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; + defm : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; + defm : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; + defm : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll index 9c7d88f111b0d0..4555d2c1c4b931 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-and-xor.ll @@ -6,8 +6,7 @@ define <vscale x 4 x i32> @and_eor_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: and_eor_u: ; CHECK: // %bb.0: -; CHECK-NEXT: eor z1.d, z0.d, z1.d -; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %and = and <vscale x 4 x i32> %a, %eor @@ -17,8 +16,7 @@ define <vscale x 4 x i32> @and_eor_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> % define <vscale x 4 x i32> @and_u_eor_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: and_u_eor_u: ; CHECK: // %bb.0: -; CHECK-NEXT: eor z1.d, z0.d, z1.d -; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg1, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor) @@ -28,8 +26,7 @@ define <vscale x 4 x i32> @and_u_eor_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> define <vscale x 4 x i32> @and_u_xor(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: and_u_xor: ; CHECK: // %bb.0: -; CHECK-NEXT: eor z1.d, z0.d, z1.d -; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %xor = xor <vscale x 4 x i32> %a, %b %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %xor) @@ -39,9 +36,7 @@ define <vscale x 4 x i32> @and_u_xor(<vscale x 4 x i1> %pg, <vscale x 4 x i32> % define <vscale x 4 x i32> @and_u_eor_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: and_u_eor_m: ; CHECK: // %bb.0: -; CHECK-NEXT: mov z2.d, z0.d -; CHECK-NEXT: eor z2.s, p0/m, z2.s, z1.s -; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor) @@ -51,8 +46,7 @@ define <vscale x 4 x i32> @and_u_eor_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> define <vscale x 4 x i32> @and_u_eor_m_commuted(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: and_u_eor_m_commuted: ; CHECK: // %bb.0: -; CHECK-NEXT: eor z1.s, p0/m, z1.s, z0.s -; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %a) %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor) @@ -83,13 +77,20 @@ define <vscale x 4 x i32> @and_eor_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> % ret <vscale x 4 x i32> %and } +define <vscale x 4 x i32> @and_u_eor_z(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { +; CHECK-LABEL: and_u_eor_z: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.z.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) + %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor) + ret <vscale x 4 x i32> %and +} + define <vscale x 4 x i32> @and_u_eor_m_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: and_u_eor_m_all_active: ; CHECK: // %bb.0: -; CHECK-NEXT: mov z2.d, z0.d -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: eor z2.s, p0/m, z2.s, z1.s -; CHECK-NEXT: and z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor) @@ -166,11 +167,20 @@ define <vscale x 4 x i32> @and_m_eor_m_all_active(<vscale x 4 x i1> %pg, <vscale ret <vscale x 4 x i32> %and } +define <vscale x 4 x i32> @and_u_eor_z_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { +; CHECK-LABEL: and_u_eor_z_all_active: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.z.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) + %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %eor) + ret <vscale x 4 x i32> %and +} + define <vscale x 16 x i8> @and_u_xor_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) { ; CHECK-LABEL: and_u_xor_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: eor z1.d, z0.d, z1.d -; CHECK-NEXT: and z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %xor = xor <vscale x 16 x i8> %a, %b %and = tail call <vscale x 16 x i8> @llvm.aarch64.sve.and.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %xor) @@ -218,6 +228,7 @@ declare <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1>, <v declare <vscale x 2 x i64> @llvm.aarch64.sve.and.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>) declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>) declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>) +declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.z.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>) ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; SME: {{.*}} ; SVE: {{.*}} diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll index 9ca1260dc7c405..8f52d72db262ea 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-xor-and.ll @@ -6,8 +6,7 @@ define <vscale x 4 x i32> @xor_and_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: xor_and_u: ; CHECK: // %bb.0: -; CHECK-NEXT: and z1.d, z0.d, z1.d -; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %xor = xor <vscale x 4 x i32> %a, %and @@ -17,8 +16,7 @@ define <vscale x 4 x i32> @xor_and_u(<vscale x 4 x i1> %pg, <vscale x 4 x i32> % define <vscale x 4 x i32> @eor_u_and_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: eor_u_and_u: ; CHECK: // %bb.0: -; CHECK-NEXT: and z1.d, z0.d, z1.d -; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.u.nxv4i32(<vscale x 4 x i1> %pg1, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) @@ -28,8 +26,7 @@ define <vscale x 4 x i32> @eor_u_and_u(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> define <vscale x 4 x i32> @eor_u_and(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: eor_u_and: ; CHECK: // %bb.0: -; CHECK-NEXT: and z1.d, z0.d, z1.d -; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = and <vscale x 4 x i32> %a, %b %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) @@ -39,9 +36,7 @@ define <vscale x 4 x i32> @eor_u_and(<vscale x 4 x i1> %pg, <vscale x 4 x i32> % define <vscale x 4 x i32> @eor_u_and_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: eor_u_and_m: ; CHECK: // %bb.0: -; CHECK-NEXT: mov z2.d, z0.d -; CHECK-NEXT: and z2.s, p0/m, z2.s, z1.s -; CHECK-NEXT: eor z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) @@ -51,14 +46,23 @@ define <vscale x 4 x i32> @eor_u_and_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> define <vscale x 4 x i32> @eor_u_and_m_commuted(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: eor_u_and_m_commuted: ; CHECK: // %bb.0: -; CHECK-NEXT: and z1.s, p0/m, z1.s, z0.s -; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %b, <vscale x 4 x i32> %a) %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) ret <vscale x 4 x i32> %eor } +define <vscale x 4 x i32> @eor_u_and_z(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { +; CHECK-LABEL: eor_u_and_z: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.z.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) + %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) + ret <vscale x 4 x i32> %eor +} + define <vscale x 4 x i32> @eor_u_and_m_pred_mismatch(<vscale x 4 x i1> %pg1, <vscale x 4 x i1> %pg2, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: eor_u_and_m_pred_mismatch: ; CHECK: // %bb.0: @@ -86,10 +90,7 @@ define <vscale x 4 x i32> @xor_and_m(<vscale x 4 x i1> %pg, <vscale x 4 x i32> % define <vscale x 4 x i32> @eor_u_and_m_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { ; CHECK-LABEL: eor_u_and_m_all_active: ; CHECK: // %bb.0: -; CHECK-NEXT: mov z2.d, z0.d -; CHECK-NEXT: ptrue p0.s -; CHECK-NEXT: and z2.s, p0/m, z2.s, z1.s -; CHECK-NEXT: eor z0.d, z0.d, z2.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) @@ -166,11 +167,20 @@ define <vscale x 4 x i32> @eor_m_and_m_all_active(<vscale x 4 x i1> %pg, <vscale ret <vscale x 4 x i32> %eor } +define <vscale x 4 x i32> @eor_u_and_z_all_active(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) { +; CHECK-LABEL: eor_u_and_z_all_active: +; CHECK: // %bb.0: +; CHECK-NEXT: bic z0.d, z0.d, z1.d +; CHECK-NEXT: ret + %and = tail call <vscale x 4 x i32> @llvm.aarch64.sve.and.z.nxv4i32(<vscale x 4 x i1> splat (i1 true), <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) + %eor = tail call <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %and) + ret <vscale x 4 x i32> %eor +} + define <vscale x 16 x i8> @eor_u_and_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) { ; CHECK-LABEL: eor_u_and_i8: ; CHECK: // %bb.0: -; CHECK-NEXT: and z1.d, z0.d, z1.d -; CHECK-NEXT: eor z0.d, z0.d, z1.d +; CHECK-NEXT: bic z0.d, z0.d, z1.d ; CHECK-NEXT: ret %and = and <vscale x 16 x i8> %a, %b %eor = tail call <vscale x 16 x i8> @llvm.aarch64.sve.eor.u.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %and) @@ -218,6 +228,7 @@ declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.u.nxv4i32(<vscale x 4 x i1>, <v declare <vscale x 4 x i32> @llvm.aarch64.sve.eor.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>) declare <vscale x 4 x i32> @llvm.aarch64.sve.and.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>) declare <vscale x 2 x i64> @llvm.aarch64.sve.eor.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>, <vscale x 2 x i64>) +declare <vscale x 4 x i32> @llvm.aarch64.sve.and.z.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>, <vscale x 4 x i32>) ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; SME: {{.*}} ; SVE: {{.*}} >From 992d7afb777755bff7f270e22365718cee2e906d Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Mon, 9 Nov 2026 09:34:01 +0000 Subject: [PATCH 2/7] fixup! [AArch64] Add patterns to combine SVE AND and XOR to BIC --- llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 4 ++-- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 19214f4469c23c..9e9be9da5eaa95 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -25884,13 +25884,13 @@ static SDValue performIntrinsicCombine(SDNode *N, case Intrinsic::aarch64_sve_subr: return convertMergedOpToPredOp(N, ISD::SUB, DAG, true, true); case Intrinsic::aarch64_sve_and_u: - return DAG.getNode(AArch64ISD::AND_U, SDLoc(N), N->getValueType(0), + return DAG.getNode(AArch64ISD::AND_PRED, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_bic_u: return DAG.getNode(AArch64ISD::BIC, SDLoc(N), N->getValueType(0), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_eor_u: - return DAG.getNode(AArch64ISD::EOR_U, SDLoc(N), N->getValueType(0), + return DAG.getNode(AArch64ISD::EOR_PRED, SDLoc(N), N->getValueType(0), N->getOperand(1), N->getOperand(2), N->getOperand(3)); case Intrinsic::aarch64_sve_orr_u: return DAG.getNode(ISD::OR, SDLoc(N), N->getValueType(0), N->getOperand(2), diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 7c51d5a2ffd19d..925d27406c69c8 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -251,8 +251,8 @@ def SDT_AArch64FMA : SDTypeProfile<1, 4, [ ]>; // Predicated operations with the result of inactive lanes being unspecified. -def AArch64and_u : SDNode<"AArch64ISD::AND_U", SDT_AArch64Arith>; -def AArch64eor_u : SDNode<"AArch64ISD::EOR_U", SDT_AArch64Arith>; +def AArch64and_p : SDNode<"AArch64ISD::AND_PRED", SDT_AArch64Arith>; +def AArch64eor_p : SDNode<"AArch64ISD::EOR_PRED", SDT_AArch64Arith>; def AArch64asr_p : SDNode<"AArch64ISD::SRA_PRED", SDT_AArch64Arith>; def AArch64lsl_p : SDNode<"AArch64ISD::SHL_PRED", SDT_AArch64Arith>; def AArch64lsr_p : SDNode<"AArch64ISD::SRL_PRED", SDT_AArch64Arith>; @@ -566,10 +566,10 @@ class CommPredBinPatFrags<SDPatternOperator intr> : PatFrags<(ops node:$pg, node:$a, node:$b), [(intr node:$pg, node:$a, node:$b), (intr node:$pg, node:$b, node:$a)]>; -def SVEAndU : CommPredBinPatFrags<AArch64and_u>; +def SVEAndU : CommPredBinPatFrags<AArch64and_p>; def SVEAndM : CommPredBinPatFrags<int_aarch64_sve_and>; def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>; -def SVEEorU : CommPredBinPatFrags<AArch64eor_u>; +def SVEEorU : CommPredBinPatFrags<AArch64eor_p>; def SVEEorM : CommPredBinPatFrags<int_aarch64_sve_eor>; def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>; @@ -799,7 +799,7 @@ let Predicates = [HasSVE_or_SME] in { // Fall back to register AND when no BIC pattern matches. // AND_U allows any result in inactive lanes. - def : Pat<(VT (AArch64and_u (PT srcvalue), VT:$a, VT:$b)), + def : Pat<(VT (AArch64and_p (PT srcvalue), VT:$a, VT:$b)), (AND_ZZZ $a, $b)>; } defm : SVEAndNotPats<nxv16i8, nxv16i1>; >From 04be5b5a88e63f393c8e450c5a0251dd8ef61ca6 Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Mon, 9 Nov 2026 09:38:51 +0000 Subject: [PATCH 3/7] fixup! [AArch64] Add patterns to combine SVE AND and XOR to BIC Reorganise placement of patterns and multiclasses. --- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 98 ++++++++----------- llvm/lib/Target/AArch64/SVEInstrFormats.td | 29 ++++++ 2 files changed, 71 insertions(+), 56 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 925d27406c69c8..04995c93bb4293 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -561,16 +561,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (and node:$op1, (xor node:$op2, (SVEAllActive))), (AArch64bic_node node:$op1, node:$op2)]>; -// Match `intr` with either data operand order, keeping the predicate unchanged. -class CommPredBinPatFrags<SDPatternOperator intr> - : PatFrags<(ops node:$pg, node:$a, node:$b), - [(intr node:$pg, node:$a, node:$b), - (intr node:$pg, node:$b, node:$a)]>; +// Match the specified binary op with either order of data operands. def SVEAndU : CommPredBinPatFrags<AArch64and_p>; -def SVEAndM : CommPredBinPatFrags<int_aarch64_sve_and>; def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>; def SVEEorU : CommPredBinPatFrags<AArch64eor_p>; -def SVEEorM : CommPredBinPatFrags<int_aarch64_sve_eor>; def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>; def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), @@ -660,9 +654,9 @@ def AArch64famax_p : PatFrags<(ops node:$pred, node:$op1, node:$op2), def AArch64add_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_add, add>; def AArch64sub_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_sub, sub>; def AArch64mul_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_mul, AArch64mul_p>; -def AArch64and_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_and, and>; -def AArch64orr_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_orr, or>; -def AArch64eor_m1 : VSelectUnpredOrPassthruPatFrags<int_aarch64_sve_eor, xor>; +def AArch64and_m1 : VSelectCommUnpredOrPassthruPatFrags<int_aarch64_sve_and, and>; +def AArch64orr_m1 : VSelectCommUnpredOrPassthruPatFrags<int_aarch64_sve_orr, or>; +def AArch64eor_m1 : VSelectCommUnpredOrPassthruPatFrags<int_aarch64_sve_eor, xor>; def AArch64smax_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_smax, AArch64smax_p>; def AArch64umax_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_umax, AArch64umax_p>; def AArch64smin_m1 : VSelectCommPredOrPassthruPatFrags<int_aarch64_sve_smin, AArch64smin_p>; @@ -767,6 +761,18 @@ let Predicates = [HasSVE_or_SME] in { defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>; defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>; + defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>; + defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">; + defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>; + + defm ORR_ZPmZ : sve_int_bin_pred_log<0b000, "orr", AArch64orr_m1, DestructiveBinaryComm>; + defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", AArch64eor_m1, DestructiveBinaryComm>; + defm AND_ZPmZ : sve_int_bin_pred_log<0b010, "and", AArch64and_m1, DestructiveBinaryComm>; + defm BIC_ZPmZ : sve_int_bin_pred_log<0b011, "bic", int_aarch64_sve_bic, DestructiveBinary>; +} // End HasSVE_or_SME + +// Rewrites specific to selecting BIC_ZZZ. +let Predicates = [HasSVE_or_SME] in { multiclass SVEAndNotPats<ValueType VT, ValueType PT> { // Ordinary AND: NOT's undefined inactive lanes may take the value ~b. def : Pat<(and VT:$a, @@ -807,6 +813,26 @@ let Predicates = [HasSVE_or_SME] in { defm : SVEAndNotPats<nxv4i32, nxv4i1>; defm : SVEAndNotPats<nxv2i64, nxv2i1>; + multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT, + Instruction PredBic> { + // Preserve A in inactive lanes. NOT must cover every active AND lane. + def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), + (PredBic $pg, $a, $b)>; + def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (PredBic $pg, $a, $b)>; + + // With both predicates all-active, no inactive lanes need preserving. + def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a, + (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), + (BIC_ZZZ $a, $b)>; + } + defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; + defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; + defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; + defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; + multiclass SVEAndXorPats<ValueType VT, ValueType PT> { // Lanes inactive in EOR may be replaced by a & ~b in result. def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)), @@ -824,7 +850,7 @@ let Predicates = [HasSVE_or_SME] in { // Matching predicate ensures that the active lanes in EOR can be // replaced by a & ~b. def : Pat<(VT (SVEAndU PT:$pg, VT:$a, - (VT (SVEEorM PT:$pg, VT:$a, VT:$b)))), + (VT (AArch64eor_m1 PT:$pg, VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of EOR. @@ -835,7 +861,7 @@ let Predicates = [HasSVE_or_SME] in { // AllActive predicate in EOR ensures that the active lane in the // result is a & ~b. def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, - (VT (SVEEorM (PT (SVEAllActive)), VT:$a, VT:$b)))), + (VT (AArch64eor_m1 (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of EOR. @@ -865,7 +891,7 @@ let Predicates = [HasSVE_or_SME] in { // Matching predicate ensures that the active lanes in EOR can be // replaced by a & ~b. def : Pat<(VT (SVEEorU PT:$pg, VT:$a, - (VT (SVEAndM PT:$pg, VT:$a, VT:$b)))), + (VT (AArch64and_m1 PT:$pg, VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of AND. @@ -876,7 +902,7 @@ let Predicates = [HasSVE_or_SME] in { // AllActive predicate in AND node ensures that the active lane // in the result may be replaced by a & ~b. def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, - (VT (SVEAndM (PT (SVEAllActive)), VT:$a, VT:$b)))), + (VT (AArch64and_m1 (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of AND. @@ -892,35 +918,6 @@ let Predicates = [HasSVE_or_SME] in { defm : SVEXorAndPats<nxv8i16, nxv8i1>; defm : SVEXorAndPats<nxv4i32, nxv4i1>; defm : SVEXorAndPats<nxv2i64, nxv2i1>; - - defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>; - defm SUB_ZPmZ : sve_int_bin_pred_arit_0<0b001, "sub", AArch64sub_m1, DestructiveBinaryCommWithRev, "SUBR_ZPmZ">; - defm SUBR_ZPmZ : sve_int_bin_pred_arit_0<0b011, "subr", AArch64subr_m1, DestructiveBinaryCommWithRev, "SUB_ZPmZ", /*isReverseInstr*/ 1>; - - defm ORR_ZPmZ : sve_int_bin_pred_log<0b000, "orr", AArch64orr_m1, DestructiveBinaryComm>; - defm EOR_ZPmZ : sve_int_bin_pred_log<0b001, "eor", AArch64eor_m1, DestructiveBinaryComm>; - defm AND_ZPmZ : sve_int_bin_pred_log<0b010, "and", AArch64and_m1, DestructiveBinaryComm>; - defm BIC_ZPmZ : sve_int_bin_pred_log<0b011, "bic", int_aarch64_sve_bic, DestructiveBinary>; - - multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT, - Instruction PredBic> { - // Preserve A in inactive lanes. NOT must cover every active AND lane. - def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, - (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), - (PredBic $pg, $a, $b)>; - def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, - (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), - (PredBic $pg, $a, $b)>; - - // With both predicates all-active, no inactive lanes need preserving. - def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a, - (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), - (BIC_ZZZ $a, $b)>; - } - defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; - defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; - defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; - defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; } // End HasSVE_or_SME let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in { @@ -984,19 +981,8 @@ let Predicates = [HasSVE_or_SME] in { // Allow AND_U and EOR_U nodes to select AND-immediate/EOR-immediate // instructions. - multiclass SVEAndUImmPat<ValueType VT, ValueType PT, ZPRRegOp ZPR, - ValueType IT, ComplexPattern CP> { - def : Pat<(VT (SVEAndU (PT (SVEAnyPredicate)), (VT ZPR:$a), - (VT (splat_vector (IT (CP i64:$imm)))))), - (AND_ZI_PSEUDO $a, i64:$imm)>; - def : Pat<(VT (SVEEorU (PT (SVEAnyPredicate)), (VT ZPR:$a), - (VT (splat_vector (IT (CP i64:$imm)))))), - (EOR_ZI_PSEUDO $a, i64:$imm)>; - } - defm : SVEAndUImmPat<nxv16i8, nxv16i1, ZPR8, i32, SVELogicalImm8Pat>; - defm : SVEAndUImmPat<nxv8i16, nxv8i1, ZPR16, i32, SVELogicalImm16Pat>; - defm : SVEAndUImmPat<nxv4i32, nxv4i1, ZPR32, i32, SVELogicalImm32Pat>; - defm : SVEAndUImmPat<nxv2i64, nxv2i1, ZPR64, i64, SVELogicalImm64Pat>; + defm : sve_log_dupimm_any_pred<SVEAndU, AND_ZI_PSEUDO>; + defm : sve_log_dupimm_any_pred<SVEEorU, EOR_ZI_PSEUDO>; defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index 7e684c43e2ccfa..0b1da3f72c74f6 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -734,6 +734,12 @@ class SVE_Shift_DupImm_Any_Predicate_Pat<ValueType vt, SDPatternOperator op, : Pat<(vt (op (pt (SVEAnyPredicate)), vt:$Rn, (vt (splat_vector (it (cast i32:$imm)))))), (inst $Rn, i32:$imm)>; +class SVE_Log_DupImm_Any_Pred_Pat<ValueType vt, SDPatternOperator op, + ValueType pt, ZPRRegOp zprty, ValueType it, + ComplexPattern cast, Instruction inst> +: Pat<(vt (op (pt (SVEAnyPredicate)), (vt zprty:$Rn), (vt (splat_vector (it (cast i64:$imm)))))), + (inst $Rn, i64:$imm)>; + class SVE_2_Op_Imm_Pat_Zero<ValueType vt, SDPatternOperator op, ValueType pt, ValueType it, ComplexPattern cpx, Instruction inst> : Pat<(vt (op pt:$Pg, (vselect pt:$Pg, vt:$Op1, (SVEDup0)), @@ -805,6 +811,22 @@ class VSelectUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOper ], [{ return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse(); }]>; +// Same as above with a commutative operation +class VSelectCommUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPatternOperator sdnode> +: PatFrags<(ops node:$Pg, node:$Op1, node:$Op2), [ + (intrinsic node:$Pg, node:$Op1, node:$Op2), + (intrinsic node:$Pg, node:$Op2, node:$Op1), + (vselect node:$Pg, (sdnode node:$Op1, node:$Op2), node:$Op1), + (vselect node:$Pg, (sdnode node:$Op2, node:$Op1), node:$Op1) + ], [{ + return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse(); + }]>; + +// Match `intr` with either data operand order, keeping the predicate unchanged. +class CommPredBinPatFrags<SDPatternOperator intr> + : PatFrags<(ops node:$pg, node:$a, node:$b), + [(intr node:$pg, node:$a, node:$b), + (intr node:$pg, node:$b, node:$a)]>; // // Pseudo -> Instruction mappings @@ -2189,6 +2211,13 @@ multiclass sve_int_log_imm_bic<SDPatternOperator op> { def : SVE_1_Op_Imm_Log_Pat<nxv2i64, op, ZPR64, i64, SVELogicalImm64NotPat, !cast<Instruction>("AND_ZI_PSEUDO")>; } +multiclass sve_log_dupimm_any_pred<SDPatternOperator op, Instruction inst> { + def : SVE_Log_DupImm_Any_Pred_Pat<nxv16i8, op, nxv16i1, ZPR8, i32, SVELogicalImm8Pat, inst>; + def : SVE_Log_DupImm_Any_Pred_Pat<nxv8i16, op, nxv8i1, ZPR16, i32, SVELogicalImm16Pat, inst>; + def : SVE_Log_DupImm_Any_Pred_Pat<nxv4i32, op, nxv4i1, ZPR32, i32, SVELogicalImm32Pat, inst>; + def : SVE_Log_DupImm_Any_Pred_Pat<nxv2i64, op, nxv2i1, ZPR64, i64, SVELogicalImm64Pat, inst>; +} + class sve_int_dup_mask_imm<string asm> : I<(outs ZPR64:$Zd), (ins logical_imm64:$imms), asm, "\t$Zd, $imms", >From 47f17f4ff0d57a00703b0174af056768e9f0ac54 Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Mon, 9 Nov 2026 09:51:53 +0000 Subject: [PATCH 4/7] fixup! [AArch64] Add patterns to combine SVE AND and XOR to BIC Rename CommPredBinPatFrags -> PredicatedCommutativeBinOpPatFrags to make the name more informative. --- llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td | 8 ++++---- llvm/lib/Target/AArch64/SVEInstrFormats.td | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 04995c93bb4293..0a129df2d8829a 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -562,10 +562,10 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (AArch64bic_node node:$op1, node:$op2)]>; // Match the specified binary op with either order of data operands. -def SVEAndU : CommPredBinPatFrags<AArch64and_p>; -def SVEAndZ : CommPredBinPatFrags<int_aarch64_sve_and_z>; -def SVEEorU : CommPredBinPatFrags<AArch64eor_p>; -def SVEEorZ : CommPredBinPatFrags<int_aarch64_sve_eor_z>; +def SVEAndU : PredicatedCommutativeBinOpPatFrags<AArch64and_p>; +def SVEAndZ : PredicatedCommutativeBinOpPatFrags<int_aarch64_sve_and_z>; +def SVEEorU : PredicatedCommutativeBinOpPatFrags<AArch64eor_p>; +def SVEEorZ : PredicatedCommutativeBinOpPatFrags<int_aarch64_sve_eor_z>; def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), [(int_aarch64_sve_and_z node:$pg, node:$op1, node:$op2), diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td index 0b1da3f72c74f6..37be38731c63a1 100644 --- a/llvm/lib/Target/AArch64/SVEInstrFormats.td +++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td @@ -822,11 +822,11 @@ class VSelectCommUnpredOrPassthruPatFrags<SDPatternOperator intrinsic, SDPattern return N->getOpcode() != ISD::VSELECT || N->getOperand(1).hasOneUse(); }]>; -// Match `intr` with either data operand order, keeping the predicate unchanged. -class CommPredBinPatFrags<SDPatternOperator intr> +// Match `op` with either data operand order, keeping the predicate unchanged. +class PredicatedCommutativeBinOpPatFrags<SDPatternOperator op> : PatFrags<(ops node:$pg, node:$a, node:$b), - [(intr node:$pg, node:$a, node:$b), - (intr node:$pg, node:$b, node:$a)]>; + [(op node:$pg, node:$a, node:$b), + (op node:$pg, node:$b, node:$a)]>; // // Pseudo -> Instruction mappings >From a7711087ad5552f5c280c8615b2b9c7e4c086b9a Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Mon, 9 Nov 2026 10:47:26 +0000 Subject: [PATCH 5/7] fixup! [AArch64] Add patterns to combine SVE AND and XOR to BIC Use proper naming conventions for the newly added multiclasses. --- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 0a129df2d8829a..de2c6a9f471e38 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -773,7 +773,7 @@ let Predicates = [HasSVE_or_SME] in { // Rewrites specific to selecting BIC_ZZZ. let Predicates = [HasSVE_or_SME] in { - multiclass SVEAndNotPats<ValueType VT, ValueType PT> { + multiclass sve_and_not_pats<ValueType VT, ValueType PT> { // Ordinary AND: NOT's undefined inactive lanes may take the value ~b. def : Pat<(and VT:$a, (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))), @@ -808,12 +808,12 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (AArch64and_p (PT srcvalue), VT:$a, VT:$b)), (AND_ZZZ $a, $b)>; } - defm : SVEAndNotPats<nxv16i8, nxv16i1>; - defm : SVEAndNotPats<nxv8i16, nxv8i1>; - defm : SVEAndNotPats<nxv4i32, nxv4i1>; - defm : SVEAndNotPats<nxv2i64, nxv2i1>; + defm : sve_and_not_pats<nxv16i8, nxv16i1>; + defm : sve_and_not_pats<nxv8i16, nxv8i1>; + defm : sve_and_not_pats<nxv4i32, nxv4i1>; + defm : sve_and_not_pats<nxv2i64, nxv2i1>; - multiclass SVEMergingAndNotPats<ValueType VT, ValueType PT, + multiclass sve_merging_and_not_pats<ValueType VT, ValueType PT, Instruction PredBic> { // Preserve A in inactive lanes. NOT must cover every active AND lane. def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, @@ -828,12 +828,12 @@ let Predicates = [HasSVE_or_SME] in { (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), (BIC_ZZZ $a, $b)>; } - defm : SVEMergingAndNotPats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; - defm : SVEMergingAndNotPats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; - defm : SVEMergingAndNotPats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; - defm : SVEMergingAndNotPats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; + defm : sve_merging_and_not_pats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; + defm : sve_merging_and_not_pats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; + defm : sve_merging_and_not_pats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; + defm : sve_merging_and_not_pats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; - multiclass SVEAndXorPats<ValueType VT, ValueType PT> { + multiclass sve_and_xor_pats<ValueType VT, ValueType PT> { // Lanes inactive in EOR may be replaced by a & ~b in result. def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)), (BIC_ZZZ $a, $b)>; @@ -869,12 +869,12 @@ let Predicates = [HasSVE_or_SME] in { (VT (SVEEorZ (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; } - defm : SVEAndXorPats<nxv16i8, nxv16i1>; - defm : SVEAndXorPats<nxv8i16, nxv8i1>; - defm : SVEAndXorPats<nxv4i32, nxv4i1>; - defm : SVEAndXorPats<nxv2i64, nxv2i1>; + defm : sve_and_xor_pats<nxv16i8, nxv16i1>; + defm : sve_and_xor_pats<nxv8i16, nxv8i1>; + defm : sve_and_xor_pats<nxv4i32, nxv4i1>; + defm : sve_and_xor_pats<nxv2i64, nxv2i1>; - multiclass SVEXorAndPats<ValueType VT, ValueType PT> { + multiclass sve_xor_and_pats<ValueType VT, ValueType PT> { // Lanes inactive in AND may be replaced by a & ~b in result. def : Pat<(xor VT:$a, (SVEAndU (PT srcvalue), VT:$a, VT:$b)), (BIC_ZZZ $a, $b)>; @@ -914,10 +914,10 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)), (EOR_ZZZ $a, $b)>; } - defm : SVEXorAndPats<nxv16i8, nxv16i1>; - defm : SVEXorAndPats<nxv8i16, nxv8i1>; - defm : SVEXorAndPats<nxv4i32, nxv4i1>; - defm : SVEXorAndPats<nxv2i64, nxv2i1>; + defm : sve_xor_and_pats<nxv16i8, nxv16i1>; + defm : sve_xor_and_pats<nxv8i16, nxv8i1>; + defm : sve_xor_and_pats<nxv4i32, nxv4i1>; + defm : sve_xor_and_pats<nxv2i64, nxv2i1>; } // End HasSVE_or_SME let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in { >From 7a619d3d87b888df21d611af4f7d76198434f3ed Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Thu, 12 Nov 2026 10:48:07 +0000 Subject: [PATCH 6/7] fixup! [AArch64] Add patterns to combine SVE AND and XOR to BIC --- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index de2c6a9f471e38..2f7dd29d750b9b 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -773,7 +773,9 @@ let Predicates = [HasSVE_or_SME] in { // Rewrites specific to selecting BIC_ZZZ. let Predicates = [HasSVE_or_SME] in { - multiclass sve_and_not_pats<ValueType VT, ValueType PT> { + multiclass sve_log_bic_pats<ValueType VT, ValueType PT, Instruction PredBic> { + // ===== AND-NOT patterns ===== + // Ordinary AND: NOT's undefined inactive lanes may take the value ~b. def : Pat<(and VT:$a, (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b))), @@ -803,18 +805,8 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), (BIC_ZZZ $a, $b)>; - // Fall back to register AND when no BIC pattern matches. - // AND_U allows any result in inactive lanes. - def : Pat<(VT (AArch64and_p (PT srcvalue), VT:$a, VT:$b)), - (AND_ZZZ $a, $b)>; - } - defm : sve_and_not_pats<nxv16i8, nxv16i1>; - defm : sve_and_not_pats<nxv8i16, nxv8i1>; - defm : sve_and_not_pats<nxv4i32, nxv4i1>; - defm : sve_and_not_pats<nxv2i64, nxv2i1>; + // ===== AND-NOT patterns for _m form of AND ===== - multiclass sve_merging_and_not_pats<ValueType VT, ValueType PT, - Instruction PredBic> { // Preserve A in inactive lanes. NOT must cover every active AND lane. def : Pat<(VT (int_aarch64_sve_and PT:$pg, VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), @@ -827,13 +819,9 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (int_aarch64_sve_and (PT (SVEAllActive)), VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), (BIC_ZZZ $a, $b)>; - } - defm : sve_merging_and_not_pats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; - defm : sve_merging_and_not_pats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; - defm : sve_merging_and_not_pats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; - defm : sve_merging_and_not_pats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; - multiclass sve_and_xor_pats<ValueType VT, ValueType PT> { + // ===== AND-XOR patterns ===== + // Lanes inactive in EOR may be replaced by a & ~b in result. def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)), (BIC_ZZZ $a, $b)>; @@ -868,13 +856,9 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (VT (SVEEorZ (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; - } - defm : sve_and_xor_pats<nxv16i8, nxv16i1>; - defm : sve_and_xor_pats<nxv8i16, nxv8i1>; - defm : sve_and_xor_pats<nxv4i32, nxv4i1>; - defm : sve_and_xor_pats<nxv2i64, nxv2i1>; - multiclass sve_xor_and_pats<ValueType VT, ValueType PT> { + // ===== XOR-AND patterns ===== + // Lanes inactive in AND may be replaced by a & ~b in result. def : Pat<(xor VT:$a, (SVEAndU (PT srcvalue), VT:$a, VT:$b)), (BIC_ZZZ $a, $b)>; @@ -909,17 +893,23 @@ let Predicates = [HasSVE_or_SME] in { def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, (VT (SVEAndZ (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; - - // Fallback. - def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)), - (EOR_ZZZ $a, $b)>; } - defm : sve_xor_and_pats<nxv16i8, nxv16i1>; - defm : sve_xor_and_pats<nxv8i16, nxv8i1>; - defm : sve_xor_and_pats<nxv4i32, nxv4i1>; - defm : sve_xor_and_pats<nxv2i64, nxv2i1>; + defm : sve_log_bic_pats<nxv16i8, nxv16i1, BIC_ZPmZ_B>; + defm : sve_log_bic_pats<nxv8i16, nxv8i1, BIC_ZPmZ_H>; + defm : sve_log_bic_pats<nxv4i32, nxv4i1, BIC_ZPmZ_S>; + defm : sve_log_bic_pats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; } // End HasSVE_or_SME +let Predicates = [HasSVE_or_SME] in { + // Fallback patterns for predicated AND and EOR nodes. + foreach I = !range(0, 4) in { + defvar VT = [nxv16i8, nxv8i16, nxv4i32, nxv2i64][I]; + defvar PT = [nxv16i1, nxv8i1, nxv4i1, nxv2i1][I]; + def : Pat<(VT (AArch64and_p (PT srcvalue), VT:$a, VT:$b)), (AND_ZZZ $a, $b)>; + def : Pat<(VT (AArch64eor_p (PT srcvalue), VT:$a, VT:$b)), (EOR_ZZZ $a, $b)>; + } +} + let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in { defm ADD_ZPZZ : sve_int_bin_pred_zeroing_bhsd<"ADD_ZPmZ", int_aarch64_sve_add>; defm SUB_ZPZZ : sve_int_bin_pred_zeroing_bhsd<"SUB_ZPmZ", int_aarch64_sve_sub>; @@ -979,7 +969,7 @@ let Predicates = [HasSVE_or_SME] in { defm AND_ZI : sve_int_log_imm<0b10, "and", "bic", and>; defm BIC_ZI : sve_int_log_imm_bic<AArch64bic>; - // Allow AND_U and EOR_U nodes to select AND-immediate/EOR-immediate + // Allow and_p/eor_p nodes to select AND-immediate/EOR-immediate // instructions. defm : sve_log_dupimm_any_pred<SVEAndU, AND_ZI_PSEUDO>; defm : sve_log_dupimm_any_pred<SVEEorU, EOR_ZI_PSEUDO>; >From f20306217410a44a44e7f42d007a7a19e5e50ca7 Mon Sep 17 00:00:00 2001 From: Vimal Patel <[email protected]> Date: Thu, 19 Nov 2026 10:25:37 +0000 Subject: [PATCH 7/7] fixup! [AArch64] Add patterns to combine SVE AND and XOR to BIC Resolve review comments. --- .../lib/Target/AArch64/AArch64SVEInstrInfo.td | 77 +++++++++---------- 1 file changed, 37 insertions(+), 40 deletions(-) diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td index 2f7dd29d750b9b..2dacb454fbd332 100644 --- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td +++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td @@ -562,9 +562,9 @@ def AArch64bic : PatFrags<(ops node:$op1, node:$op2), (AArch64bic_node node:$op1, node:$op2)]>; // Match the specified binary op with either order of data operands. -def SVEAndU : PredicatedCommutativeBinOpPatFrags<AArch64and_p>; +def SVEAndP : PredicatedCommutativeBinOpPatFrags<AArch64and_p>; def SVEAndZ : PredicatedCommutativeBinOpPatFrags<int_aarch64_sve_and_z>; -def SVEEorU : PredicatedCommutativeBinOpPatFrags<AArch64eor_p>; +def SVEEorP : PredicatedCommutativeBinOpPatFrags<AArch64eor_p>; def SVEEorZ : PredicatedCommutativeBinOpPatFrags<int_aarch64_sve_eor_z>; def AArch64and_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), @@ -589,6 +589,13 @@ def AArch64orr_z : PatFrags<(ops node:$pg, node:$op1, node:$op2), [(int_aarch64_sve_orr_z node:$pg, node:$op1, node:$op2), (and node:$pg, (or node:$op1, node:$op2))]>; +def AArch64and : PatFrags<(ops node:$op1, node:$op2), + [(AArch64and_p srcvalue, node:$op1, node:$op2), + (and node:$op1, node:$op2)]>; +def AArch64eor : PatFrags<(ops node:$op1, node:$op2), + [(AArch64eor_p srcvalue, node:$op1, node:$op2), + (xor node:$op1, node:$op2)]>; + def AArch64subr : PatFrag<(ops node:$op1, node:$op2), (sub node:$op2, node:$op1)>; @@ -756,9 +763,9 @@ let Predicates = [HasSVE_or_SME] in { defm SQSUB_ZZZ : sve_int_bin_cons_arit_0<0b110, "sqsub", ssubsat>; defm UQSUB_ZZZ : sve_int_bin_cons_arit_0<0b111, "uqsub", usubsat>; - defm AND_ZZZ : sve_int_bin_cons_log<0b00, "and", and>; + defm AND_ZZZ : sve_int_bin_cons_log<0b00, "and", AArch64and>; defm ORR_ZZZ : sve_int_bin_cons_log<0b01, "orr", or>; - defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", xor>; + defm EOR_ZZZ : sve_int_bin_cons_log<0b10, "eor", AArch64eor>; defm BIC_ZZZ : sve_int_bin_cons_log<0b11, "bic", AArch64bic>; defm ADD_ZPmZ : sve_int_bin_pred_arit_0<0b000, "add", AArch64add_m1, DestructiveBinaryComm>; @@ -785,24 +792,24 @@ let Predicates = [HasSVE_or_SME] in { (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b))), (BIC_ZZZ $a, $b)>; - // AND_U: matching predicates ensure NOT produces ~b in every active AND lane. - def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + // AND_P: matching predicates ensure NOT produces ~b in every active AND lane. + def : Pat<(VT (SVEAndP PT:$pg, VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), PT:$pg, VT:$b)))), (BIC_ZZZ $a, $b)>; - // AND_U: NOT's undefined inactive lanes may take the value ~b, + // AND_P: NOT's undefined inactive lanes may take the value ~b, // even when the predicates differ. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + def : Pat<(VT (SVEAndP (PT srcvalue), VT:$a, (VT (int_aarch64_sve_not (VT (undef)), (PT srcvalue), VT:$b)))), (BIC_ZZZ $a, $b)>; - // AND_U: an all-active NOT produces ~b regardless of AND_U's predicate. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + // AND_P: an all-active NOT produces ~b regardless of AND_P's predicate. + def : Pat<(VT (SVEAndP (PT srcvalue), VT:$a, (VT (int_aarch64_sve_not (VT srcvalue), (PT (SVEAllActive)), VT:$b)))), (BIC_ZZZ $a, $b)>; - // AND_U with ordinary LLVM NOT, which produces ~b in every lane. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (vnot VT:$b))), + // AND_P with ordinary LLVM NOT, which produces ~b in every lane. + def : Pat<(VT (SVEAndP (PT srcvalue), VT:$a, (vnot VT:$b))), (BIC_ZZZ $a, $b)>; // ===== AND-NOT patterns for _m form of AND ===== @@ -823,74 +830,74 @@ let Predicates = [HasSVE_or_SME] in { // ===== AND-XOR patterns ===== // Lanes inactive in EOR may be replaced by a & ~b in result. - def : Pat<(and VT:$a, (SVEEorU (PT srcvalue), VT:$a, VT:$b)), + def : Pat<(and VT:$a, (SVEEorP (PT (SVEAnyPredicate)), VT:$a, VT:$b)), (BIC_ZZZ $a, $b)>; // Inactive lanes in AND or EOR may be replaced by a & ~b. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, - (VT (SVEEorU (PT srcvalue), VT:$a, VT:$b)))), + def : Pat<(VT (SVEAndP (PT srcvalue), VT:$a, + (VT (SVEEorP (PT srcvalue), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for normal LLVM XOR. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, (xor VT:$a, VT:$b))), + def : Pat<(VT (SVEAndP (PT (SVEAnyPredicate)), VT:$a, (xor VT:$a, VT:$b))), (BIC_ZZZ $a, $b)>; // Matching predicate ensures that the active lanes in EOR can be // replaced by a & ~b. - def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + def : Pat<(VT (SVEAndP PT:$pg, VT:$a, (VT (AArch64eor_m1 PT:$pg, VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of EOR. - def : Pat<(VT (SVEAndU PT:$pg, VT:$a, + def : Pat<(VT (SVEAndP PT:$pg, VT:$a, (VT (SVEEorZ PT:$pg, VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // AllActive predicate in EOR ensures that the active lane in the // result is a & ~b. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + def : Pat<(VT (SVEAndP (PT (SVEAnyPredicate)), VT:$a, (VT (AArch64eor_m1 (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of EOR. - def : Pat<(VT (SVEAndU (PT srcvalue), VT:$a, + def : Pat<(VT (SVEAndP (PT (SVEAnyPredicate)), VT:$a, (VT (SVEEorZ (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // ===== XOR-AND patterns ===== // Lanes inactive in AND may be replaced by a & ~b in result. - def : Pat<(xor VT:$a, (SVEAndU (PT srcvalue), VT:$a, VT:$b)), + def : Pat<(xor VT:$a, (SVEAndP (PT (SVEAnyPredicate)), VT:$a, VT:$b)), (BIC_ZZZ $a, $b)>; // Inactive lanes in EOR or AND may be replaced by a & ~b. - def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, - (VT (SVEAndU (PT srcvalue), VT:$a, VT:$b)))), + def : Pat<(VT (SVEEorP (PT srcvalue), VT:$a, + (VT (SVEAndP (PT srcvalue), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for normal LLVM AND. - def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, (and VT:$a, VT:$b))), + def : Pat<(VT (SVEEorP (PT (SVEAnyPredicate)), VT:$a, (and VT:$a, VT:$b))), (BIC_ZZZ $a, $b)>; // Matching predicate ensures that the active lanes in EOR can be // replaced by a & ~b. - def : Pat<(VT (SVEEorU PT:$pg, VT:$a, + def : Pat<(VT (SVEEorP PT:$pg, VT:$a, (VT (AArch64and_m1 PT:$pg, VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of AND. - def : Pat<(VT (SVEEorU PT:$pg, VT:$a, + def : Pat<(VT (SVEEorP PT:$pg, VT:$a, (VT (SVEAndZ PT:$pg, VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // AllActive predicate in AND node ensures that the active lane // in the result may be replaced by a & ~b. - def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, + def : Pat<(VT (SVEEorP (PT (SVEAnyPredicate)), VT:$a, (VT (AArch64and_m1 (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; // Similarly, for _z form of AND. - def : Pat<(VT (SVEEorU (PT srcvalue), VT:$a, + def : Pat<(VT (SVEEorP (PT (SVEAnyPredicate)), VT:$a, (VT (SVEAndZ (PT (SVEAllActive)), VT:$a, VT:$b)))), (BIC_ZZZ $a, $b)>; } @@ -900,16 +907,6 @@ let Predicates = [HasSVE_or_SME] in { defm : sve_log_bic_pats<nxv2i64, nxv2i1, BIC_ZPmZ_D>; } // End HasSVE_or_SME -let Predicates = [HasSVE_or_SME] in { - // Fallback patterns for predicated AND and EOR nodes. - foreach I = !range(0, 4) in { - defvar VT = [nxv16i8, nxv8i16, nxv4i32, nxv2i64][I]; - defvar PT = [nxv16i1, nxv8i1, nxv4i1, nxv2i1][I]; - def : Pat<(VT (AArch64and_p (PT srcvalue), VT:$a, VT:$b)), (AND_ZZZ $a, $b)>; - def : Pat<(VT (AArch64eor_p (PT srcvalue), VT:$a, VT:$b)), (EOR_ZZZ $a, $b)>; - } -} - let Predicates = [HasSVE_or_SME, UseExperimentalZeroingPseudos] in { defm ADD_ZPZZ : sve_int_bin_pred_zeroing_bhsd<"ADD_ZPmZ", int_aarch64_sve_add>; defm SUB_ZPZZ : sve_int_bin_pred_zeroing_bhsd<"SUB_ZPmZ", int_aarch64_sve_sub>; @@ -971,8 +968,8 @@ let Predicates = [HasSVE_or_SME] in { // Allow and_p/eor_p nodes to select AND-immediate/EOR-immediate // instructions. - defm : sve_log_dupimm_any_pred<SVEAndU, AND_ZI_PSEUDO>; - defm : sve_log_dupimm_any_pred<SVEEorU, EOR_ZI_PSEUDO>; + defm : sve_log_dupimm_any_pred<SVEAndP, AND_ZI_PSEUDO>; + defm : sve_log_dupimm_any_pred<SVEEorP, EOR_ZI_PSEUDO>; defm SMAX_ZI : sve_int_arith_imm1<0b00, "smax", AArch64smax_p>; defm SMIN_ZI : sve_int_arith_imm1<0b10, "smin", AArch64smin_p>; _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
