https://github.com/paulwalker-arm created https://github.com/llvm/llvm-project/pull/218667
This makes the casting behaviour more explicit: > to/from_svbool : lane-count changing casts > to/from_svcount : reinterpretation casts This is the final piece after https://github.com/llvm/llvm-project/issues/217256 and https://github.com/llvm/llvm-project/issues/217360. >From 2716c6735b589cc21f11c5444890fa43760d458e Mon Sep 17 00:00:00 2001 From: Paul Walker <[email protected]> Date: Tue, 18 Aug 2026 17:27:45 +0000 Subject: [PATCH] [LLVM][SVE] Add dedicated intrinsics to cast between svcount_t and svbool_t. This makes the casting behaviour more explicit: to/from_svbool : lane-count changing casts to/from_svcount : reinterpretation casts --- clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 8 ++----- .../acle_sme2_reinterpret_svcount_svbool.c | 8 +++---- llvm/include/llvm/IR/IntrinsicsAArch64.td | 16 ++++++++----- llvm/lib/IR/AutoUpgrade.cpp | 22 ++++++++++++++++++ .../Target/AArch64/AArch64ISelLowering.cpp | 7 +++--- .../Bitcode/upgrade-aarch64-sve-intrinsics.ll | 18 +++++++++++++++ .../AArch64/sve-intrinsics-reinterpret.ll | 23 ++----------------- 7 files changed, 61 insertions(+), 41 deletions(-) diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp index 5cddeef5e0f37..f247226aafa07 100644 --- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp +++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp @@ -4064,17 +4064,13 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, return nullptr; case SVE::BI__builtin_sve_svreinterpret_b: { - auto SVCountTy = - llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount"); Function *CastFromSVCountF = - CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy); + CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svcount); return Builder.CreateCall(CastFromSVCountF, Ops[0]); } case SVE::BI__builtin_sve_svreinterpret_c: { - auto SVCountTy = - llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount"); Function *CastToSVCountF = - CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy); + CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svcount); return Builder.CreateCall(CastToSVCountF, Ops[0]); } diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c index 4c059e350a182..abffa8c7be3b3 100644 --- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c +++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c @@ -26,12 +26,12 @@ // CHECK-LABEL: @test_svreinterpret_svbool_svcnt( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]]) +// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svcount(target("aarch64.svcount") [[CNT:%.*]]) // CHECK-NEXT: ret <vscale x 16 x i1> [[TMP0]] // // CPP-CHECK-LABEL: @_Z31test_svreinterpret_svbool_svcntu11__SVCount_t( // CPP-CHECK-NEXT: entry: -// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]]) +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svcount(target("aarch64.svcount") [[CNT:%.*]]) // CPP-CHECK-NEXT: ret <vscale x 16 x i1> [[TMP0]] // svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) MODE_ATTR @@ -41,12 +41,12 @@ svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) MODE_ATTR // CHECK-LABEL: @test_svreinterpret_svcnt_svbool( // CHECK-NEXT: entry: -// CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[PG:%.*]]) +// CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.to.svcount(<vscale x 16 x i1> [[PG:%.*]]) // CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // // CPP-CHECK-LABEL: @_Z31test_svreinterpret_svcnt_svboolu10__SVBool_t( // CPP-CHECK-NEXT: entry: -// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> [[PG:%.*]]) +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.to.svcount(<vscale x 16 x i1> [[PG:%.*]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] // svcount_t test_svreinterpret_svcnt_svbool(svbool_t pg) MODE_ATTR diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td index 8fbe68f98cc63..8356bbb8640a1 100644 --- a/llvm/include/llvm/IR/IntrinsicsAArch64.td +++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td @@ -2339,13 +2339,17 @@ def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic<[IntrSpeculatable] // Reinterpreting data // -def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_any_ty], - [llvm_nxv16i1_ty], - [IntrNoMem, IntrSpeculatable]>; +def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic< + [llvm_anyvector_ty], [llvm_nxv16i1_ty], [IntrNoMem, IntrSpeculatable]>; -def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], - [llvm_any_ty], - [IntrNoMem, IntrSpeculatable]>; +def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic< + [llvm_nxv16i1_ty], [llvm_anyvector_ty], [IntrNoMem, IntrSpeculatable]>; + +def int_aarch64_sve_convert_from_svcount : DefaultAttrsIntrinsic< + [llvm_nxv16i1_ty], [llvm_aarch64_svcount_ty], [IntrNoMem, IntrSpeculatable]>; + +def int_aarch64_sve_convert_to_svcount : DefaultAttrsIntrinsic< + [llvm_aarch64_svcount_ty], [llvm_nxv16i1_ty], [IntrNoMem, IntrSpeculatable]>; // // Gather loads: scalar base + vector offsets diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 9865816e36959..59366f69b2709 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -1047,6 +1047,28 @@ static bool upgradeArmOrAarch64IntrinsicFunction(bool IsArm, Function *F, return true; } + if (Name.consume_front("convert.from.svbool")) { + // 'aarch64.sve.convert.from.svbool' + auto *TTy = dyn_cast<TargetExtType>(F->getReturnType()); + if (!TTy || TTy->getName() != "aarch64.svcount") + return false; + + Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_to_svcount; + NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID); + return true; + } + + if (Name.consume_front("convert.to.svbool")) { + // 'aarch64.sve.convert.to.svbool' + auto *TTy = dyn_cast<TargetExtType>(F->arg_begin()->getType()); + if (!TTy || TTy->getName() != "aarch64.svcount") + return false; + + Intrinsic::ID ID = Intrinsic::aarch64_sve_convert_from_svcount; + NewFn = Intrinsic::getOrInsertDeclaration(F->getParent(), ID); + return true; + } + if (Name.consume_front("addqv")) { // 'aarch64.sve.addqv'. if (!F->getReturnType()->isFPOrFPVectorTy()) diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index fc743256bc1a3..c5af54a97fe32 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -7034,13 +7034,12 @@ SDValue AArch64TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, case Intrinsic::aarch64_sve_dupq_lane: return LowerDUPQLane(Op, DAG); case Intrinsic::aarch64_sve_convert_from_svbool: - if (Op.getValueType() == MVT::aarch64svcount) - return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Op.getOperand(1)); return getSVEPredicateBitCast(Op.getValueType(), Op.getOperand(1), DAG); case Intrinsic::aarch64_sve_convert_to_svbool: - if (Op.getOperand(1).getValueType() == MVT::aarch64svcount) - return DAG.getNode(ISD::BITCAST, DL, MVT::nxv16i1, Op.getOperand(1)); return getSVEPredicateBitCast(MVT::nxv16i1, Op.getOperand(1), DAG); + case Intrinsic::aarch64_sve_convert_from_svcount: + case Intrinsic::aarch64_sve_convert_to_svcount: + return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Op.getOperand(1)); case Intrinsic::aarch64_sve_fneg: return DAG.getNode(AArch64ISD::FNEG_MERGE_PASSTHRU, DL, Op.getValueType(), Op.getOperand(2), Op.getOperand(3), Op.getOperand(1)); diff --git a/llvm/test/Bitcode/upgrade-aarch64-sve-intrinsics.ll b/llvm/test/Bitcode/upgrade-aarch64-sve-intrinsics.ll index 832c2c4e072f4..7be91c47b1e2d 100644 --- a/llvm/test/Bitcode/upgrade-aarch64-sve-intrinsics.ll +++ b/llvm/test/Bitcode/upgrade-aarch64-sve-intrinsics.ll @@ -207,6 +207,24 @@ define <vscale x 4 x float> @bfmmla_f32(<vscale x 4 x float> %a, <vscale x 8 x b ret <vscale x 4 x float> %out } +declare target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1>) +define target("aarch64.svcount") @convert_svbool_to_svcount(<vscale x 16 x i1> %pg) "target-features"="+sme2" { +; CHECK-LABEL: @convert_svbool_to_svcount +; CHECK: %out = call target("aarch64.svcount") @llvm.aarch64.sve.convert.to.svcount(<vscale x 16 x i1> %pg) +; CHECK-NEXT: ret target("aarch64.svcount") %out + %out = call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> %pg) + ret target("aarch64.svcount") %out +} + +declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount")) +define <vscale x 16 x i1> @convert_svcount_svbool(target("aarch64.svcount") %pg) "target-features"="+sme2" { +; CHECK-LABEL: @convert_svcount_svbool +; CHECK: %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svcount(target("aarch64.svcount") %pg) +; CHECK-NEXT: ret <vscale x 16 x i1> %out + %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") %pg) + ret <vscale x 16 x i1> %out +} + declare <vscale x 32 x i8> @llvm.aarch64.sve.tuple.create2.nxv32i8.nxv16i8(<vscale x 16 x i8>, <vscale x 16 x i8>) declare <vscale x 32 x i8> @llvm.aarch64.sve.tuple.create2.nxv32i8(<vscale x 16 x i8>, <vscale x 16 x i8>) declare <vscale x 32 x i8> @llvm.aarch64.sve.tuple.create2(<vscale x 16 x i8>, <vscale x 16 x i8>) diff --git a/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll index 73a8a5e38ddb4..366ec95ebc658 100644 --- a/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll +++ b/llvm/test/CodeGen/AArch64/sve-intrinsics-reinterpret.ll @@ -59,7 +59,7 @@ define <vscale x 16 x i1> @reinterpret_bool_from_svcount(target("aarch64.svcount ; CHECK-LABEL: reinterpret_bool_from_svcount: ; CHECK: // %bb.0: ; CHECK-NEXT: ret - %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") %pg) + %out = call <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svcount(target("aarch64.svcount") %pg) ret <vscale x 16 x i1> %out } @@ -111,7 +111,7 @@ define target("aarch64.svcount") @reinterpret_bool_to_svcount(<vscale x 16 x i1> ; CHECK-LABEL: reinterpret_bool_to_svcount: ; CHECK: // %bb.0: ; CHECK-NEXT: ret - %out = call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1> %pg) + %out = call target("aarch64.svcount") @llvm.aarch64.sve.convert.to.svcount(<vscale x 16 x i1> %pg) ret target("aarch64.svcount") %out } @@ -183,22 +183,3 @@ define <vscale x 16 x i1> @reinterpret_scalar_bool_q(i1 %x){ %out = tail call <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1> %.splat) ret <vscale x 16 x i1> %out } - - -declare <vscale x 8 x i1> @llvm.aarch64.sve.ptrue.nxv8i1(i32 immarg) -declare <vscale x 16 x i1> @llvm.aarch64.sve.ptrue.nxv16i1(i32 immarg) -declare <vscale x 8 x i1> @llvm.aarch64.sve.cmpgt.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>, <vscale x 8 x i16>) - -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv16i1(<vscale x 16 x i1>) -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv8i1(<vscale x 8 x i1>) -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv4i1(<vscale x 4 x i1>) -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv2i1(<vscale x 2 x i1>) -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.nxv1i1(<vscale x 1 x i1>) -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount")) - -declare <vscale x 16 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv16i1(<vscale x 16 x i1>) -declare <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1>) -declare <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1>) -declare <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1>) -declare <vscale x 1 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv1i1(<vscale x 16 x i1>) -declare target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt(<vscale x 16 x i1>) _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
