https://github.com/skc7 updated https://github.com/llvm/llvm-project/pull/204283
>From 6d3caa24d62bb992d2d639984027c79b48393041 Mon Sep 17 00:00:00 2001 From: skc7 <[email protected]> Date: Wed, 17 Jun 2026 10:18:51 +0530 Subject: [PATCH 1/2] [CIR] Add invariant attribute to cir.load --- .../CIR/Dialect/Builder/CIRBaseBuilder.h | 10 +++++---- clang/include/clang/CIR/Dialect/IR/CIROps.td | 10 ++++++++- clang/lib/CIR/CodeGen/CIRGenBuilder.h | 6 ++++-- .../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 6 ++++-- .../TargetLowering/CIRABIRewriteContext.cpp | 3 ++- .../TargetLowering/LowerItaniumCXXABI.cpp | 15 ++++++++----- .../CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp | 2 +- clang/test/CIR/IR/load-invariant.cir | 20 ++++++++++++++++++ clang/test/CIR/Lowering/load-invariant.cir | 21 +++++++++++++++++++ 9 files changed, 77 insertions(+), 16 deletions(-) create mode 100644 clang/test/CIR/IR/load-invariant.cir create mode 100644 clang/test/CIR/Lowering/load-invariant.cir diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h index 0db205f8d5b79..7cedc8bcfab6f 100644 --- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h +++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h @@ -230,11 +230,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr, bool isVolatile = false, uint64_t alignment = 0, - bool isNontemporal = false) { + bool isNontemporal = false, + bool isInvariant = false) { mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment); return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, isVolatile, - isNontemporal, alignmentAttr, - cir::SyncScopeKindAttr{}, cir::MemOrderAttr{}); + isNontemporal, alignmentAttr, cir::SyncScopeKindAttr{}, + cir::MemOrderAttr{}, isInvariant); } mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr, @@ -429,7 +430,8 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { return cir::LoadOp::create(*this, loc, addr, /*isDeref=*/false, /*isVolatile=*/false, /*nontemporal=*/false, alignmentAttr, - /*sync_scope=*/{}, /*mem_order=*/{}); + /*sync_scope=*/{}, /*mem_order=*/{}, + /*invariant=*/false); } cir::PtrStrideOp createPtrStride(mlir::Location loc, mlir::Value base, diff --git a/clang/include/clang/CIR/Dialect/IR/CIROps.td b/clang/include/clang/CIR/Dialect/IR/CIROps.td index f4f22cd297ea6..5cc7c964abfa1 100644 --- a/clang/include/clang/CIR/Dialect/IR/CIROps.td +++ b/clang/include/clang/CIR/Dialect/IR/CIROps.td @@ -671,6 +671,9 @@ def CIR_LoadOp : CIR_Op<"load", [ `alignment` can be used to specify an alignment that's different from the default, which is computed from `result`'s type ABI data layout. + A unit attribute `invariant` can be used to indicate that the loaded memory + never changes, mapping to LLVM IR's `!invariant.load` metadata. + Example: ``` @@ -685,6 +688,9 @@ def CIR_LoadOp : CIR_Op<"load", [ // Perform a volatile load from address in %0. %4 = cir.load volatile %0 : !cir.ptr<i32>, i32 + // Perform an invariant load from address in %0. + %5 = cir.load invariant %0 : !cir.ptr<i32>, i32 + // Others %x = cir.load align(16) atomic(seq_cst) %0 : !cir.ptr<i32>, i32 ``` @@ -697,13 +703,15 @@ def CIR_LoadOp : CIR_Op<"load", [ UnitAttr:$is_nontemporal, OptionalAttr<I64Attr>:$alignment, OptionalAttr<CIR_SyncScopeKind>:$sync_scope, - OptionalAttr<CIR_MemOrder>:$mem_order); + OptionalAttr<CIR_MemOrder>:$mem_order, + UnitAttr:$invariant); let results = (outs CIR_AnyType:$result); let assemblyFormat = [{ (`deref` $isDeref^)? (`volatile` $is_volatile^)? (`nontemporal` $is_nontemporal^)? + (`invariant` $invariant^)? (`align` `(` $alignment^ `)`)? (`syncscope` `(` $sync_scope^ `)`)? (`atomic` `(` $mem_order^ `)`)? diff --git a/clang/lib/CIR/CodeGen/CIRGenBuilder.h b/clang/lib/CIR/CodeGen/CIRGenBuilder.h index b8db0d9157aa6..cc1ff1e32521f 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuilder.h +++ b/clang/lib/CIR/CodeGen/CIRGenBuilder.h @@ -583,7 +583,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy { isVolatile, isNontemporal, /*alignment=*/align, /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr{}); + /*mem_order=*/cir::MemOrderAttr{}, + /*invariant=*/false); } cir::LoadOp createAlignedLoad(mlir::Location loc, mlir::Type ty, @@ -596,7 +597,8 @@ class CIRGenBuilderTy : public cir::CIRBaseBuilderTy { /*isVolatile=*/false, /*isNontemporal=*/false, alignAttr, /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr{}); + /*mem_order=*/cir::MemOrderAttr{}, + /*invariant=*/false); } cir::LoadOp diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp index c487e645e30cd..57507240ddfd8 100644 --- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp +++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp @@ -982,7 +982,8 @@ class CIRCleanupScopeOpFlattening cir::LoadOp::create(rewriter, loc, alloca, /*isDeref=*/false, /*isVolatile=*/false, /*isNontemporal=*/false, /*alignment=*/mlir::IntegerAttr(), - cir::SyncScopeKindAttr(), cir::MemOrderAttr()); + cir::SyncScopeKindAttr(), cir::MemOrderAttr(), + /*invariant=*/false); returnValues.push_back(loaded); } } @@ -1296,7 +1297,8 @@ class CIRCleanupScopeOpFlattening cir::LoadOp::create(rewriter, loc, destSlot, /*isDeref=*/false, /*isVolatile=*/false, /*isNontemporal=*/false, /*alignment=*/mlir::IntegerAttr(), - cir::SyncScopeKindAttr(), cir::MemOrderAttr()); + cir::SyncScopeKindAttr(), cir::MemOrderAttr() + /*invariant=*/false); // Create destination blocks for each exit and collect switch case info. llvm::SmallVector<mlir::APInt, 8> caseValues; diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp index b252ed188c408..00a41456b303b 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/CIRABIRewriteContext.cpp @@ -534,7 +534,8 @@ void rewriteIndirectReturnCall(cir::CallOp call, /*isVolatile=*/mlir::UnitAttr(), /*alignment=*/mlir::IntegerAttr(), /*sync_scope=*/cir::SyncScopeKindAttr(), - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/mlir::UnitAttr()); call.getResult().replaceAllUsesWith(load); } call->erase(); diff --git a/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp b/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp index 6e12a13787a2a..5218d4979f353 100644 --- a/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp +++ b/clang/lib/CIR/Dialect/Transforms/TargetLowering/LowerItaniumCXXABI.cpp @@ -385,7 +385,8 @@ void LowerItaniumCXXABI::lowerGetMethod( /*isNontemporal=*/false, /*alignment=*/mlir::IntegerAttr(), /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); // Apply the offset. // On ARM64, to reserve extra space in virtual member function pointers, @@ -413,7 +414,8 @@ void LowerItaniumCXXABI::lowerGetMethod( /*isNontemporal=*/false, /*alignment=*/mlir::IntegerAttr(), /*sync_scope=*/cir::SyncScopeKindAttr{}, - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); cir::YieldOp::create(b, loc, fnPtr.getResult()); assert(!cir::MissingFeatures::emitCFICheck()); @@ -787,7 +789,8 @@ static mlir::Value buildDynamicCastToVoidAfterNullCheck( /*isNontemporal=*/false, /*alignment=*/builder.getI64IntegerAttr(vtableElemAlign), /*sync_scope=*/cir::SyncScopeKindAttr(), - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); mlir::Value elementPtr = cir::CastOp::create(builder, loc, vtableElemPtrTy, cir::CastKind::bitcast, vptr); mlir::Value minusTwo = @@ -801,7 +804,8 @@ static mlir::Value buildDynamicCastToVoidAfterNullCheck( /*isNontemporal=*/false, /*alignment=*/builder.getI64IntegerAttr(vtableElemAlign), /*sync_scope=*/cir::SyncScopeKindAttr(), - /*mem_order=*/cir::MemOrderAttr()); + /*mem_order=*/cir::MemOrderAttr(), + /*invariant=*/false); auto voidPtrTy = cir::PointerType::get(cir::VoidType::get(builder.getContext())); @@ -910,7 +914,8 @@ mlir::Value LowerItaniumCXXABI::readArrayCookieImpl( builder, loc, countPtr, /*isDeref=*/false, /*isVolatile=*/false, /*isNontemporal=*/false, builder.getI64IntegerAttr(countAlignment.getQuantity()), - cir::SyncScopeKindAttr(), cir::MemOrderAttr()); + cir::SyncScopeKindAttr(), cir::MemOrderAttr(), + /*invariant=*/false); } } // namespace cir diff --git a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp index 27eba4ee326a5..199b3f0d2233d 100644 --- a/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp +++ b/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp @@ -1866,7 +1866,7 @@ mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite( mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create( rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment, op.getIsVolatile(), /*isNonTemporal=*/op.getIsNontemporal(), - /*isInvariant=*/false, /*isInvariantGroup=*/false, ordering, + /*isInvariant=*/op.getInvariant(), /*isInvariantGroup=*/false, ordering, llvmSyncScope.value_or(std::string())); // Convert adapted result to its original type if needed. diff --git a/clang/test/CIR/IR/load-invariant.cir b/clang/test/CIR/IR/load-invariant.cir new file mode 100644 index 0000000000000..4b014666db3a9 --- /dev/null +++ b/clang/test/CIR/IR/load-invariant.cir @@ -0,0 +1,20 @@ +// RUN: cir-opt %s --verify-roundtrip | FileCheck %s + +!s32i = !cir.int<s, 32> + +module { + // CHECK-LABEL: cir.func @invariant_load + // CHECK: %{{.*}} = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i + cir.func @invariant_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } + + // A plain load must not print the invariant keyword. + // CHECK-LABEL: cir.func @plain_load + // CHECK: %{{.*}} = cir.load %arg0 : !cir.ptr<!s32i>, !s32i + cir.func @plain_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } +} diff --git a/clang/test/CIR/Lowering/load-invariant.cir b/clang/test/CIR/Lowering/load-invariant.cir new file mode 100644 index 0000000000000..40c929ba227df --- /dev/null +++ b/clang/test/CIR/Lowering/load-invariant.cir @@ -0,0 +1,21 @@ +// RUN: cir-opt %s -cir-to-llvm -o - | FileCheck %s + +// The invariant flag on cir.load lowers to the LLVM dialect invariant load. + +!s32i = !cir.int<s, 32> + +module { + // CHECK-LABEL: llvm.func @invariant_load + // CHECK: llvm.load %{{.*}} invariant {{.*}}: !llvm.ptr -> i32 + cir.func @invariant_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load invariant %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } + + // CHECK-LABEL: llvm.func @plain_load + // CHECK-NOT: invariant + cir.func @plain_load(%arg0: !cir.ptr<!s32i>) -> !s32i { + %0 = cir.load %arg0 : !cir.ptr<!s32i>, !s32i + cir.return %0 : !s32i + } +} >From fc6603e59ffe89aec698efa6751a5860e63e16fa Mon Sep 17 00:00:00 2001 From: skc7 <[email protected]> Date: Tue, 23 Jun 2026 12:30:13 +0530 Subject: [PATCH 2/2] set it on virtual call loads --- .../CIR/Dialect/Builder/CIRBaseBuilder.h | 8 +++---- clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp | 11 +++++---- .../lib/CIR/Dialect/Transforms/FlattenCFG.cpp | 2 +- .../CIR/CodeGen/vtable-load-invariant.cpp | 23 +++++++++++++++++++ 4 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 clang/test/CIR/CodeGen/vtable-load-invariant.cpp diff --git a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h index 7cedc8bcfab6f..423630858f0a6 100644 --- a/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h +++ b/clang/include/clang/CIR/Dialect/Builder/CIRBaseBuilder.h @@ -230,12 +230,12 @@ class CIRBaseBuilderTy : public mlir::OpBuilder { cir::LoadOp createLoad(mlir::Location loc, mlir::Value ptr, bool isVolatile = false, uint64_t alignment = 0, - bool isNontemporal = false, - bool isInvariant = false) { + bool isNontemporal = false) { mlir::IntegerAttr alignmentAttr = getAlignmentAttr(alignment); return cir::LoadOp::create(*this, loc, ptr, /*isDeref=*/false, isVolatile, - isNontemporal, alignmentAttr, cir::SyncScopeKindAttr{}, - cir::MemOrderAttr{}, isInvariant); + isNontemporal, alignmentAttr, + cir::SyncScopeKindAttr{}, cir::MemOrderAttr{}, + /*invariant=*/false); } mlir::Value createAlignedLoad(mlir::Location loc, mlir::Value ptr, diff --git a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp index 552d73966e97b..f97538e47d871 100644 --- a/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenItaniumCXXABI.cpp @@ -1960,16 +1960,17 @@ CIRGenCallee CIRGenItaniumCXXABI::getVirtualFunctionPointer( cgf.getPointerAlign()); } - // Add !invariant.load md to virtual function load to indicate that - // function didn't change inside vtable. + // Set invariant on the cir.load of virtual function pointer to indicate + // that function didn't change inside vtable. // It's safe to add it without -fstrict-vtable-pointers, but it would not // help in devirtualization because it will only matter if we will have 2 // the same virtual function loads from the same vtable load, which won't // happen without enabled devirtualization with -fstrict-vtable-pointers. if (cgm.getCodeGenOpts().OptimizationLevel > 0 && - cgm.getCodeGenOpts().StrictVTablePointers) { - cgm.errorNYI(loc, "getVirtualFunctionPointer: strictVTablePointers"); - } + cgm.getCodeGenOpts().StrictVTablePointers) + if (auto loadOp = vfuncLoad.getDefiningOp<cir::LoadOp>()) + loadOp.setInvariant(true); + vfunc = vfuncLoad; } diff --git a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp index 57507240ddfd8..0b8e6b57c6e7a 100644 --- a/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp +++ b/clang/lib/CIR/Dialect/Transforms/FlattenCFG.cpp @@ -1297,7 +1297,7 @@ class CIRCleanupScopeOpFlattening cir::LoadOp::create(rewriter, loc, destSlot, /*isDeref=*/false, /*isVolatile=*/false, /*isNontemporal=*/false, /*alignment=*/mlir::IntegerAttr(), - cir::SyncScopeKindAttr(), cir::MemOrderAttr() + cir::SyncScopeKindAttr(), cir::MemOrderAttr(), /*invariant=*/false); // Create destination blocks for each exit and collect switch case info. diff --git a/clang/test/CIR/CodeGen/vtable-load-invariant.cpp b/clang/test/CIR/CodeGen/vtable-load-invariant.cpp new file mode 100644 index 0000000000000..afbe07b6483e5 --- /dev/null +++ b/clang/test/CIR/CodeGen/vtable-load-invariant.cpp @@ -0,0 +1,23 @@ +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -O1 -fstrict-vtable-pointers -fclangir -emit-cir %s -o %t.cir +// RUN: FileCheck --check-prefix=CIR-STRICT --input-file=%t.cir %s +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu -fclangir -emit-cir %s -o %t-plain.cir +// RUN: FileCheck --check-prefix=CIR --input-file=%t-plain.cir %s + +struct A { + virtual void f(char); +}; + +void f1(A *a) { + a->f('c'); +} + +// Under -O1 -fstrict-vtable-pointers the virtual function pointer load is +// marked invariant; the vptr load preceding it is not. +// CIR-STRICT-LABEL: cir.func{{.*}}@_Z2f1P1A +// CIR-STRICT: %[[VPTR:.*]] = cir.load{{.*}} %{{.*}} : !cir.ptr<!cir.vptr>, !cir.vptr +// CIR-STRICT: %[[FN_PTR_PTR:.*]] = cir.vtable.get_virtual_fn_addr %[[VPTR]][0] +// CIR-STRICT: %[[FN_PTR:.*]] = cir.load invariant {{.*}} %[[FN_PTR_PTR]] + +// Without the flags the same load must not be invariant. +// CIR-LABEL: cir.func{{.*}}@_Z2f1P1A +// CIR-NOT: cir.load invariant _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
