https://github.com/Lancern updated https://github.com/llvm/llvm-project/pull/206054
>From c494de3990914289997c8131aea7a245b247c102 Mon Sep 17 00:00:00 2001 From: Sirui Mu <[email protected]> Date: Fri, 26 Jun 2026 20:37:30 +0800 Subject: [PATCH] [CIR][NFC] Align dynamic dispatch of sync scopes to memory ordering This patch is a minor refactoring that aligns the dynamic dispatch path of atomic sync scopes to the dynamic dispatch path of atomic memory ordering. --- clang/lib/CIR/CodeGen/CIRGenAtomic.cpp | 190 +++++++++++------------- clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp | 3 +- clang/lib/CIR/CodeGen/CIRGenFunction.h | 20 ++- clang/test/CIR/CodeGen/atomic-libcall.c | 24 +-- 4 files changed, 122 insertions(+), 115 deletions(-) diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp index a758057f781678..017157c1aeaea4 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp @@ -980,7 +980,6 @@ static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest, // Map clang sync scope to CIR sync scope. static cir::SyncScopeKind convertSyncScopeToCIR(CIRGenFunction &cgf, - SourceRange range, clang::SyncScope scope) { switch (scope) { case clang::SyncScope::SingleScope: @@ -1022,76 +1021,6 @@ static cir::SyncScopeKind convertSyncScopeToCIR(CIRGenFunction &cgf, llvm_unreachable("unhandled sync scope"); } -static void emitAtomicOp(CIRGenFunction &cgf, AtomicExpr *expr, Address dest, - Address ptr, Address val1, Address val2, - Address expectedResult, Expr *isWeakExpr, - Expr *failureOrderExpr, int64_t size, - cir::MemOrder order, - const std::optional<Expr::EvalResult> &scopeConst, - mlir::Value scopeValue) { - std::unique_ptr<AtomicScopeModel> scopeModel = expr->getScopeModel(); - - if (!scopeModel) { - emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr, - failureOrderExpr, size, order, cir::SyncScopeKind::System); - return; - } - - if (scopeConst.has_value()) { - cir::SyncScopeKind mappedScope = convertSyncScopeToCIR( - cgf, expr->getScope()->getSourceRange(), - scopeModel->map(scopeConst->Val.getInt().getZExtValue())); - emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, isWeakExpr, - failureOrderExpr, size, order, mappedScope); - return; - } - - // The sync scope is not a compile-time constant. Emit a switch statement to - // handle each possible value of the sync scope. - CIRGenBuilderTy &builder = cgf.getBuilder(); - mlir::Location loc = cgf.getLoc(expr->getSourceRange()); - llvm::ArrayRef<unsigned> allScopes = scopeModel->getRuntimeValues(); - unsigned fallback = scopeModel->getFallBackValue(); - - cir::SwitchOp::create( - builder, loc, scopeValue, - [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) { - mlir::Block *switchBlock = builder.getBlock(); - - // Default case -- use fallback scope - cir::SyncScopeKind fallbackScope = convertSyncScopeToCIR( - cgf, expr->getScope()->getSourceRange(), scopeModel->map(fallback)); - emitDefaultCaseLabel(builder, loc); - emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, - isWeakExpr, failureOrderExpr, size, order, fallbackScope); - builder.createBreak(loc); - builder.setInsertionPointToEnd(switchBlock); - - // Emit a switch case for each non-fallback runtime scope value - for (unsigned scope : allScopes) { - if (scope == fallback) - continue; - - cir::SyncScopeKind cirScope = convertSyncScopeToCIR( - cgf, expr->getScope()->getSourceRange(), scopeModel->map(scope)); - - mlir::ArrayAttr casesAttr = builder.getArrayAttr( - {cir::IntAttr::get(scopeValue.getType(), scope)}); - mlir::OpBuilder::InsertPoint insertPoint; - cir::CaseOp::create(builder, loc, casesAttr, cir::CaseOpKind::Equal, - insertPoint); - - builder.restoreInsertionPoint(insertPoint); - emitAtomicOp(cgf, expr, dest, ptr, val1, val2, expectedResult, - isWeakExpr, failureOrderExpr, size, order, cirScope); - builder.createBreak(loc); - builder.setInsertionPointToEnd(switchBlock); - } - - builder.createYield(loc); - }); -} - static std::optional<cir::MemOrder> getEffectiveAtomicMemOrder(cir::MemOrder oriOrder, bool isStore, bool isLoad, bool isFence) { @@ -1181,13 +1110,55 @@ static void emitAtomicExprWithDynamicMemOrder( }); } +static void emitAtomicExprWithDynamicSyncScope( + CIRGenFunction &cgf, const AtomicScopeModel *scopeModel, mlir::Value scope, + llvm::function_ref<void(cir::SyncScopeKind)> emitAtomicOp) { + CIRGenBuilderTy &builder = cgf.getBuilder(); + llvm::ArrayRef<unsigned> allScopes = scopeModel->getRuntimeValues(); + unsigned fallback = scopeModel->getFallBackValue(); + + cir::SwitchOp::create( + builder, scope.getLoc(), scope, + [&](mlir::OpBuilder &, mlir::Location loc, mlir::OperationState &) { + mlir::Block *switchBlock = builder.getBlock(); + + // Default case -- use fallback scope + cir::SyncScopeKind fallbackScope = + convertSyncScopeToCIR(cgf, scopeModel->map(fallback)); + emitDefaultCaseLabel(builder, loc); + emitAtomicOp(fallbackScope); + builder.createBreak(loc); + builder.setInsertionPointToEnd(switchBlock); + + // Emit a switch case for each non-fallback runtime scope value + for (unsigned runtimeScopeValue : allScopes) { + if (runtimeScopeValue == fallback) + continue; + + cir::SyncScopeKind cirScope = + convertSyncScopeToCIR(cgf, scopeModel->map(runtimeScopeValue)); + + mlir::ArrayAttr casesAttr = builder.getArrayAttr( + {cir::IntAttr::get(scope.getType(), runtimeScopeValue)}); + mlir::OpBuilder::InsertPoint insertPoint; + cir::CaseOp::create(builder, loc, casesAttr, cir::CaseOpKind::Equal, + insertPoint); + + builder.restoreInsertionPoint(insertPoint); + emitAtomicOp(cirScope); + builder.createBreak(loc); + builder.setInsertionPointToEnd(switchBlock); + } + + builder.createYield(loc); + }); +} + void CIRGenFunction::emitAtomicExprWithMemOrder( - const Expr *memOrder, bool isStore, bool isLoad, bool isFence, + const EmittedOrderOrScope &order, bool isStore, bool isLoad, bool isFence, llvm::function_ref<void(cir::MemOrder)> emitAtomicOpFn) { - // Emit the memory order operand, and try to evaluate it as a constant. - Expr::EvalResult eval; - if (memOrder->EvaluateAsInt(eval, getContext())) { - uint64_t constOrder = eval.Val.getInt().getZExtValue(); + if (order.eval.has_value()) { + uint64_t constOrder = order.eval->Val.getInt().getZExtValue(); // We should not ever get to a case where the ordering isn't a valid CABI // value, but it's hard to enforce that in general. if (!cir::isValidCIRAtomicOrderingCABI(constOrder)) @@ -1199,11 +1170,29 @@ void CIRGenFunction::emitAtomicExprWithMemOrder( return; } - // Otherwise, handle variable memory ordering. Emit `SwitchOp` to convert - // dynamic value to static value. - mlir::Value dynOrder = emitScalarExpr(memOrder); - emitAtomicExprWithDynamicMemOrder(*this, dynOrder, isStore, isLoad, isFence, - emitAtomicOpFn); + // Emit `SwitchOp` to convert dynamic value to static value. + emitAtomicExprWithDynamicMemOrder(*this, order.value, isStore, isLoad, + isFence, emitAtomicOpFn); +} + +void CIRGenFunction::emitAtomicExprWithSyncScope( + const AtomicScopeModel *scopeModel, const EmittedOrderOrScope &scope, + llvm::function_ref<void(cir::SyncScopeKind)> emitAtomicOp) { + if (!scopeModel || !scope.value) { + emitAtomicOp(cir::SyncScopeKind::System); + return; + } + + if (scope.eval.has_value()) { + cir::SyncScopeKind mappedScope = convertSyncScopeToCIR( + *this, scopeModel->map(scope.eval->Val.getInt().getZExtValue())); + emitAtomicOp(mappedScope); + return; + } + + // Emit a switch op to match dynamic sync scope values to static sync scopes. + emitAtomicExprWithDynamicSyncScope(*this, scopeModel, scope.value, + emitAtomicOp); } static RValue emitAtomicLibCall(CIRGenFunction &cgf, llvm::StringRef funcName, @@ -1221,11 +1210,10 @@ static RValue emitAtomicLibCall(CIRGenFunction &cgf, llvm::StringRef funcName, /*isMustTail=*/false); } -static RValue emitLibCallForAtomicExpr(CIRGenFunction &cgf, AtomicExpr *e, - Address atomicPtr, Address dest, - Address val1, Address val2, - uint64_t atomicTySize, - QualType resultTy) { +static RValue emitLibCallForAtomicExpr( + CIRGenFunction &cgf, AtomicExpr *e, Address atomicPtr, Address dest, + Address val1, Address val2, uint64_t atomicTySize, QualType resultTy, + const CIRGenFunction::EmittedOrderOrScope &successOrder) { mlir::Location loc = cgf.getLoc(e->getSourceRange()); CallArgList args; @@ -1249,7 +1237,7 @@ static RValue emitLibCallForAtomicExpr(CIRGenFunction &cgf, AtomicExpr *e, e->getPtr()->getType())), cgf.getContext().VoidPtrTy); - mlir::Value order = cgf.emitScalarExpr(e->getOrder()); + mlir::Value order = successOrder.value; // The next 1-3 parameters are op-dependent. llvm::StringRef calleeName; @@ -1463,13 +1451,10 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) { TypeInfoChars typeInfo = getContext().getTypeInfoInChars(atomicTy); uint64_t size = typeInfo.Width.getQuantity(); - // Emit the sync scope operand, and try to evaluate it as a constant. - mlir::Value scope = - e->getScopeModel() ? emitScalarExpr(e->getScope()) : nullptr; - std::optional<Expr::EvalResult> scopeConst; - if (Expr::EvalResult eval; - e->getScopeModel() && e->getScope()->EvaluateAsInt(eval, getContext())) - scopeConst.emplace(std::move(eval)); + EmittedOrderOrScope order(*this, e->getOrder()); + EmittedOrderOrScope scope; + if (e->getScopeModel()) + scope = EmittedOrderOrScope(*this, e->getScope()); switch (e->getOp()) { default: @@ -1687,7 +1672,7 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) { // See: https://llvm.org/docs/Atomics.html#libcalls-atomic if (useLibCall) return emitLibCallForAtomicExpr(*this, e, ptr, dest, val1, val2, size, - resultTy); + resultTy, order); bool isStore = e->getOp() == AtomicExpr::AO__c11_atomic_store || e->getOp() == AtomicExpr::AO__opencl_atomic_store || @@ -1705,12 +1690,15 @@ RValue CIRGenFunction::emitAtomicExpr(AtomicExpr *e) { e->getOp() == AtomicExpr::AO__scoped_atomic_load || e->getOp() == AtomicExpr::AO__scoped_atomic_load_n; - auto emitAtomicOpCallBackFn = [&](cir::MemOrder memOrder) { - emitAtomicOp(*this, e, dest, ptr, val1, val2, originalVal1, isWeakExpr, - orderFailExpr, size, memOrder, scopeConst, scope); - }; - emitAtomicExprWithMemOrder(e->getOrder(), isStore, isLoad, /*isFence*/ false, - emitAtomicOpCallBackFn); + emitAtomicExprWithMemOrder( + order, isStore, isLoad, /*isFence*/ false, [&](cir::MemOrder memOrder) { + std::unique_ptr<AtomicScopeModel> scopeModel = e->getScopeModel(); + emitAtomicExprWithSyncScope( + scopeModel.get(), scope, [&](cir::SyncScopeKind scope) { + emitAtomicOp(*this, e, dest, ptr, val1, val2, originalVal1, + isWeakExpr, orderFailExpr, size, memOrder, scope); + }); + }); if (resultTy->isVoidType()) return RValue::get(nullptr); diff --git a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp index a94139cff20ffb..db55957b6d00be 100644 --- a/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenBuiltin.cpp @@ -470,7 +470,8 @@ static void emitAtomicFenceOp(CIRGenFunction &cgf, const CallExpr *expr, cir::SyncScopeKindAttr::get(&cgf.getMLIRContext(), syncScope)); }; - cgf.emitAtomicExprWithMemOrder(expr->getArg(0), /*isStore*/ false, + CIRGenFunction::EmittedOrderOrScope order(cgf, expr->getArg(0)); + cgf.emitAtomicExprWithMemOrder(order, /*isStore*/ false, /*isLoad*/ false, /*isFence*/ true, emitAtomicOpCallBackFn); } diff --git a/clang/lib/CIR/CodeGen/CIRGenFunction.h b/clang/lib/CIR/CodeGen/CIRGenFunction.h index 90da139c0f19c7..caa2b5e67eb993 100644 --- a/clang/lib/CIR/CodeGen/CIRGenFunction.h +++ b/clang/lib/CIR/CodeGen/CIRGenFunction.h @@ -1908,9 +1908,27 @@ class CIRGenFunction : public CIRGenTypeCache { void emitAtomicStore(RValue rvalue, LValue dest, bool isInit); void emitAtomicStore(RValue rvalue, LValue dest, cir::MemOrder order, bool isVolatile, bool isInit); + + /// An emitted atomic order or scope value with optional constant evaluation. + struct EmittedOrderOrScope { + mlir::Value value; + std::optional<Expr::EvalResult> eval; + + EmittedOrderOrScope() = default; + EmittedOrderOrScope(CIRGenFunction &cgf, const Expr *expr) + : value(cgf.emitScalarExpr(expr)) { + Expr::EvalResult evalResult; + if (expr->EvaluateAsInt(evalResult, cgf.getContext())) + eval.emplace(std::move(evalResult)); + } + }; + void emitAtomicExprWithMemOrder( - const Expr *memOrder, bool isStore, bool isLoad, bool isFence, + const EmittedOrderOrScope &order, bool isStore, bool isLoad, bool isFence, llvm::function_ref<void(cir::MemOrder)> emitAtomicOp); + void emitAtomicExprWithSyncScope( + const AtomicScopeModel *scopeModel, const EmittedOrderOrScope &scope, + llvm::function_ref<void(cir::SyncScopeKind)> emitAtomicOp); mlir::Value makeBinaryAtomicValue( cir::AtomicFetchKind kind, const clang::CallExpr *expr, diff --git a/clang/test/CIR/CodeGen/atomic-libcall.c b/clang/test/CIR/CodeGen/atomic-libcall.c index b20eefce22b76b..65d4e16c58e1ed 100644 --- a/clang/test/CIR/CodeGen/atomic-libcall.c +++ b/clang/test/CIR/CodeGen/atomic-libcall.c @@ -17,11 +17,11 @@ void load(struct Big *ptr) { __atomic_load(ptr, &b, __ATOMIC_RELAXED); // CIR: %[[DEST_SLOT:.+]] = cir.alloca "b" align(4) : !cir.ptr<!rec_Big> // CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[PTR_INTPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[DEST_INTPTR:.+]] = cir.cast bitcast %[[DEST_SLOT]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[PTR_VOIDPTR:.+]] = cir.cast bitcast %[[PTR_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[DEST_VOIDPTR:.+]] = cir.cast bitcast %[[DEST_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_load(%[[SIZE]], %[[PTR_VOIDPTR]], %[[DEST_VOIDPTR]], %[[ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -38,11 +38,11 @@ void scoped_load(struct Big *ptr) { __scoped_atomic_load(ptr, &b, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // CIR: %[[DEST_SLOT:.+]] = cir.alloca "b" align(4) : !cir.ptr<!rec_Big> // CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[PTR_INTPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[DEST_INTPTR:.+]] = cir.cast bitcast %[[DEST_SLOT]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[PTR_VOIDPTR:.+]] = cir.cast bitcast %[[PTR_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[DEST_VOIDPTR:.+]] = cir.cast bitcast %[[DEST_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_load(%[[SIZE]], %[[PTR_VOIDPTR]], %[[DEST_VOIDPTR]], %[[ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -59,11 +59,11 @@ void c11_load(_Atomic(struct Big) *ptr) { // CIR: %[[DEST_SLOT:.+]] = cir.alloca "b" align(4) init : !cir.ptr<!rec_Big> // CIR-NEXT: %[[TEMP_SLOT:.+]] = cir.alloca "atomic-temp" align(4) : !cir.ptr<!rec_Big> // CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[PTR_INTPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[TEMP_INTPTR:.+]] = cir.cast bitcast %[[TEMP_SLOT]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[PTR_VOIDPTR:.+]] = cir.cast bitcast %[[PTR_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[TEMP_VOIDPTR:.+]] = cir.cast bitcast %[[TEMP_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_load(%[[SIZE]], %[[PTR_VOIDPTR]], %[[TEMP_VOIDPTR]], %[[ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () // CIR-NEXT: %[[TEMP_CAST:.+]] = cir.cast bitcast %[[TEMP_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!rec_Big> @@ -82,12 +82,12 @@ void store(struct Big *dest, struct Big *val) { __atomic_store(dest, val, __ATOMIC_RELAXED); // CIR: %[[DEST_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VALUE_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[DEST_INTPTR:.+]] = cir.cast bitcast %[[DEST_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[VALUE_INTPTR:.+]] = cir.cast bitcast %[[VALUE_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[DEST_VOIDPTR:.+]] = cir.cast bitcast %[[DEST_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VALUE_VOIDPTR:.+]] = cir.cast bitcast %[[VALUE_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_store(%[[SIZE]], %[[DEST_VOIDPTR]], %[[VALUE_VOIDPTR]], %[[ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -102,12 +102,12 @@ void scoped_store(struct Big *dest, struct Big *val) { __scoped_atomic_store(dest, val, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // CIR: %[[DEST_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VALUE_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[DEST_INTPTR:.+]] = cir.cast bitcast %[[DEST_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[VALUE_INTPTR:.+]] = cir.cast bitcast %[[VALUE_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[DEST_VOIDPTR:.+]] = cir.cast bitcast %[[DEST_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VALUE_VOIDPTR:.+]] = cir.cast bitcast %[[VALUE_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_store(%[[SIZE]], %[[DEST_VOIDPTR]], %[[VALUE_VOIDPTR]], %[[ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -122,13 +122,13 @@ void c11_store(_Atomic(struct Big) *dest, struct Big *val) { __c11_atomic_store(dest, *val, __ATOMIC_RELAXED); // CIR: %[[DEST_PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VALUE_PTR:.+]] = cir.load deref align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: cir.copy %[[VALUE_PTR]] align(4) to %[[TEMP_SLOT:.+]] align(4) : !cir.ptr<!rec_Big> // CIR-NEXT: %[[DEST_INTPTR:.+]] = cir.cast bitcast %[[DEST_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[VALUE_INTPTR:.+]] = cir.cast bitcast %[[TEMP_SLOT]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[DEST_VOIDPTR:.+]] = cir.cast bitcast %[[DEST_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VALUE_VOIDPTR:.+]] = cir.cast bitcast %[[VALUE_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_store(%[[SIZE]], %[[DEST_VOIDPTR]], %[[VALUE_VOIDPTR]], %[[ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -147,6 +147,7 @@ void exchange(struct Big *ptr, struct Big *val, struct Big *old) { // CIR-NEXT: %[[VAL:.*]] = cir.alloca "val" align(8) init : !cir.ptr<!cir.ptr<!rec_Big>> // CIR-NEXT: %[[OLD:.*]] = cir.alloca "old" align(8) init : !cir.ptr<!cir.ptr<!rec_Big>> // CIR: %[[LOAD_PTR:.*]] = cir.load align(8) %[[PTR]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[MEM_ORDER:.*]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[LOAD_VAL:.*]] = cir.load align(8) %[[VAL]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[LOAD_OLD:.*]] = cir.load align(8) %[[OLD]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[CAST_PTR:.*]] = cir.cast bitcast %[[LOAD_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> @@ -154,7 +155,6 @@ void exchange(struct Big *ptr, struct Big *val, struct Big *old) { // CIR-NEXT: %[[CAST_OLD:.*]] = cir.cast bitcast %[[LOAD_OLD]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.*]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[VOID_PTR:.*]] = cir.cast bitcast %[[CAST_PTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[MEM_ORDER:.*]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VOID_VAL:.*]] = cir.cast bitcast %[[CAST_VAL]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: %[[VOID_OLD:.*]] = cir.cast bitcast %[[CAST_OLD]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_exchange(%[[SIZE]], %[[VOID_PTR]], %[[VOID_VAL]], %[[VOID_OLD]], %[[MEM_ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -177,6 +177,7 @@ void scoped_exchange(struct Big *ptr, struct Big *val, struct Big *old) { // CIR-NEXT: %[[VAL:.*]] = cir.alloca "val" align(8) init : !cir.ptr<!cir.ptr<!rec_Big>> // CIR-NEXT: %[[OLD:.*]] = cir.alloca "old" align(8) init : !cir.ptr<!cir.ptr<!rec_Big>> // CIR: %[[LOAD_PTR:.*]] = cir.load align(8) %[[PTR]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[MEM_ORDER:.*]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[LOAD_VAL:.*]] = cir.load align(8) %[[VAL]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[LOAD_OLD:.*]] = cir.load align(8) %[[OLD]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[CAST_PTR:.*]] = cir.cast bitcast %[[LOAD_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> @@ -184,7 +185,6 @@ void scoped_exchange(struct Big *ptr, struct Big *val, struct Big *old) { // CIR-NEXT: %[[CAST_OLD:.*]] = cir.cast bitcast %[[LOAD_OLD]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.*]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[VOID_PTR:.*]] = cir.cast bitcast %[[CAST_PTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[MEM_ORDER:.*]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VOID_VAL:.*]] = cir.cast bitcast %[[CAST_VAL]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: %[[VOID_OLD:.*]] = cir.cast bitcast %[[CAST_OLD]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_exchange(%[[SIZE]], %[[VOID_PTR]], %[[VOID_VAL]], %[[VOID_OLD]], %[[MEM_ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -208,13 +208,13 @@ struct Big c11_exchange(_Atomic(struct Big) *ptr, struct Big val) { // CIR-NEXT: %[[TMP1:.*]] = cir.alloca ".atomictmp" align(4) : !cir.ptr<!rec_Big> // CIR-NEXT: %[[TMP2:.*]] = cir.alloca "atomic-temp" align(4) : !cir.ptr<!rec_Big> // CIR: %[[LOAD_PTR:.*]] = cir.load align(8) %[[PTR]] : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[MEM_ORDER:.*]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: cir.copy %[[VAL]] align(4) to %[[TMP1]] align(4) : !cir.ptr<!rec_Big> // CIR-NEXT: %[[CAST_PTR:.*]] = cir.cast bitcast %[[LOAD_PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[CAST_TMP1:.*]] = cir.cast bitcast %[[TMP1]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[CAST_TMP2:.*]] = cir.cast bitcast %[[TMP2]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.*]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[VOID_PTR:.*]] = cir.cast bitcast %[[CAST_PTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[MEM_ORDER:.*]] = cir.const #cir.int<0> : !s32i // CIR-NEXT: %[[VOID_TMP1:.*]] = cir.cast bitcast %[[CAST_TMP1]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: %[[VOID_TMP2:.*]] = cir.cast bitcast %[[CAST_TMP2]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: cir.call @__atomic_exchange(%[[SIZE]], %[[VOID_PTR]], %[[VOID_TMP1]], %[[VOID_TMP2]], %[[MEM_ORDER]]) : (!u64i {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !cir.ptr<!void> {llvm.noundef}, !s32i {llvm.noundef}) -> () @@ -231,6 +231,7 @@ void cmpxchg(struct Big *ptr, struct Big *expected, struct Big *desired) { __atomic_compare_exchange(ptr, expected, desired, /*weak=*/0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); // CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<5> : !s32i // CIR-NEXT: %[[EXPECTED:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[DESIRED:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[PTR_INTPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> @@ -238,7 +239,6 @@ void cmpxchg(struct Big *ptr, struct Big *expected, struct Big *desired) { // CIR-NEXT: %[[DESIRED_INTPTR:.+]] = cir.cast bitcast %[[DESIRED]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[PTR_VOIDPTR:.+]] = cir.cast bitcast %[[PTR_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<5> : !s32i // CIR-NEXT: %[[ORDER_FAIL:.+]] = cir.const #cir.int<2> : !s32i // CIR-NEXT: %[[EXPECTED_VOIDPTR:.+]] = cir.cast bitcast %[[EXPECTED_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: %[[DESIRED_VOIDPTR:.+]] = cir.cast bitcast %[[DESIRED_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> @@ -256,6 +256,7 @@ void c11_cmpxchg(_Atomic(struct Big) *ptr, struct Big *expected, struct Big *des __c11_atomic_compare_exchange_weak(ptr, expected, *desired, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE); // CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<5> : !s32i // CIR-NEXT: %[[EXPECTED:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[DESIRED:.+]] = cir.load deref align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: cir.copy %[[DESIRED]] align(4) to %[[TEMP_SLOT:.+]] align(4) : !cir.ptr<!rec_Big> @@ -264,7 +265,6 @@ void c11_cmpxchg(_Atomic(struct Big) *ptr, struct Big *expected, struct Big *des // CIR-NEXT: %[[DESIRED_INTPTR:.+]] = cir.cast bitcast %[[TEMP_SLOT]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[PTR_VOIDPTR:.+]] = cir.cast bitcast %[[PTR_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<5> : !s32i // CIR-NEXT: %[[ORDER_FAIL:.+]] = cir.const #cir.int<2> : !s32i // CIR-NEXT: %[[EXPECTED_VOIDPTR:.+]] = cir.cast bitcast %[[EXPECTED_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: %[[DESIRED_VOIDPTR:.+]] = cir.cast bitcast %[[DESIRED_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> @@ -284,6 +284,7 @@ void scoped_cmpxchg(struct Big *ptr, struct Big *expected, struct Big *desired) __scoped_atomic_compare_exchange(ptr, expected, desired, /*weak=*/0, __ATOMIC_SEQ_CST, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_SYSTEM); // CIR: %[[PTR:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> + // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<5> : !s32i // CIR-NEXT: %[[EXPECTED:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[DESIRED:.+]] = cir.load align(8) %{{.+}} : !cir.ptr<!cir.ptr<!rec_Big>>, !cir.ptr<!rec_Big> // CIR-NEXT: %[[PTR_INTPTR:.+]] = cir.cast bitcast %[[PTR]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> @@ -291,7 +292,6 @@ void scoped_cmpxchg(struct Big *ptr, struct Big *expected, struct Big *desired) // CIR-NEXT: %[[DESIRED_INTPTR:.+]] = cir.cast bitcast %[[DESIRED]] : !cir.ptr<!rec_Big> -> !cir.ptr<!cir.int<u, 192>> // CIR-NEXT: %[[SIZE:.+]] = cir.const #cir.int<24> : !u64i // CIR-NEXT: %[[PTR_VOIDPTR:.+]] = cir.cast bitcast %[[PTR_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> - // CIR-NEXT: %[[ORDER:.+]] = cir.const #cir.int<5> : !s32i // CIR-NEXT: %[[ORDER_FAIL:.+]] = cir.const #cir.int<2> : !s32i // CIR-NEXT: %[[EXPECTED_VOIDPTR:.+]] = cir.cast bitcast %[[EXPECTED_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> // CIR-NEXT: %[[DESIRED_VOIDPTR:.+]] = cir.cast bitcast %[[DESIRED_INTPTR]] : !cir.ptr<!cir.int<u, 192>> -> !cir.ptr<!void> _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
