https://github.com/akash-manna-sky updated https://github.com/llvm/llvm-project/pull/214653
>From cbfb43dcbf42645cbd8d5192ed5ed721ec1cf755 Mon Sep 17 00:00:00 2001 From: Akash Manna <[email protected]> Date: Fri, 7 Aug 2026 12:42:01 +0530 Subject: [PATCH 1/4] [Clang] Accept vector types in the atomic builtins atomicrmw operates elementwise on vector operands, but Sema rejected every vector value type, so __scoped_atomic_fetch_add(half2 *, ...) and its __atomic, __c11_atomic, __hip_atomic and __opencl_atomic equivalents all failed with: "address argument to atomic operation must be a pointer to integer, pointer or supported floating point type." BuildAtomicExpr tested isIntegerType(), isPointerType() and isFloatingType() directly on the value type. Since all three return false for VectorType, vector operands were rejected. Instead, check the element type, matching the LLVM verifier's rule for atomicrmw. Vectors of _Bool remain rejected because their elements are not individually addressable. CodeGen made two matching assumptions. The atomicrmw opcode was selected from getValueType()->isFloatingType(), which chose the integer Add opcode for vectors of floating-point elements. In addition, shouldCastToInt() bitcast every non-scalar operand to iN. That conversion is required for cmpxchg, which has no vector form, and is harmless for load and store, but it is incorrect for arithmetic operations. For example, an atomicrmw add on an i32 propagates carries across the lanes of a <2 x i16>. Vectors whose size is not a power of two are still accessed as integers because they are not valid atomic operands. Arithmetic atomic operations have no libcall fallback, so Sema now rejects vector types that cannot be emitted inline instead of reaching the llvm_unreachable() in EmitAtomicExpr(). Mirror the shouldCastToInt() rule in CIRGenAtomic.cpp so that -fclangir reports its existing unsupported type error instead of silently emitting an integer operation. Fixes #213237 --- clang/docs/ReleaseNotes.md | 10 +- .../clang/Basic/DiagnosticSemaKinds.td | 4 + clang/lib/CIR/CodeGen/CIRGenAtomic.cpp | 5 + clang/lib/CodeGen/CGAtomic.cpp | 72 ++++++++------ clang/lib/Sema/SemaChecking.cpp | 34 ++++++- clang/test/CodeGen/atomic-ops-vector.c | 95 +++++++++++++++++++ clang/test/Sema/atomic-ops-vector.c | 65 +++++++++++++ 7 files changed, 254 insertions(+), 31 deletions(-) create mode 100644 clang/test/CodeGen/atomic-ops-vector.c create mode 100644 clang/test/Sema/atomic-ops-vector.c diff --git a/clang/docs/ReleaseNotes.md b/clang/docs/ReleaseNotes.md index d9b9c92950c98..083523fd570ef 100644 --- a/clang/docs/ReleaseNotes.md +++ b/clang/docs/ReleaseNotes.md @@ -172,6 +172,14 @@ features cannot lower the translation-unit ABI level; - Clang now allows GNU computed `goto` extension in `constexpr` functions, matching the relaxed `constexpr` function body rules introduced in C++23. +- The `__atomic`, `__c11_atomic`, `__hip_atomic`, `__opencl_atomic` and + `__scoped_atomic` builtins now accept vector types. The operation is performed + elementwise by a single atomic instruction. The arithmetic operations require + the vector to have a power-of-two size of at most 16 bytes, as they cannot be + lowered to a libcall. As a result, an atomic load or store of a + vector, such as one written with `_Atomic` or with `#pragma omp atomic`, is now + emitted with the vector type rather than an integer of the same size. This + does not change the generated code. ### New Compiler Flags @@ -610,4 +618,4 @@ this release by going into the "`clang/docs/`" directory in the Clang tree. If you have any questions or comments about Clang, please feel free to -contact us on the [Discourse forums (Clang Frontend category)](https://discourse.llvm.org/c/clang/6). +contact us on the [Discourse forums (Clang Frontend category)](https://discourse.llvm.org/c/clang/6). \ No newline at end of file diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 4e9c6fd8cbf0e..709e97ee20123 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -9708,6 +9708,10 @@ def err_atomic_op_needs_atomic_int : Error< def err_atomic_op_needs_atomic_fp : Error<"address argument to atomic operation must be a pointer to " "%select{|atomic }0floating point type (%1 invalid)">; +def err_atomic_op_needs_supported_vector + : Error<"address argument to atomic operation must be a pointer to a " + "vector with a power-of-two size of at most %0 bytes " + "(%1 invalid)">; def warn_atomic_op_has_invalid_memory_order : Warning< "%select{|success |failure }0memory order argument to atomic operation is invalid">, InGroup<DiagGroup<"atomic-memory-ordering">>; diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp index 4a028e0be34f7..21b145a9cbee4 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp @@ -326,6 +326,11 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const { /// floating point operands. TODO: Allow compare-and-exchange and FP - see /// comment in CIRGenAtomicExpandPass.cpp. static bool shouldCastToInt(mlir::Type valueTy, bool cmpxchg) { + // The atomic operations act on a vector directly, which is required for the + // elementwise arithmetic operations. cmpxchg has no vector form, so it is + // still handled as an integer of the same size. + if (isa<cir::VectorType>(valueTy)) + return cmpxchg; if (cir::isAnyFloatingPointType(valueTy)) return isa<cir::FP80Type>(valueTy) || cmpxchg; return !isa<cir::IntType>(valueTy) && !isa<cir::PointerType>(valueTy); diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index cd7b278eab15d..2760cd9b03eb6 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -21,6 +21,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Intrinsics.h" +#include "llvm/Support/MathExtras.h" using namespace clang; using namespace CodeGen; @@ -529,7 +530,7 @@ static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, bool IsSigned, llvm::Value *OldVal, llvm::Value *RHS) { - const bool IsFP = OldVal->getType()->isFloatingPointTy(); + const bool IsFP = OldVal->getType()->isFPOrFPVectorTy(); if (IsFP) { llvm::Intrinsic::ID IID = (Op == AtomicExpr::AO__atomic_max_fetch || @@ -567,6 +568,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, bool PostOpMinMax = false; unsigned PostOp = 0; + // Atomic operations on a vector are performed elementwise, so it is the + // element type which selects between the integer and the floating point + // form of an operation. + QualType ElemTy = E->getValueType(); + if (const auto *VecTy = ElemTy->getAs<VectorType>()) + ElemTy = VecTy->getElementType(); + switch (E->getOp()) { case AtomicExpr::AO__c11_atomic_init: case AtomicExpr::AO__opencl_atomic_init: @@ -666,30 +674,30 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__atomic_add_fetch: case AtomicExpr::AO__scoped_atomic_add_fetch: - PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd - : llvm::Instruction::Add; + PostOp = ElemTy->isFloatingType() ? llvm::Instruction::FAdd + : llvm::Instruction::Add; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__scoped_atomic_fetch_add: - Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd - : llvm::AtomicRMWInst::Add; + Op = ElemTy->isFloatingType() ? llvm::AtomicRMWInst::FAdd + : llvm::AtomicRMWInst::Add; break; case AtomicExpr::AO__atomic_sub_fetch: case AtomicExpr::AO__scoped_atomic_sub_fetch: - PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub - : llvm::Instruction::Sub; + PostOp = ElemTy->isFloatingType() ? llvm::Instruction::FSub + : llvm::Instruction::Sub; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__scoped_atomic_fetch_sub: - Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub - : llvm::AtomicRMWInst::Sub; + Op = ElemTy->isFloatingType() ? llvm::AtomicRMWInst::FSub + : llvm::AtomicRMWInst::Sub; break; case AtomicExpr::AO__atomic_min_fetch: @@ -701,23 +709,22 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__scoped_atomic_fetch_min: - Op = E->getValueType()->isFloatingType() - ? llvm::AtomicRMWInst::FMin - : (E->getValueType()->isSignedIntegerType() - ? llvm::AtomicRMWInst::Min - : llvm::AtomicRMWInst::UMin); + Op = ElemTy->isFloatingType() ? llvm::AtomicRMWInst::FMin + : (ElemTy->isSignedIntegerType() + ? llvm::AtomicRMWInst::Min + : llvm::AtomicRMWInst::UMin); break; case AtomicExpr::AO__atomic_fetch_fminimum: case AtomicExpr::AO__scoped_atomic_fetch_fminimum: - assert(E->getValueType()->isFloatingType() && + assert(ElemTy->isFloatingType() && "fminimum operations only support floating-point types"); Op = llvm::AtomicRMWInst::FMinimum; break; case AtomicExpr::AO__atomic_fetch_fminimum_num: case AtomicExpr::AO__scoped_atomic_fetch_fminimum_num: - assert(E->getValueType()->isFloatingType() && + assert(ElemTy->isFloatingType() && "fminimum_num operations only support floating-point types"); Op = llvm::AtomicRMWInst::FMinimumNum; break; @@ -731,23 +738,22 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_fetch_max: case AtomicExpr::AO__atomic_fetch_max: case AtomicExpr::AO__scoped_atomic_fetch_max: - Op = E->getValueType()->isFloatingType() - ? llvm::AtomicRMWInst::FMax - : (E->getValueType()->isSignedIntegerType() - ? llvm::AtomicRMWInst::Max - : llvm::AtomicRMWInst::UMax); + Op = ElemTy->isFloatingType() ? llvm::AtomicRMWInst::FMax + : (ElemTy->isSignedIntegerType() + ? llvm::AtomicRMWInst::Max + : llvm::AtomicRMWInst::UMax); break; case AtomicExpr::AO__atomic_fetch_fmaximum: case AtomicExpr::AO__scoped_atomic_fetch_fmaximum: - assert(E->getValueType()->isFloatingType() && + assert(ElemTy->isFloatingType() && "fmaximum operations only support floating-point types"); Op = llvm::AtomicRMWInst::FMaximum; break; case AtomicExpr::AO__atomic_fetch_fmaximum_num: case AtomicExpr::AO__scoped_atomic_fetch_fmaximum_num: - assert(E->getValueType()->isFloatingType() && + assert(ElemTy->isFloatingType() && "fmaximum_num operations only support floating-point types"); Op = llvm::AtomicRMWInst::FMaximumNum; break; @@ -840,8 +846,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, llvm::Value *Result = RMWI; if (PostOpMinMax) Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), - E->getValueType()->isSignedIntegerType(), - RMWI, LoadVal1); + ElemTy->isSignedIntegerType(), RMWI, + LoadVal1); else if (PostOp) Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, LoadVal1); @@ -868,6 +874,13 @@ EmitValToTemp(CodeGenFunction &CGF, Expr *E) { /// floating point operands. TODO: Allow compare-and-exchange and FP - see /// comment in AtomicExpandPass.cpp. static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { + // The atomic instructions operate on a vector directly, which is required for + // the elementwise arithmetic operations. cmpxchg has no vector form, and a + // vector whose size is not a power of two is not a valid atomic operand, so + // both are still handled as an integer of the enclosing atomic type's size. + if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(ValTy)) + return CmpXchg || + !llvm::isPowerOf2_64(VecTy->getPrimitiveSizeInBits().getFixedValue()); if (ValTy->isFloatingPointTy()) return ValTy->isX86_FP80Ty() || CmpXchg; return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); @@ -1571,10 +1584,11 @@ RValue AtomicInfo::ConvertToValueOrAtomic(llvm::Value *Val, SourceLocation Loc, bool AsValue, bool CmpXchg) const { // Try not to in some easy cases. - assert((Val->getType()->isIntegerTy() || Val->getType()->isPointerTy() || - Val->getType()->isIEEELikeFPTy()) && - "Expected integer, pointer or floating point value when converting " - "result."); + llvm::Type *ValScalarTy = Val->getType()->getScalarType(); + assert((ValScalarTy->isIntegerTy() || ValScalarTy->isPointerTy() || + ValScalarTy->isIEEELikeFPTy()) && + "Expected integer, pointer or floating point value, or a vector " + "thereof, when converting result."); if (getEvaluationKind() == TEK_Scalar && (((!LVal.isBitField() || LVal.getBitFieldInfo().Size == ValueSizeInBits) && diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index 0db040ed90e3f..bd9252f409f3d 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5335,6 +5335,15 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, // trivial type errors. auto IsAllowedValueType = [&](QualType ValType, unsigned AllowedType) -> bool { + // Atomic operations on a vector are performed elementwise, so it is the + // element type which decides whether the operation is well-formed. A + // vector of _Bool is rejected outright, as its elements are not + // individually addressable. + if (const auto *VecTy = ValType->getAs<VectorType>()) { + if (VecTy->getElementType()->isBooleanType()) + return false; + ValType = VecTy->getElementType(); + } bool IsX87LongDouble = ValType->isSpecificBuiltinType(BuiltinType::LongDouble) && &Context.getTargetInfo().getLongDoubleFormat() == @@ -5367,6 +5376,26 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, << IsC11 << Ptr->getType() << Ptr->getSourceRange(); return ExprError(); } + // An arithmetic operation on a vector is always emitted as a single + // atomicrmw instruction; unlike the other forms there is no libcall to fall + // back on for a size which cannot be handled inline. The size of the vector + // itself is checked here, as the type holding it is padded out when the + // element count is not a power of two. + if (Form == Arithmetic) { + if (const auto *VecTy = ValType->getAs<VectorType>()) { + // The largest size EmitAtomicExpr() emits without a libcall. + constexpr unsigned MaxVectorSizeInBytes = 16; + uint64_t VecSizeInBits = Context.getTypeSize(VecTy->getElementType()) * + VecTy->getNumElements(); + if (!llvm::isPowerOf2_64(VecSizeInBits) || + VecSizeInBits > MaxVectorSizeInBytes * Context.getCharWidth()) { + Diag(ExprRange.getBegin(), diag::err_atomic_op_needs_supported_vector) + << MaxVectorSizeInBytes << Ptr->getType() + << Ptr->getSourceRange(); + return ExprError(); + } + } + } if (IsC11 && ValType->isPointerType() && RequireCompleteType(Ptr->getBeginLoc(), ValType->getPointeeType(), diag::err_incomplete_type)) { @@ -5635,7 +5664,10 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, ? 0 : 1); - if (ValType->isBitIntType()) { + QualType BitIntCandidateTy = ValType; + if (const auto *VecTy = ValType->getAs<VectorType>()) + BitIntCandidateTy = VecTy->getElementType(); + if (BitIntCandidateTy->isBitIntType()) { Diag(Ptr->getExprLoc(), diag::err_atomic_builtin_bit_int_prohibit); return ExprError(); } diff --git a/clang/test/CodeGen/atomic-ops-vector.c b/clang/test/CodeGen/atomic-ops-vector.c new file mode 100644 index 0000000000000..a6bb8d13fef39 --- /dev/null +++ b/clang/test/CodeGen/atomic-ops-vector.c @@ -0,0 +1,95 @@ +// RUN: %clang_cc1 %s -emit-llvm -o - -triple=x86_64-unknown-linux-gnu \ +// RUN: -target-feature +cx16 -Wno-atomic-alignment \ +// RUN: | FileCheck --check-prefixes=CHECK,X64 %s +// RUN: %clang_cc1 %s -emit-llvm -o - -triple=amdgcn-amd-amdhsa \ +// RUN: -Wno-atomic-alignment | FileCheck --check-prefixes=CHECK,AMDGCN %s + +// Atomic operations on a vector are performed elementwise by a single atomic +// instruction, rather than being bitcast to an integer of the same size. + +typedef _Float16 half2 __attribute__((ext_vector_type(2))); +typedef __bf16 bfloat2 __attribute__((ext_vector_type(2))); +typedef float float2 __attribute__((ext_vector_type(2))); +typedef float float3 __attribute__((ext_vector_type(3))); +typedef int int2 __attribute__((ext_vector_type(2))); + +// CHECK-LABEL: @test_load( +half2 test_load(half2 *p) { + // CHECK: load atomic <2 x half>, ptr {{.*}} monotonic + return __atomic_load_n(p, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_store( +void test_store(half2 *p, half2 v) { + // CHECK: store atomic <2 x half> {{.*}}, ptr {{.*}} monotonic + __atomic_store_n(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_exchange( +half2 test_exchange(half2 *p, half2 v) { + // CHECK: atomicrmw xchg ptr {{.*}}, <2 x half> {{.*}} monotonic + return __atomic_exchange_n(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_fetch_add_half2( +half2 test_fetch_add_half2(half2 *p, half2 v) { + // CHECK: atomicrmw fadd ptr {{.*}}, <2 x half> {{.*}} monotonic + return __atomic_fetch_add(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_fetch_add_bfloat2( +bfloat2 test_fetch_add_bfloat2(bfloat2 *p, bfloat2 v) { + // CHECK: atomicrmw fadd ptr {{.*}}, <2 x bfloat> {{.*}} monotonic + return __atomic_fetch_add(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_add_fetch_float2( +float2 test_add_fetch_float2(float2 *p, float2 v) { + // CHECK: atomicrmw fadd ptr {{.*}}, <2 x float> {{.*}} monotonic + // CHECK: fadd <2 x float> + return __atomic_add_fetch(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_fetch_fmaximum_float2( +float2 test_fetch_fmaximum_float2(float2 *p, float2 v) { + // CHECK: atomicrmw fmaximum ptr {{.*}}, <2 x float> {{.*}} monotonic + return __atomic_fetch_fmaximum(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_max_fetch_int2( +int2 test_max_fetch_int2(int2 *p, int2 v) { + // CHECK: atomicrmw max ptr {{.*}}, <2 x i32> {{.*}} monotonic + // CHECK: icmp sgt <2 x i32> + // CHECK: select <2 x i1> + return __atomic_max_fetch(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_nand_fetch_int2( +int2 test_nand_fetch_int2(int2 *p, int2 v) { + // CHECK: atomicrmw nand ptr {{.*}}, <2 x i32> {{.*}} monotonic + // CHECK: and <2 x i32> + // CHECK: xor <2 x i32> + return __atomic_nand_fetch(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_c11_fetch_add( +half2 test_c11_fetch_add(_Atomic(half2) *p, half2 v) { + // CHECK: atomicrmw fadd ptr {{.*}}, <2 x half> {{.*}} monotonic + return __c11_atomic_fetch_add(p, v, __ATOMIC_RELAXED); +} + +// CHECK-LABEL: @test_scoped_fetch_add( +half2 test_scoped_fetch_add(half2 *p, half2 v) { + // X64: atomicrmw fadd ptr {{.*}}, <2 x half> {{.*}} monotonic + // AMDGCN: atomicrmw fadd ptr {{.*}}, <2 x half> {{.*}} syncscope("agent") monotonic + return __scoped_atomic_fetch_add(p, v, __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE); +} + +// A vector whose size is not a power of two is not a valid atomic operand, so +// it is still accessed as an integer of the size of the type holding it. +// CHECK-LABEL: @test_load_float3( +float3 test_load_float3(float3 *p) { + // CHECK: load atomic i128, ptr {{.*}} monotonic + return __atomic_load_n(p, __ATOMIC_RELAXED); +} diff --git a/clang/test/Sema/atomic-ops-vector.c b/clang/test/Sema/atomic-ops-vector.c new file mode 100644 index 0000000000000..752bb4a5bb933 --- /dev/null +++ b/clang/test/Sema/atomic-ops-vector.c @@ -0,0 +1,65 @@ +// RUN: %clang_cc1 %s -verify -fsyntax-only -triple=x86_64-unknown-linux-gnu -std=c11 +// RUN: %clang_cc1 %s -verify -fsyntax-only -triple=amdgcn-amd-amdhsa -std=c11 + +// Atomic builtins accept vector types; the operation is performed elementwise +// by a single atomicrmw instruction. + +typedef _Float16 half2 __attribute__((ext_vector_type(2))); +typedef __bf16 bfloat2 __attribute__((ext_vector_type(2))); +typedef float float2 __attribute__((ext_vector_type(2))); +typedef float float3 __attribute__((ext_vector_type(3))); +typedef float float8 __attribute__((ext_vector_type(8))); +typedef int int2 __attribute__((ext_vector_type(2))); +typedef unsigned int uint4 __attribute__((ext_vector_type(4))); +typedef _Bool bool8 __attribute__((ext_vector_type(8))); +typedef _BitInt(16) bitint2 __attribute__((ext_vector_type(2))); + +void test_gnu(half2 *h2, bfloat2 *b2, float2 *f2, int2 *i2, uint4 *u4) { + (void)__atomic_load_n(h2, __ATOMIC_RELAXED); + __atomic_store_n(h2, *h2, __ATOMIC_RELAXED); + (void)__atomic_exchange_n(h2, *h2, __ATOMIC_RELAXED); + + (void)__atomic_fetch_add(h2, *h2, __ATOMIC_RELAXED); + (void)__atomic_fetch_add(b2, *b2, __ATOMIC_RELAXED); + (void)__atomic_fetch_sub(f2, *f2, __ATOMIC_RELAXED); + (void)__atomic_add_fetch(f2, *f2, __ATOMIC_RELAXED); + (void)__atomic_fetch_min(f2, *f2, __ATOMIC_RELAXED); + (void)__atomic_max_fetch(i2, *i2, __ATOMIC_RELAXED); + (void)__atomic_fetch_fmaximum(f2, *f2, __ATOMIC_RELAXED); + (void)__atomic_fetch_and(i2, *i2, __ATOMIC_RELAXED); + (void)__atomic_or_fetch(u4, *u4, __ATOMIC_RELAXED); + (void)__atomic_nand_fetch(i2, *i2, __ATOMIC_RELAXED); + + // The integer operations still reject a vector of floating point elements. + (void)__atomic_fetch_and(f2, *f2, __ATOMIC_RELAXED); // expected-error {{must be a pointer to integer}} + // The f-prefixed operations still reject a vector of integer elements. + (void)__atomic_fetch_fminimum(i2, *i2, __ATOMIC_RELAXED); // expected-error {{must be a pointer to floating point type}} +} + +void test_c11(_Atomic(half2) *h2, half2 h2v, _Atomic(int2) *i2, int2 i2v) { + (void)__c11_atomic_load(h2, __ATOMIC_RELAXED); + __c11_atomic_store(h2, h2v, __ATOMIC_RELAXED); + (void)__c11_atomic_exchange(h2, h2v, __ATOMIC_RELAXED); + (void)__c11_atomic_fetch_add(h2, h2v, __ATOMIC_RELAXED); + (void)__c11_atomic_fetch_sub(h2, h2v, __ATOMIC_RELAXED); + (void)__c11_atomic_fetch_xor(i2, i2v, __ATOMIC_RELAXED); +} + +void test_scoped(half2 *h2, int2 *i2) { + (void)__scoped_atomic_load_n(h2, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store_n(h2, *h2, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + (void)__scoped_atomic_fetch_add(h2, *h2, __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE); + (void)__scoped_atomic_fetch_or(i2, *i2, __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE); +} + +void test_unsupported(float3 *f3, float8 *f8, bool8 *b8, bitint2 *bi2) { + // A vector whose size is not a power of two cannot be an atomic operand. + (void)__atomic_fetch_add(f3, *f3, __ATOMIC_RELAXED); // expected-error {{must be a pointer to a vector with a power-of-two size of at most 16 bytes}} + // A vector larger than the largest atomic emitted inline would need a + // libcall, and there is no libcall for the arithmetic operations. + (void)__atomic_fetch_add(f8, *f8, __ATOMIC_RELAXED); // expected-error {{must be a pointer to a vector with a power-of-two size of at most 16 bytes}} + (void)__atomic_fetch_add(b8, *b8, __ATOMIC_RELAXED); // expected-error {{must be a pointer to integer, pointer or supported floating point type}} + (void)__atomic_fetch_add(bi2, *bi2, __ATOMIC_RELAXED); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}} +} >From c357a0ca34fde85b6fab23421ff0c39a6b8877ac Mon Sep 17 00:00:00 2001 From: Akash Manna <[email protected]> Date: Sat, 8 Aug 2026 23:05:41 +0530 Subject: [PATCH 2/4] [CodeGen] Refactor atomic operation emission for improved readability --- clang/lib/CodeGen/CGAtomic.cpp | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index 2760cd9b03eb6..7164ee8c650d4 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -709,10 +709,10 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__scoped_atomic_fetch_min: - Op = ElemTy->isFloatingType() ? llvm::AtomicRMWInst::FMin - : (ElemTy->isSignedIntegerType() - ? llvm::AtomicRMWInst::Min - : llvm::AtomicRMWInst::UMin); + Op = ElemTy->isFloatingType() + ? llvm::AtomicRMWInst::FMin + : (ElemTy->isSignedIntegerType() ? llvm::AtomicRMWInst::Min + : llvm::AtomicRMWInst::UMin); break; case AtomicExpr::AO__atomic_fetch_fminimum: @@ -738,10 +738,10 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_fetch_max: case AtomicExpr::AO__atomic_fetch_max: case AtomicExpr::AO__scoped_atomic_fetch_max: - Op = ElemTy->isFloatingType() ? llvm::AtomicRMWInst::FMax - : (ElemTy->isSignedIntegerType() - ? llvm::AtomicRMWInst::Max - : llvm::AtomicRMWInst::UMax); + Op = ElemTy->isFloatingType() + ? llvm::AtomicRMWInst::FMax + : (ElemTy->isSignedIntegerType() ? llvm::AtomicRMWInst::Max + : llvm::AtomicRMWInst::UMax); break; case AtomicExpr::AO__atomic_fetch_fmaximum: @@ -845,9 +845,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, // determine the value which was written. llvm::Value *Result = RMWI; if (PostOpMinMax) - Result = EmitPostAtomicMinMax(CGF.Builder, E->getOp(), - ElemTy->isSignedIntegerType(), RMWI, - LoadVal1); + Result = EmitPostAtomicMinMax( + CGF.Builder, E->getOp(), ElemTy->isSignedIntegerType(), RMWI, LoadVal1); else if (PostOp) Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, LoadVal1); @@ -879,8 +878,8 @@ static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { // vector whose size is not a power of two is not a valid atomic operand, so // both are still handled as an integer of the enclosing atomic type's size. if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(ValTy)) - return CmpXchg || - !llvm::isPowerOf2_64(VecTy->getPrimitiveSizeInBits().getFixedValue()); + return CmpXchg || !llvm::isPowerOf2_64( + VecTy->getPrimitiveSizeInBits().getFixedValue()); if (ValTy->isFloatingPointTy()) return ValTy->isX86_FP80Ty() || CmpXchg; return !ValTy->isIntegerTy() && !ValTy->isPointerTy(); >From 48cbbb5f1f28b48cb5725865d8339ed09b39c115 Mon Sep 17 00:00:00 2001 From: Akash Manna <[email protected]> Date: Sun, 9 Aug 2026 00:01:16 +0530 Subject: [PATCH 3/4] [Clang] Update OpenMP atomic read test for vector atomic loads An atomic load of a vector is now emitted with the vector type rather than an integer of the same size, so the two CHECK blocks covering int4x and float2x expected a load atomic i128 and a load atomic i64 that are no longer generated. The remaining i128 expectations in the file cover long double and _Complex, which still use the integer path, and the write, update and capture tests reach these globals through cmpxchg, which has no vector form and is also unchanged. --- clang/test/OpenMP/atomic_read_codegen.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/clang/test/OpenMP/atomic_read_codegen.c b/clang/test/OpenMP/atomic_read_codegen.c index 8079d5fd557a3..36e1a1a7d28b3 100644 --- a/clang/test/OpenMP/atomic_read_codegen.c +++ b/clang/test/OpenMP/atomic_read_codegen.c @@ -231,8 +231,8 @@ int main(void) { // CHECK: store double #pragma omp atomic read cdv = llx; -// CHECK: [[I128VAL:%.+]] = load atomic i128, ptr @{{.+}} monotonic, align 16 -// CHECK: store i128 [[I128VAL]], ptr [[LDTEMP:%.+]] +// CHECK: [[VECVAL:%.+]] = load atomic <4 x i32>, ptr @{{.+}} monotonic, align 16 +// CHECK: store <4 x i32> [[VECVAL]], ptr [[LDTEMP:%.+]] // CHECK: [[LD:%.+]] = load <4 x i32>, ptr [[LDTEMP]] // CHECK: extractelement <4 x i32> [[LD]] // CHECK: store i8 @@ -318,8 +318,8 @@ int main(void) { // CHECK: store x86_fp80 #pragma omp atomic read acquire ldv = bfx4_packed.b; -// CHECK: [[LD:%.+]] = load atomic i64, ptr @{{.+}} monotonic, align 8 -// CHECK: store i64 [[LD]], ptr [[LDTEMP:%.+]] +// CHECK: [[VECVAL:%.+]] = load atomic <2 x float>, ptr @{{.+}} monotonic, align 8 +// CHECK: store <2 x float> [[VECVAL]], ptr [[LDTEMP:%.+]] // CHECK: [[LD:%.+]] = load <2 x float>, ptr [[LDTEMP]] // CHECK: extractelement <2 x float> [[LD]] // CHECK: store i64 >From f0893fbc1b256f30a3f3438b2ed61b8b8668ffd0 Mon Sep 17 00:00:00 2001 From: Akash Manna <[email protected]> Date: Wed, 12 Aug 2026 20:05:42 +0530 Subject: [PATCH 4/4] [Clang] Refactor comments in atomic operation handling for clarity and consistency --- clang/lib/CIR/CodeGen/CIRGenAtomic.cpp | 5 ++--- clang/lib/CodeGen/CGAtomic.cpp | 12 +++++------- clang/lib/Sema/SemaChecking.cpp | 15 +++++---------- clang/test/CodeGen/atomic-ops-vector.c | 6 +----- clang/test/Sema/atomic-ops-vector.c | 8 -------- 5 files changed, 13 insertions(+), 33 deletions(-) diff --git a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp index 21b145a9cbee4..822c07d139df9 100644 --- a/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp +++ b/clang/lib/CIR/CodeGen/CIRGenAtomic.cpp @@ -326,9 +326,8 @@ bool AtomicInfo::emitMemSetZeroIfNecessary() const { /// floating point operands. TODO: Allow compare-and-exchange and FP - see /// comment in CIRGenAtomicExpandPass.cpp. static bool shouldCastToInt(mlir::Type valueTy, bool cmpxchg) { - // The atomic operations act on a vector directly, which is required for the - // elementwise arithmetic operations. cmpxchg has no vector form, so it is - // still handled as an integer of the same size. + // The atomic operations act on a vector directly, but cmpxchg has no vector + // form, so it is still handled as an integer. if (isa<cir::VectorType>(valueTy)) return cmpxchg; if (cir::isAnyFloatingPointType(valueTy)) diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index 7164ee8c650d4..39262da455a7c 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -568,9 +568,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, bool PostOpMinMax = false; unsigned PostOp = 0; - // Atomic operations on a vector are performed elementwise, so it is the - // element type which selects between the integer and the floating point - // form of an operation. + // A vector is operated on elementwise, so its element type selects between + // the integer and the floating point form of an operation. QualType ElemTy = E->getValueType(); if (const auto *VecTy = ElemTy->getAs<VectorType>()) ElemTy = VecTy->getElementType(); @@ -873,10 +872,9 @@ EmitValToTemp(CodeGenFunction &CGF, Expr *E) { /// floating point operands. TODO: Allow compare-and-exchange and FP - see /// comment in AtomicExpandPass.cpp. static bool shouldCastToInt(llvm::Type *ValTy, bool CmpXchg) { - // The atomic instructions operate on a vector directly, which is required for - // the elementwise arithmetic operations. cmpxchg has no vector form, and a - // vector whose size is not a power of two is not a valid atomic operand, so - // both are still handled as an integer of the enclosing atomic type's size. + // The atomic instructions operate on a vector directly. cmpxchg has no vector + // form, and a vector whose size is not a power of two is not a valid atomic + // operand, so both are still handled as an integer. if (auto *VecTy = dyn_cast<llvm::FixedVectorType>(ValTy)) return CmpXchg || !llvm::isPowerOf2_64( VecTy->getPrimitiveSizeInBits().getFixedValue()); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index bd9252f409f3d..b7d630fa34fde 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5335,10 +5335,8 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, // trivial type errors. auto IsAllowedValueType = [&](QualType ValType, unsigned AllowedType) -> bool { - // Atomic operations on a vector are performed elementwise, so it is the - // element type which decides whether the operation is well-formed. A - // vector of _Bool is rejected outright, as its elements are not - // individually addressable. + // A vector is operated on elementwise, so its element type decides. The + // elements of a vector of _Bool are not individually addressable. if (const auto *VecTy = ValType->getAs<VectorType>()) { if (VecTy->getElementType()->isBooleanType()) return false; @@ -5376,14 +5374,11 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, << IsC11 << Ptr->getType() << Ptr->getSourceRange(); return ExprError(); } - // An arithmetic operation on a vector is always emitted as a single - // atomicrmw instruction; unlike the other forms there is no libcall to fall - // back on for a size which cannot be handled inline. The size of the vector - // itself is checked here, as the type holding it is padded out when the - // element count is not a power of two. + // An arithmetic operation always becomes an atomicrmw, with no libcall to + // fall back on. The size is computed from the elements, as the type + // holding them is padded when their count is not a power of two. if (Form == Arithmetic) { if (const auto *VecTy = ValType->getAs<VectorType>()) { - // The largest size EmitAtomicExpr() emits without a libcall. constexpr unsigned MaxVectorSizeInBytes = 16; uint64_t VecSizeInBits = Context.getTypeSize(VecTy->getElementType()) * VecTy->getNumElements(); diff --git a/clang/test/CodeGen/atomic-ops-vector.c b/clang/test/CodeGen/atomic-ops-vector.c index a6bb8d13fef39..0ec4253eb9b4f 100644 --- a/clang/test/CodeGen/atomic-ops-vector.c +++ b/clang/test/CodeGen/atomic-ops-vector.c @@ -4,9 +4,6 @@ // RUN: %clang_cc1 %s -emit-llvm -o - -triple=amdgcn-amd-amdhsa \ // RUN: -Wno-atomic-alignment | FileCheck --check-prefixes=CHECK,AMDGCN %s -// Atomic operations on a vector are performed elementwise by a single atomic -// instruction, rather than being bitcast to an integer of the same size. - typedef _Float16 half2 __attribute__((ext_vector_type(2))); typedef __bf16 bfloat2 __attribute__((ext_vector_type(2))); typedef float float2 __attribute__((ext_vector_type(2))); @@ -86,8 +83,7 @@ half2 test_scoped_fetch_add(half2 *p, half2 v) { __MEMORY_SCOPE_DEVICE); } -// A vector whose size is not a power of two is not a valid atomic operand, so -// it is still accessed as an integer of the size of the type holding it. +// A vector whose size is not a power of two is still accessed as an integer. // CHECK-LABEL: @test_load_float3( float3 test_load_float3(float3 *p) { // CHECK: load atomic i128, ptr {{.*}} monotonic diff --git a/clang/test/Sema/atomic-ops-vector.c b/clang/test/Sema/atomic-ops-vector.c index 752bb4a5bb933..83745da59e4f9 100644 --- a/clang/test/Sema/atomic-ops-vector.c +++ b/clang/test/Sema/atomic-ops-vector.c @@ -1,9 +1,6 @@ // RUN: %clang_cc1 %s -verify -fsyntax-only -triple=x86_64-unknown-linux-gnu -std=c11 // RUN: %clang_cc1 %s -verify -fsyntax-only -triple=amdgcn-amd-amdhsa -std=c11 -// Atomic builtins accept vector types; the operation is performed elementwise -// by a single atomicrmw instruction. - typedef _Float16 half2 __attribute__((ext_vector_type(2))); typedef __bf16 bfloat2 __attribute__((ext_vector_type(2))); typedef float float2 __attribute__((ext_vector_type(2))); @@ -30,9 +27,7 @@ void test_gnu(half2 *h2, bfloat2 *b2, float2 *f2, int2 *i2, uint4 *u4) { (void)__atomic_or_fetch(u4, *u4, __ATOMIC_RELAXED); (void)__atomic_nand_fetch(i2, *i2, __ATOMIC_RELAXED); - // The integer operations still reject a vector of floating point elements. (void)__atomic_fetch_and(f2, *f2, __ATOMIC_RELAXED); // expected-error {{must be a pointer to integer}} - // The f-prefixed operations still reject a vector of integer elements. (void)__atomic_fetch_fminimum(i2, *i2, __ATOMIC_RELAXED); // expected-error {{must be a pointer to floating point type}} } @@ -55,10 +50,7 @@ void test_scoped(half2 *h2, int2 *i2) { } void test_unsupported(float3 *f3, float8 *f8, bool8 *b8, bitint2 *bi2) { - // A vector whose size is not a power of two cannot be an atomic operand. (void)__atomic_fetch_add(f3, *f3, __ATOMIC_RELAXED); // expected-error {{must be a pointer to a vector with a power-of-two size of at most 16 bytes}} - // A vector larger than the largest atomic emitted inline would need a - // libcall, and there is no libcall for the arithmetic operations. (void)__atomic_fetch_add(f8, *f8, __ATOMIC_RELAXED); // expected-error {{must be a pointer to a vector with a power-of-two size of at most 16 bytes}} (void)__atomic_fetch_add(b8, *b8, __ATOMIC_RELAXED); // expected-error {{must be a pointer to integer, pointer or supported floating point type}} (void)__atomic_fetch_add(bi2, *bi2, __ATOMIC_RELAXED); // expected-error {{argument to atomic builtin of type '_BitInt' is not supported}} _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
