Author: Yonah Goldberg Date: 2026-07-25T17:39:46+02:00 New Revision: 3b484a37805b99e0b617b606aea1c3d45cf18765
URL: https://github.com/llvm/llvm-project/commit/3b484a37805b99e0b617b606aea1c3d45cf18765 DIFF: https://github.com/llvm/llvm-project/commit/3b484a37805b99e0b617b606aea1c3d45cf18765.diff LOG: port Added: Modified: llvm/docs/LangRef.md llvm/lib/AsmParser/LLParser.cpp llvm/lib/IR/Verifier.cpp llvm/test/Assembler/atomic.ll llvm/test/Assembler/invalid-atomicrmw-elementwise.ll llvm/test/Bitcode/atomicrmw-elementwise.ll llvm/test/Bitcode/compatibility.ll llvm/unittests/IR/VerifierTest.cpp Removed: ################################################################################ diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md index f555ca543ffdc..61d23eeee39cc 100644 --- a/llvm/docs/LangRef.md +++ b/llvm/docs/LangRef.md @@ -4177,6 +4177,29 @@ Otherwise, an atomic operation that is not marked monotonic modification order with other operations that are not marked `syncscope("singlethread")` or `syncscope("<target-scope>")`. +(elementwise-atomics)= + +### Elementwise Atomic Operations + +The {ref}`atomicrmw <i_atomicrmw>` instruction may be marked `elementwise`. +The access type must then be a fixed vector type whose total bit width is a +power of two and whose element type is supported by the corresponding scalar +atomic instruction. The {ref}`ordering <ordering>` of an `elementwise` +instruction may not be `seq_cst`. + +An `elementwise` atomic instruction behaves as if it were expanded into one +scalar version of that instruction for each vector element. Each resulting +scalar operation has the same {ref}`ordering <ordering>` and `syncscope` as the +original instruction. Each scalar operation occupies the original instruction's +position in program order relative to other operations, but the scalar +operations are not related in program order with respect to one another. +Synchronizing with one scalar operation therefore does not, by itself, +establish a happens-before relationship with any other scalar operation from +the same `elementwise` instruction. + +Without `elementwise`, vector `atomicrmw` instructions are performed atomically +over the entire vector operation. + (floatenv)= ### Floating-Point Environment @@ -12129,9 +12152,8 @@ isn't specified. An `atomicrmw` instruction can also take an optional "{ref}`syncscope <syncscope>`" argument. -If the `elementwise` modifier is present, the instruction has per-element vector -atomic semantics. It behaves as if it were expanded into one scalar `atomicrmw` per element, that are not ordered with respect to each other. -Without `elementwise`, vector `atomicrmw` keeps whole-value atomic semantics. +If `atomicrmw` is marked `elementwise`, the instruction has +{ref}`elementwise atomic semantics <elementwise-atomics>`. ##### Semantics: diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 38d10587b104e..618abb1bd423a 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -9204,6 +9204,8 @@ int LLParser::parseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { if (Ordering == AtomicOrdering::Unordered) return tokError("atomicrmw cannot be unordered"); + if (IsElementwise && Ordering == AtomicOrdering::SequentiallyConsistent) + return tokError("atomicrmw elementwise cannot be sequentially consistent"); if (!Ptr->getType()->isPointerTy()) return error(PtrLoc, "atomicrmw operand must be a pointer"); if (Val->getType()->isScalableTy()) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index fa0fe7a2e6092..9a455fbc9256a 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -4706,6 +4706,8 @@ void Verifier::visitAtomicRMWInst(AtomicRMWInst &RMWI) { Type *ElTy = RMWI.getOperand(1)->getType(); Type *ScalarTy = ElTy; if (RMWI.isElementwise()) { + Check(RMWI.getOrdering() != AtomicOrdering::SequentiallyConsistent, + "atomicrmw elementwise cannot be sequentially consistent.", &RMWI); auto *VecTy = dyn_cast<FixedVectorType>(ElTy); Check(VecTy, "atomicrmw elementwise operand must have fixed vector type!", &RMWI, ElTy); diff --git a/llvm/test/Assembler/atomic.ll b/llvm/test/Assembler/atomic.ll index 609cd33f61b88..fab483d1cfd27 100644 --- a/llvm/test/Assembler/atomic.ll +++ b/llvm/test/Assembler/atomic.ll @@ -154,8 +154,8 @@ define void @fp_vector_atomicrmw(ptr %x, <2 x half> %val) { ; CHECK: %atomic.elem.fadd = atomicrmw elementwise fadd ptr %x, <2 x half> %val monotonic %atomic.elem.fadd = atomicrmw elementwise fadd ptr %x, <2 x half> %val monotonic - ; CHECK: %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val seq_cst - %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val seq_cst + ; CHECK: %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val acq_rel + %atomic.elem.fadd.vol = atomicrmw volatile elementwise fadd ptr %x, <2 x half> %val acq_rel ret void } diff --git a/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll b/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll index 2900779420b0c..3c134cbb3b8c2 100644 --- a/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll +++ b/llvm/test/Assembler/invalid-atomicrmw-elementwise.ll @@ -3,6 +3,7 @@ ; RUN: not llvm-as -disable-output %t/odd-sized.ll 2>&1 | FileCheck %t/odd-sized.ll ; RUN: not llvm-as -disable-output %t/add-must-be-integer.ll 2>&1 | FileCheck %t/add-must-be-integer.ll ; RUN: not llvm-as -disable-output %t/fadd-must-be-fp.ll 2>&1 | FileCheck %t/fadd-must-be-fp.ll +; RUN: not llvm-as -disable-output %t/seq-cst.ll 2>&1 | FileCheck %t/seq-cst.ll ;--- scalar.ll ; CHECK: atomicrmw elementwise operand must be a fixed vector type @@ -31,3 +32,10 @@ define <4 x i32> @bad_fadd(ptr %p, <4 x i32> %v) { %old = atomicrmw elementwise fadd ptr %p, <4 x i32> %v monotonic ret <4 x i32> %old } + +;--- seq-cst.ll +; CHECK: atomicrmw elementwise cannot be sequentially consistent +define <4 x i32> @bad_seq_cst(ptr %p, <4 x i32> %v) { + %old = atomicrmw elementwise add ptr %p, <4 x i32> %v seq_cst + ret <4 x i32> %old +} diff --git a/llvm/test/Bitcode/atomicrmw-elementwise.ll b/llvm/test/Bitcode/atomicrmw-elementwise.ll index db9c48a80047e..c150a7bf14470 100644 --- a/llvm/test/Bitcode/atomicrmw-elementwise.ll +++ b/llvm/test/Bitcode/atomicrmw-elementwise.ll @@ -10,7 +10,7 @@ define <4 x i32> @elem_add(ptr %p, <4 x i32> %v) { define <4 x float> @elem_fadd(ptr %p, <4 x float> %v) { ; CHECK-LABEL: @elem_fadd( -; CHECK: %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v seq_cst, align 16 - %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v seq_cst +; CHECK: %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v acq_rel, align 16 + %old = atomicrmw elementwise fadd ptr %p, <4 x float> %v acq_rel ret <4 x float> %old } diff --git a/llvm/test/Bitcode/compatibility.ll b/llvm/test/Bitcode/compatibility.ll index 0ac6da0c9ed29..f2e73395617de 100644 --- a/llvm/test/Bitcode/compatibility.ll +++ b/llvm/test/Bitcode/compatibility.ll @@ -1033,8 +1033,8 @@ define void @elementwise_atomics(ptr %word, <4 x i32> %ival, <4 x float> %fval) ; CHECK: %atomicrmw.add = atomicrmw elementwise add ptr %word, <4 x i32> %ival monotonic, align 16 %atomicrmw.add = atomicrmw elementwise add ptr %word, <4 x i32> %ival monotonic, align 16 -; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16 - %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval seq_cst, align 16 +; CHECK: %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval acq_rel, align 16 + %atomicrmw.fadd = atomicrmw elementwise fadd ptr %word, <4 x float> %fval acq_rel, align 16 ret void } diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp index d680dd250d66e..c74390f596093 100644 --- a/llvm/unittests/IR/VerifierTest.cpp +++ b/llvm/unittests/IR/VerifierTest.cpp @@ -429,6 +429,31 @@ TEST(VerifierTest, AtomicRMWElementwiseScalar) { << Error; } +TEST(VerifierTest, AtomicRMWElementwiseSequentiallyConsistent) { + LLVMContext C; + Module M("M", C); + FunctionType *FTy = FunctionType::get(Type::getVoidTy(C), /*isVarArg=*/false); + Function *F = Function::Create(FTy, Function::ExternalLinkage, "foo", M); + BasicBlock *Entry = BasicBlock::Create(C, "entry", F); + Value *Ptr = PoisonValue::get(PointerType::get(C, 0)); + + Type *I32Ty = Type::getInt32Ty(C); + Constant *CV = ConstantVector::getSplat(ElementCount::getFixed(2), + ConstantInt::get(I32Ty, 0)); + + new AtomicRMWInst(AtomicRMWInst::Add, Ptr, CV, Align(8), + AtomicOrdering::SequentiallyConsistent, SyncScope::System, + /*Elementwise=*/true, Entry); + ReturnInst::Create(C, Entry); + + std::string Error; + raw_string_ostream ErrorOS(Error); + EXPECT_TRUE(verifyFunction(*F, &ErrorOS)); + EXPECT_TRUE(StringRef(Error).starts_with( + "atomicrmw elementwise cannot be sequentially consistent.")) + << Error; +} + TEST(VerifierTest, AtomicRMWElementwiseIntOpOnFPVector) { LLVMContext C; Module M("M", C); _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
