https://github.com/jofrn created https://github.com/llvm/llvm-project/pull/197861
This commit casts floats to ints in an atomic store during AtomicExpand to support floating point types. It also is required to support 128 bit vectors in SSE/AVX. Smaller FP vectors (\`<N x half>\`, \`<N x bfloat>\`) are left to the DAG widen path on subtargets without native FP16/BF16 support; the v8f16/v8bf16 bitconvert variants added to the PR5 patterns let the widened path collapse to a single instruction on AVX+ targets. Store-side counterpart to #148899. Stacked on top of #197860. >From 6b8f49c3817d741d22c94fc46cf140e878094aac Mon Sep 17 00:00:00 2001 From: jofrn <[email protected]> Date: Thu, 14 May 2026 12:55:03 -0700 Subject: [PATCH] [X86] Cast atomic vectors in IR to support floats This commit casts floats to ints in an atomic store during AtomicExpand to support floating point types. It also is required to support 128 bit vectors in SSE/AVX. Smaller FP vectors (<N x half>, <N x bfloat>) are left to the DAG widen path on subtargets without native FP16/BF16 support, since the IR-level bitcast to scalar integer otherwise gets scalarized by the type legalizer into a pextrw+shl+or sequence. Added v8f16/v8bf16 bitconvert variants to PR5's atomic_store_32/64 patterns so the widened path produces a single instruction on AVX+ targets. --- .../include/llvm/Target/TargetSelectionDAG.td | 14 +++ llvm/lib/Target/X86/X86ISelLowering.cpp | 13 +++ llvm/lib/Target/X86/X86ISelLowering.h | 2 + llvm/lib/Target/X86/X86InstrCompiler.td | 77 +++++++++++++++ llvm/test/CodeGen/X86/atomic-load-store.ll | 99 ++++++++++++++++--- 5 files changed, 193 insertions(+), 12 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 35848f76897b3..60ed4ed1a410f 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -2393,6 +2393,20 @@ def atomic_store_128 : let MemoryVT = i128; } +def atomic_store_128_v2i64 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = v2i64; +} + +def atomic_store_128_v4i32 : + PatFrag<(ops node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)> { + let IsAtomic = true; + let MemoryVT = v4i32; +} + //===----------------------------------------------------------------------===// // Selection DAG Pattern Support. // diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index fea1caf0854f5..064a1e7c138fc 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -32967,6 +32967,19 @@ X86TargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const { return AtomicExpansionKind::None; } +TargetLowering::AtomicExpansionKind +X86TargetLowering::shouldCastAtomicStoreInIR(StoreInst *SI) const { + Type *Ty = SI->getValueOperand()->getType(); + if (!Ty->getScalarType()->isFloatingPointTy()) + return AtomicExpansionKind::None; + // Sub-128-bit FP vectors codegen better when DAG widening folds the value + // into an extractelt-from-XMM pattern, instead of an IR-level bitcast to a + // scalar integer (which the type legalizer scalarizes). + if (Ty->isVectorTy() && Ty->getPrimitiveSizeInBits() < 128) + return AtomicExpansionKind::None; + return AtomicExpansionKind::CastToInteger; +} + LoadInst * X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const { unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32; diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 9a958525057b6..b26f95ddea388 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -894,6 +894,8 @@ namespace llvm { shouldExpandLogicAtomicRMWInIR(const AtomicRMWInst *AI) const; TargetLoweringBase::AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override; + TargetLoweringBase::AtomicExpansionKind + shouldCastAtomicStoreInIR(StoreInst *SI) const override; void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override; void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override; diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td index b2a7bce8d7571..f40edbf911e6f 100644 --- a/llvm/lib/Target/X86/X86InstrCompiler.td +++ b/llvm/lib/Target/X86/X86InstrCompiler.td @@ -1242,6 +1242,21 @@ def : Pat<(v4i32 (atomic_load_128_v4i32 addr:$src)), def : Pat<(v4i32 (atomic_load_128_v4i32 addr:$src)), (VMOVAPDZ128rm addr:$src)>, Requires<[HasAVX512]>; +// store atomic <2 x i64> +def : Pat<(atomic_store_128_v2i64 (v2i64 VR128:$src), addr:$dst), + (MOVAPDmr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; +def : Pat<(atomic_store_128_v2i64 (v2i64 VR128:$src), addr:$dst), + (VMOVAPDmr addr:$dst, VR128:$src)>, Requires<[UseAVX]>; +def : Pat<(atomic_store_128_v2i64 (v2i64 VR128X:$src), addr:$dst), + (VMOVAPDZ128mr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; +// store atomic <4 x i32> +def : Pat<(atomic_store_128_v4i32 (v4i32 VR128:$src), addr:$dst), + (MOVAPDmr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; +def : Pat<(atomic_store_128_v4i32 (v4i32 VR128:$src), addr:$dst), + (VMOVAPDmr addr:$dst, VR128:$src)>, Requires<[UseAVX]>; +def : Pat<(atomic_store_128_v4i32 (v4i32 VR128X:$src), addr:$dst), + (VMOVAPDZ128mr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; + // store atomic <2 x i8> def : Pat<(atomic_store_16 (i16 (trunc (i32 (extractelt @@ -1293,6 +1308,37 @@ def : Pat<(atomic_store_32 (v4i32 (bitconvert (v16i8 VR128X:$src))), (iPTR 0))), addr:$dst), (VMOVPDI2DIZmr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; +// store atomic <2 x half>, <2 x bfloat> (via widened v8f16, v8bf16) +def : Pat<(atomic_store_32 + (i32 (extractelt + (v4i32 (bitconvert (v8f16 VR128:$src))), (iPTR 0))), + addr:$dst), + (MOVPDI2DImr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; +def : Pat<(atomic_store_32 + (i32 (extractelt + (v4i32 (bitconvert (v8bf16 VR128:$src))), (iPTR 0))), + addr:$dst), + (MOVPDI2DImr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; +def : Pat<(atomic_store_32 + (i32 (extractelt + (v4i32 (bitconvert (v8f16 VR128:$src))), (iPTR 0))), + addr:$dst), + (VMOVPDI2DImr addr:$dst, VR128:$src)>, Requires<[UseAVX]>; +def : Pat<(atomic_store_32 + (i32 (extractelt + (v4i32 (bitconvert (v8bf16 VR128:$src))), (iPTR 0))), + addr:$dst), + (VMOVPDI2DImr addr:$dst, VR128:$src)>, Requires<[UseAVX]>; +def : Pat<(atomic_store_32 + (i32 (extractelt + (v4i32 (bitconvert (v8f16 VR128X:$src))), (iPTR 0))), + addr:$dst), + (VMOVPDI2DIZmr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; +def : Pat<(atomic_store_32 + (i32 (extractelt + (v4i32 (bitconvert (v8bf16 VR128X:$src))), (iPTR 0))), + addr:$dst), + (VMOVPDI2DIZmr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; // store atomic <2 x i32,float>, <4 x i16>, <2 x ptr addrspace(270)> def : Pat<(atomic_store_64 @@ -1340,6 +1386,37 @@ def : Pat<(atomic_store_64 (v2i64 (bitconvert (v8i16 VR128X:$src))), (iPTR 0))), addr:$dst), (VMOVPQI2QIZmr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; +// store atomic <4 x half>, <4 x bfloat> (via widened v8f16, v8bf16) +def : Pat<(atomic_store_64 + (i64 (extractelt + (v2i64 (bitconvert (v8f16 VR128:$src))), (iPTR 0))), + addr:$dst), + (MOVPQI2QImr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; +def : Pat<(atomic_store_64 + (i64 (extractelt + (v2i64 (bitconvert (v8bf16 VR128:$src))), (iPTR 0))), + addr:$dst), + (MOVPQI2QImr addr:$dst, VR128:$src)>, Requires<[UseSSE2]>; +def : Pat<(atomic_store_64 + (i64 (extractelt + (v2i64 (bitconvert (v8f16 VR128:$src))), (iPTR 0))), + addr:$dst), + (VMOVPQI2QImr addr:$dst, VR128:$src)>, Requires<[UseAVX]>; +def : Pat<(atomic_store_64 + (i64 (extractelt + (v2i64 (bitconvert (v8bf16 VR128:$src))), (iPTR 0))), + addr:$dst), + (VMOVPQI2QImr addr:$dst, VR128:$src)>, Requires<[UseAVX]>; +def : Pat<(atomic_store_64 + (i64 (extractelt + (v2i64 (bitconvert (v8f16 VR128X:$src))), (iPTR 0))), + addr:$dst), + (VMOVPQI2QIZmr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; +def : Pat<(atomic_store_64 + (i64 (extractelt + (v2i64 (bitconvert (v8bf16 VR128X:$src))), (iPTR 0))), + addr:$dst), + (VMOVPQI2QIZmr addr:$dst, VR128X:$src)>, Requires<[HasAVX512]>; // Floating point loads/stores. def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst), diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll index 066842739fb61..1d0d0d4dc5c6b 100644 --- a/llvm/test/CodeGen/X86/atomic-load-store.ll +++ b/llvm/test/CodeGen/X86/atomic-load-store.ll @@ -765,8 +765,7 @@ define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) { ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_half: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O3-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec2_half: @@ -788,8 +787,7 @@ define void @store_atomic_vec2_half(ptr %x, <2 x half> %v) { ; ; CHECK-AVX-O0-LABEL: store_atomic_vec2_half: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O0-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <2 x half> %v, ptr %x release, align 4 ret void @@ -809,8 +807,7 @@ define void @store_atomic_vec2_bfloat(ptr %x, <2 x bfloat> %v) { ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_bfloat: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O3-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec2_bfloat: @@ -932,8 +929,7 @@ define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind { ; ; CHECK-AVX-O3-LABEL: store_atomic_vec4_half: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O3-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE2-O0-LABEL: store_atomic_vec4_half: @@ -1007,8 +1003,7 @@ define void @store_atomic_vec4_half(ptr %x, <4 x half> %v) nounwind { ; ; CHECK-AVX-O0-LABEL: store_atomic_vec4_half: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O0-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <4 x half> %v, ptr %x release, align 8 ret void @@ -1060,8 +1055,7 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind { ; ; CHECK-AVX-O3-LABEL: store_atomic_vec4_bfloat: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O3-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec4_bfloat: @@ -1201,6 +1195,87 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind { ret void } +define void @store_atomic_vec4_float_align(ptr %x, <4 x float> %v) nounwind { +; CHECK-SSE2-O3-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE2-O3: # %bb.0: +; CHECK-SSE2-O3-NEXT: pushq %rax +; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi +; CHECK-SSE2-O3-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx +; CHECK-SSE2-O3-NEXT: movl $3, %ecx +; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT +; CHECK-SSE2-O3-NEXT: popq %rax +; CHECK-SSE2-O3-NEXT: retq +; +; CHECK-SSE4-O3-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE4-O3: # %bb.0: +; CHECK-SSE4-O3-NEXT: pushq %rbx +; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm0, %rcx +; CHECK-SSE4-O3-NEXT: movq %xmm0, %rbx +; CHECK-SSE4-O3-NEXT: movq (%rdi), %rax +; CHECK-SSE4-O3-NEXT: movq 8(%rdi), %rdx +; CHECK-SSE4-O3-NEXT: .p2align 4 +; CHECK-SSE4-O3-NEXT: .LBB39_1: # %atomicrmw.start +; CHECK-SSE4-O3-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi) +; CHECK-SSE4-O3-NEXT: jne .LBB39_1 +; CHECK-SSE4-O3-NEXT: # %bb.2: # %atomicrmw.end +; CHECK-SSE4-O3-NEXT: popq %rbx +; CHECK-SSE4-O3-NEXT: retq +; +; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_align: +; CHECK-AVX-O3: # %bb.0: +; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi) +; CHECK-AVX-O3-NEXT: retq +; +; CHECK-SSE2-O0-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE2-O0: # %bb.0: +; CHECK-SSE2-O0-NEXT: pushq %rax +; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi +; CHECK-SSE2-O0-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx +; CHECK-SSE2-O0-NEXT: movl $3, %ecx +; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT +; CHECK-SSE2-O0-NEXT: popq %rax +; CHECK-SSE2-O0-NEXT: retq +; +; CHECK-SSE4-O0-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE4-O0: # %bb.0: +; CHECK-SSE4-O0-NEXT: pushq %rbx +; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm0, %rax +; CHECK-SSE4-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: movq %xmm0, %rax +; CHECK-SSE4-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: movq (%rdi), %rax +; CHECK-SSE4-O0-NEXT: movq 8(%rdi), %rdx +; CHECK-SSE4-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: jmp .LBB39_1 +; CHECK-SSE4-O0-NEXT: .LBB39_1: # %atomicrmw.start +; CHECK-SSE4-O0-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload +; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload +; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload +; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload +; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload +; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rsi) +; CHECK-SSE4-O0-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: jne .LBB39_1 +; CHECK-SSE4-O0-NEXT: jmp .LBB39_2 +; CHECK-SSE4-O0-NEXT: .LBB39_2: # %atomicrmw.end +; CHECK-SSE4-O0-NEXT: popq %rbx +; CHECK-SSE4-O0-NEXT: retq +; +; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_align: +; CHECK-AVX-O0: # %bb.0: +; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rdi) +; CHECK-AVX-O0-NEXT: retq + store atomic <4 x float> %v, ptr %x release, align 16 + ret void +} + define <2 x half> @atomic_vec2_half(ptr %x) { ; CHECK-SSE-O3-LABEL: atomic_vec2_half: ; CHECK-SSE-O3: # %bb.0: _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
