https://github.com/jofrn updated
https://github.com/llvm/llvm-project/pull/138635
>From 109bc6009d93645b42e0be8fbf858368770f49e7 Mon Sep 17 00:00:00 2001
From: jofernau_amdeng <joe.fer...@amd.com>
Date: Tue, 6 May 2025 01:48:11 -0400
Subject: [PATCH] [X86] Remove extra MOV after widening atomic load
This change adds patterns to optimize out an extra MOV
present after widening the atomic load. It also casts floats
to ints in an atomic load during AtomicExpand to support 128 bit
vectors in SSE/AVX.
commit-id:45989503
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 7 +
llvm/lib/Target/X86/X86ISelLowering.h | 2 +
llvm/lib/Target/X86/X86InstrCompiler.td | 7 +
llvm/test/CodeGen/X86/atomic-load-store.ll | 249 +++++++--------------
4 files changed, 102 insertions(+), 163 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp
b/llvm/lib/Target/X86/X86ISelLowering.cpp
index a203c84630fe1..1fc50a36fda72 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -32070,6 +32070,13 @@
X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
}
}
+TargetLowering::AtomicExpansionKind
+X86TargetLowering::shouldCastAtomicLoadInIR(LoadInst *LI) const {
+ if (LI->getType()->getScalarType()->isFloatingPointTy())
+ return AtomicExpansionKind::CastToInteger;
+ return AtomicExpansionKind::None;
+}
+
LoadInst *
X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h
b/llvm/lib/Target/X86/X86ISelLowering.h
index 359f24768b3da..14b79e8d726f6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -1838,6 +1838,8 @@ namespace llvm {
shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
TargetLoweringBase::AtomicExpansionKind
shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const;
+ TargetLoweringBase::AtomicExpansionKind
+ shouldCastAtomicLoadInIR(LoadInst *LI) const override;
void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td
b/llvm/lib/Target/X86/X86InstrCompiler.td
index 927b2c8b22f05..26b76dd1ca83a 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1204,6 +1204,13 @@ def : Pat<(i16 (atomic_load_nonext_16 addr:$src)),
(MOV16rm addr:$src)>;
def : Pat<(i32 (atomic_load_nonext_32 addr:$src)), (MOV32rm addr:$src)>;
def : Pat<(i64 (atomic_load_nonext_64 addr:$src)), (MOV64rm addr:$src)>;
+def : Pat<(v4i32 (scalar_to_vector (i32 (zext (i16 (atomic_load_16
addr:$src)))))),
+ (MOVDI2PDIrm addr:$src)>; // load atomic <2 x i8>
+def : Pat<(v4i32 (scalar_to_vector (i32 (atomic_load_32 addr:$src)))),
+ (MOVDI2PDIrm addr:$src)>; // load atomic <2 x i16>
+def : Pat<(v2i64 (scalar_to_vector (i64 (atomic_load_64 addr:$src)))),
+ (MOV64toPQIrm addr:$src)>; // load atomic <2 x i32,float>
+
// Floating point loads/stores.
def : Pat<(atomic_store_32 (i32 (bitconvert (f32 FR32:$src))), addr:$dst),
(MOVSSmr addr:$dst, FR32:$src)>, Requires<[UseSSE1]>;
diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll
b/llvm/test/CodeGen/X86/atomic-load-store.ll
index ff5391f44bbe3..535a87316e162 100644
--- a/llvm/test/CodeGen/X86/atomic-load-store.ll
+++ b/llvm/test/CodeGen/X86/atomic-load-store.ll
@@ -207,19 +207,19 @@ define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_bfloat:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-O3-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-O3-NEXT: movd %eax, %xmm0
; CHECK-O3-NEXT: retq
;
; CHECK-SSE-O3-LABEL: atomic_vec1_bfloat:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-SSE-O3-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_bfloat:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-AVX-O3-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_bfloat:
@@ -227,8 +227,7 @@ define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-O0-NEXT: movw (%rdi), %cx
; CHECK-O0-NEXT: # implicit-def: $eax
; CHECK-O0-NEXT: movw %cx, %ax
-; CHECK-O0-NEXT: # implicit-def: $xmm0
-; CHECK-O0-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-O0-NEXT: movd %eax, %xmm0
; CHECK-O0-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_bfloat:
@@ -236,8 +235,7 @@ define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-SSE-O0-NEXT: movw (%rdi), %cx
; CHECK-SSE-O0-NEXT: # implicit-def: $eax
; CHECK-SSE-O0-NEXT: movw %cx, %ax
-; CHECK-SSE-O0-NEXT: # implicit-def: $xmm0
-; CHECK-SSE-O0-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_bfloat:
@@ -245,8 +243,7 @@ define <1 x bfloat> @atomic_vec1_bfloat(ptr %x) {
; CHECK-AVX-O0-NEXT: movw (%rdi), %cx
; CHECK-AVX-O0-NEXT: # implicit-def: $eax
; CHECK-AVX-O0-NEXT: movw %cx, %ax
-; CHECK-AVX-O0-NEXT: # implicit-def: $xmm0
-; CHECK-AVX-O0-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x bfloat>, ptr %x acquire, align 2
ret <1 x bfloat> %ret
@@ -319,159 +316,60 @@ define <2 x i8> @atomic_vec2_i8(ptr %x) {
define <2 x i16> @atomic_vec2_i16(ptr %x) {
; CHECK-O3-LABEL: atomic_vec2_i16:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movl (%rdi), %eax
-; CHECK-O3-NEXT: movd %eax, %xmm0
+; CHECK-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-O3-NEXT: retq
;
; CHECK-SSE-O3-LABEL: atomic_vec2_i16:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movl (%rdi), %eax
-; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
+; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec2_i16:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: movl (%rdi), %eax
-; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
+; CHECK-AVX-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec2_i16:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movl (%rdi), %eax
-; CHECK-O0-NEXT: movd %eax, %xmm0
+; CHECK-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-O0-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec2_i16:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movl (%rdi), %eax
-; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec2_i16:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: movl (%rdi), %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
+; CHECK-AVX-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <2 x i16>, ptr %x acquire, align 4
ret <2 x i16> %ret
}
define <2 x ptr addrspace(270)> @atomic_vec2_ptr270(ptr %x) {
-; CHECK-O3-LABEL: atomic_vec2_ptr270:
-; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movq (%rdi), %rax
-; CHECK-O3-NEXT: movq %rax, %xmm0
-; CHECK-O3-NEXT: retq
-;
-; CHECK-SSE-O3-LABEL: atomic_vec2_ptr270:
-; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq (%rdi), %rax
-; CHECK-SSE-O3-NEXT: movq %rax, %xmm0
-; CHECK-SSE-O3-NEXT: retq
-;
-; CHECK-AVX-O3-LABEL: atomic_vec2_ptr270:
-; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O3-NEXT: vmovq %rax, %xmm0
-; CHECK-AVX-O3-NEXT: retq
-;
-; CHECK-O0-LABEL: atomic_vec2_ptr270:
-; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movq (%rdi), %rax
-; CHECK-O0-NEXT: movq %rax, %xmm0
-; CHECK-O0-NEXT: retq
-;
-; CHECK-SSE-O0-LABEL: atomic_vec2_ptr270:
-; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq (%rdi), %rax
-; CHECK-SSE-O0-NEXT: movq %rax, %xmm0
-; CHECK-SSE-O0-NEXT: retq
-;
-; CHECK-AVX-O0-LABEL: atomic_vec2_ptr270:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O0-NEXT: vmovq %rax, %xmm0
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-LABEL: atomic_vec2_ptr270:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq (%rdi), %xmm0
+; CHECK-NEXT: retq
%ret = load atomic <2 x ptr addrspace(270)>, ptr %x acquire, align 8
ret <2 x ptr addrspace(270)> %ret
}
define <2 x i32> @atomic_vec2_i32_align(ptr %x) {
-; CHECK-O3-LABEL: atomic_vec2_i32_align:
-; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movq (%rdi), %rax
-; CHECK-O3-NEXT: movq %rax, %xmm0
-; CHECK-O3-NEXT: retq
-;
-; CHECK-SSE-O3-LABEL: atomic_vec2_i32_align:
-; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq (%rdi), %rax
-; CHECK-SSE-O3-NEXT: movq %rax, %xmm0
-; CHECK-SSE-O3-NEXT: retq
-;
-; CHECK-AVX-O3-LABEL: atomic_vec2_i32_align:
-; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O3-NEXT: vmovq %rax, %xmm0
-; CHECK-AVX-O3-NEXT: retq
-;
-; CHECK-O0-LABEL: atomic_vec2_i32_align:
-; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movq (%rdi), %rax
-; CHECK-O0-NEXT: movq %rax, %xmm0
-; CHECK-O0-NEXT: retq
-;
-; CHECK-SSE-O0-LABEL: atomic_vec2_i32_align:
-; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq (%rdi), %rax
-; CHECK-SSE-O0-NEXT: movq %rax, %xmm0
-; CHECK-SSE-O0-NEXT: retq
-;
-; CHECK-AVX-O0-LABEL: atomic_vec2_i32_align:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O0-NEXT: vmovq %rax, %xmm0
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-LABEL: atomic_vec2_i32_align:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq (%rdi), %xmm0
+; CHECK-NEXT: retq
%ret = load atomic <2 x i32>, ptr %x acquire, align 8
ret <2 x i32> %ret
}
define <2 x float> @atomic_vec2_float_align(ptr %x) {
-; CHECK-O3-LABEL: atomic_vec2_float_align:
-; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movq (%rdi), %rax
-; CHECK-O3-NEXT: movq %rax, %xmm0
-; CHECK-O3-NEXT: retq
-;
-; CHECK-SSE-O3-LABEL: atomic_vec2_float_align:
-; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq (%rdi), %rax
-; CHECK-SSE-O3-NEXT: movq %rax, %xmm0
-; CHECK-SSE-O3-NEXT: retq
-;
-; CHECK-AVX-O3-LABEL: atomic_vec2_float_align:
-; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O3-NEXT: vmovq %rax, %xmm0
-; CHECK-AVX-O3-NEXT: retq
-;
-; CHECK-O0-LABEL: atomic_vec2_float_align:
-; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movq (%rdi), %rax
-; CHECK-O0-NEXT: movq %rax, %xmm0
-; CHECK-O0-NEXT: retq
-;
-; CHECK-SSE-O0-LABEL: atomic_vec2_float_align:
-; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq (%rdi), %rax
-; CHECK-SSE-O0-NEXT: movq %rax, %xmm0
-; CHECK-SSE-O0-NEXT: retq
-;
-; CHECK-AVX-O0-LABEL: atomic_vec2_float_align:
-; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O0-NEXT: vmovq %rax, %xmm0
-; CHECK-AVX-O0-NEXT: retq
+; CHECK-LABEL: atomic_vec2_float_align:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq (%rdi), %xmm0
+; CHECK-NEXT: retq
%ret = load atomic <2 x float>, ptr %x acquire, align 8
ret <2 x float> %ret
}
@@ -556,19 +454,19 @@ define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-O3-LABEL: atomic_vec1_half:
; CHECK-O3: # %bb.0:
; CHECK-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-O3-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-O3-NEXT: movd %eax, %xmm0
; CHECK-O3-NEXT: retq
;
; CHECK-SSE-O3-LABEL: atomic_vec1_half:
; CHECK-SSE-O3: # %bb.0:
; CHECK-SSE-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-SSE-O3-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec1_half:
; CHECK-AVX-O3: # %bb.0:
; CHECK-AVX-O3-NEXT: movzwl (%rdi), %eax
-; CHECK-AVX-O3-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec1_half:
@@ -576,8 +474,7 @@ define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-O0-NEXT: movw (%rdi), %cx
; CHECK-O0-NEXT: # implicit-def: $eax
; CHECK-O0-NEXT: movw %cx, %ax
-; CHECK-O0-NEXT: # implicit-def: $xmm0
-; CHECK-O0-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-O0-NEXT: movd %eax, %xmm0
; CHECK-O0-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec1_half:
@@ -585,8 +482,7 @@ define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-SSE-O0-NEXT: movw (%rdi), %cx
; CHECK-SSE-O0-NEXT: # implicit-def: $eax
; CHECK-SSE-O0-NEXT: movw %cx, %ax
-; CHECK-SSE-O0-NEXT: # implicit-def: $xmm0
-; CHECK-SSE-O0-NEXT: pinsrw $0, %eax, %xmm0
+; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec1_half:
@@ -594,8 +490,7 @@ define <1 x half> @atomic_vec1_half(ptr %x) {
; CHECK-AVX-O0-NEXT: movw (%rdi), %cx
; CHECK-AVX-O0-NEXT: # implicit-def: $eax
; CHECK-AVX-O0-NEXT: movw %cx, %ax
-; CHECK-AVX-O0-NEXT: # implicit-def: $xmm0
-; CHECK-AVX-O0-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0
+; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <1 x half>, ptr %x acquire, align 2
ret <1 x half> %ret
@@ -900,81 +795,109 @@ define <2 x i32> @atomic_vec2_i32(ptr %x) nounwind {
define <4 x i8> @atomic_vec4_i8(ptr %x) nounwind {
; CHECK-O3-LABEL: atomic_vec4_i8:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movl (%rdi), %eax
-; CHECK-O3-NEXT: movd %eax, %xmm0
+; CHECK-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-O3-NEXT: retq
;
; CHECK-SSE-O3-LABEL: atomic_vec4_i8:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movl (%rdi), %eax
-; CHECK-SSE-O3-NEXT: movd %eax, %xmm0
+; CHECK-SSE-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O3-NEXT: retq
;
; CHECK-AVX-O3-LABEL: atomic_vec4_i8:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: movl (%rdi), %eax
-; CHECK-AVX-O3-NEXT: vmovd %eax, %xmm0
+; CHECK-AVX-O3-NEXT: movss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O3-NEXT: retq
;
; CHECK-O0-LABEL: atomic_vec4_i8:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movl (%rdi), %eax
-; CHECK-O0-NEXT: movd %eax, %xmm0
+; CHECK-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-O0-NEXT: retq
;
; CHECK-SSE-O0-LABEL: atomic_vec4_i8:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movl (%rdi), %eax
-; CHECK-SSE-O0-NEXT: movd %eax, %xmm0
+; CHECK-SSE-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-SSE-O0-NEXT: retq
;
; CHECK-AVX-O0-LABEL: atomic_vec4_i8:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: movl (%rdi), %eax
-; CHECK-AVX-O0-NEXT: vmovd %eax, %xmm0
+; CHECK-AVX-O0-NEXT: movd {{.*#+}} xmm0 = mem[0],zero,zero,zero
; CHECK-AVX-O0-NEXT: retq
%ret = load atomic <4 x i8>, ptr %x acquire, align 4
ret <4 x i8> %ret
}
define <4 x i16> @atomic_vec4_i16(ptr %x) nounwind {
-; CHECK-O3-LABEL: atomic_vec4_i16:
+; CHECK-LABEL: atomic_vec4_i16:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq (%rdi), %xmm0
+; CHECK-NEXT: retq
+ %ret = load atomic <4 x i16>, ptr %x acquire, align 8
+ ret <4 x i16> %ret
+}
+
+define <4 x float> @atomic_vec4_float_align(ptr %x) nounwind {
+; CHECK-O3-LABEL: atomic_vec4_float_align:
; CHECK-O3: # %bb.0:
-; CHECK-O3-NEXT: movq (%rdi), %rax
+; CHECK-O3-NEXT: pushq %rax
+; CHECK-O3-NEXT: movl $2, %esi
+; CHECK-O3-NEXT: callq __atomic_load_16@PLT
+; CHECK-O3-NEXT: movq %rdx, %xmm1
; CHECK-O3-NEXT: movq %rax, %xmm0
+; CHECK-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-O3-NEXT: popq %rax
; CHECK-O3-NEXT: retq
;
-; CHECK-SSE-O3-LABEL: atomic_vec4_i16:
+; CHECK-SSE-O3-LABEL: atomic_vec4_float_align:
; CHECK-SSE-O3: # %bb.0:
-; CHECK-SSE-O3-NEXT: movq (%rdi), %rax
+; CHECK-SSE-O3-NEXT: pushq %rbx
+; CHECK-SSE-O3-NEXT: xorl %eax, %eax
+; CHECK-SSE-O3-NEXT: xorl %edx, %edx
+; CHECK-SSE-O3-NEXT: xorl %ecx, %ecx
+; CHECK-SSE-O3-NEXT: xorl %ebx, %ebx
+; CHECK-SSE-O3-NEXT: lock cmpxchg16b (%rdi)
+; CHECK-SSE-O3-NEXT: movq %rdx, %xmm1
; CHECK-SSE-O3-NEXT: movq %rax, %xmm0
+; CHECK-SSE-O3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-SSE-O3-NEXT: popq %rbx
; CHECK-SSE-O3-NEXT: retq
;
-; CHECK-AVX-O3-LABEL: atomic_vec4_i16:
+; CHECK-AVX-O3-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O3: # %bb.0:
-; CHECK-AVX-O3-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O3-NEXT: vmovq %rax, %xmm0
+; CHECK-AVX-O3-NEXT: vmovaps (%rdi), %xmm0
; CHECK-AVX-O3-NEXT: retq
;
-; CHECK-O0-LABEL: atomic_vec4_i16:
+; CHECK-O0-LABEL: atomic_vec4_float_align:
; CHECK-O0: # %bb.0:
-; CHECK-O0-NEXT: movq (%rdi), %rax
+; CHECK-O0-NEXT: pushq %rax
+; CHECK-O0-NEXT: movl $2, %esi
+; CHECK-O0-NEXT: callq __atomic_load_16@PLT
+; CHECK-O0-NEXT: movq %rdx, %xmm1
; CHECK-O0-NEXT: movq %rax, %xmm0
+; CHECK-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-O0-NEXT: popq %rax
; CHECK-O0-NEXT: retq
;
-; CHECK-SSE-O0-LABEL: atomic_vec4_i16:
+; CHECK-SSE-O0-LABEL: atomic_vec4_float_align:
; CHECK-SSE-O0: # %bb.0:
-; CHECK-SSE-O0-NEXT: movq (%rdi), %rax
+; CHECK-SSE-O0-NEXT: pushq %rbx
+; CHECK-SSE-O0-NEXT: xorl %eax, %eax
+; CHECK-SSE-O0-NEXT: movl %eax, %ebx
+; CHECK-SSE-O0-NEXT: movq %rbx, %rax
+; CHECK-SSE-O0-NEXT: movq %rbx, %rdx
+; CHECK-SSE-O0-NEXT: movq %rbx, %rcx
+; CHECK-SSE-O0-NEXT: lock cmpxchg16b (%rdi)
+; CHECK-SSE-O0-NEXT: movq %rdx, %xmm1
; CHECK-SSE-O0-NEXT: movq %rax, %xmm0
+; CHECK-SSE-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-SSE-O0-NEXT: popq %rbx
; CHECK-SSE-O0-NEXT: retq
;
-; CHECK-AVX-O0-LABEL: atomic_vec4_i16:
+; CHECK-AVX-O0-LABEL: atomic_vec4_float_align:
; CHECK-AVX-O0: # %bb.0:
-; CHECK-AVX-O0-NEXT: movq (%rdi), %rax
-; CHECK-AVX-O0-NEXT: vmovq %rax, %xmm0
+; CHECK-AVX-O0-NEXT: vmovaps (%rdi), %xmm0
; CHECK-AVX-O0-NEXT: retq
- %ret = load atomic <4 x i16>, ptr %x acquire, align 8
- ret <4 x i16> %ret
+ %ret = load atomic <4 x float>, ptr %x acquire, align 16
+ ret <4 x float> %ret
}
define <4 x float> @atomic_vec4_float(ptr %x) nounwind {
_______________________________________________
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
