https://github.com/jofrn updated https://github.com/llvm/llvm-project/pull/197619
>From b20b8d4b2b2d067b5243a20e17860a6f1aa23d53 Mon Sep 17 00:00:00 2001 From: jofrn <[email protected]> Date: Wed, 13 May 2026 22:27:20 -0700 Subject: [PATCH] [X86] Remove extra MOV after widening atomic store This change adds patterns to optimize out an extra MOV present after widening the atomic store. Covers <2 x i8> (SSE4.1+), <2 x i16>, <4 x i8>, <2 x i32>, <2 x float>, <4 x i16>, <2 x ptr addrspace(270)>. --- .../include/llvm/Target/TargetSelectionDAG.td | 35 ++++++ llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +- llvm/lib/Target/X86/X86InstrAVX512.td | 12 +- llvm/lib/Target/X86/X86InstrSSE.td | 20 ++-- llvm/test/CodeGen/X86/atomic-load-store.ll | 111 ++++++++---------- llvm/test/CodeGen/X86/atomic-unordered.ll | 54 +++++---- 6 files changed, 129 insertions(+), 105 deletions(-) diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index 35848f76897b3..3b6c280db09ca 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -2393,6 +2393,41 @@ def atomic_store_128 : let MemoryVT = i128; } +// Matches store or atomic_store, no alignment requirement. +def unalignedstore : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)]>; + +def any_store_8 : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)]> { + let MemoryVT = i8; +} + +def any_store_16 : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)]> { + let MemoryVT = i16; +} + +def any_store_32 : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)]> { + let MemoryVT = i32; +} + +def any_store_64 : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)]> { + let MemoryVT = i64; +} + +def any_store_128 : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)]> { + let MemoryVT = i128; +} + //===----------------------------------------------------------------------===// // Selection DAG Pattern Support. // diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index fea1caf0854f5..fe11137b86e53 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -2949,7 +2949,7 @@ bool X86::mayFoldIntoStore(SDValue Op) { return false; User = *User->user_begin(); } - return ISD::isNormalStore(User); + return ISD::isNormalStore(User) || User->getOpcode() == ISD::ATOMIC_STORE; } bool X86::mayFoldIntoZeroExtend(SDValue Op) { diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index 95c75165ed4eb..599981fb1f517 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -3869,8 +3869,8 @@ def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$s def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128X:$src), "vmovd\t{$src, $dst|$dst, $src}", - [(store (i32 (extractelt (v4i32 VR128X:$src), - (iPTR 0))), addr:$dst)]>, + [(any_store_32 (i32 (extractelt (v4i32 VR128X:$src), + (iPTR 0))), addr:$dst)]>, EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecStore]>; } // ExeDomain = SSEPackedInt @@ -3893,8 +3893,8 @@ def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src), def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src), "vmovq\t{$src, $dst|$dst, $src}", - [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)), - addr:$dst)]>, + [(any_store_64 (extractelt (v2i64 VR128X:$src), (iPTR 0)), + addr:$dst)]>, EVEX, TB, PD, REX_W, EVEX_CD8<64, CD8VT1>, Sched<[WriteVecStore]>, Requires<[HasAVX512]>; @@ -11476,8 +11476,8 @@ multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode, def mri : AVX512Ii8<opc, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2), OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", - [(store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))), - addr:$dst)]>, + [(unalignedstore (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), timm:$src2))), + addr:$dst)]>, EVEX, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecExtractSt]>; } diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td index 203344cd81aab..e93b3c91b7861 100644 --- a/llvm/lib/Target/X86/X86InstrSSE.td +++ b/llvm/lib/Target/X86/X86InstrSSE.td @@ -4206,8 +4206,8 @@ def VMOVPDI2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src), "movd\t{$src, $dst|$dst, $src}", - [(store (i32 (extractelt (v4i32 VR128:$src), - (iPTR 0))), addr:$dst)]>, + [(any_store_32 (i32 (extractelt (v4i32 VR128:$src), + (iPTR 0))), addr:$dst)]>, VEX, Sched<[WriteVecStore]>; def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), "movd\t{$src, $dst|$dst, $src}", @@ -4216,8 +4216,8 @@ def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), Sched<[WriteVecMoveToGpr]>; def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src), "movd\t{$src, $dst|$dst, $src}", - [(store (i32 (extractelt (v4i32 VR128:$src), - (iPTR 0))), addr:$dst)]>, + [(any_store_32 (i32 (extractelt (v4i32 VR128:$src), + (iPTR 0))), addr:$dst)]>, Sched<[WriteVecStore]>; } // ExeDomain = SSEPackedInt @@ -4346,13 +4346,13 @@ def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), let ExeDomain = SSEPackedInt, SchedRW = [WriteVecStore] in { def VMOVPQI2QImr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), "movq\t{$src, $dst|$dst, $src}", - [(store (i64 (extractelt (v2i64 VR128:$src), - (iPTR 0))), addr:$dst)]>, + [(any_store_64 (i64 (extractelt (v2i64 VR128:$src), + (iPTR 0))), addr:$dst)]>, VEX, WIG; def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), "movq\t{$src, $dst|$dst, $src}", - [(store (i64 (extractelt (v2i64 VR128:$src), - (iPTR 0))), addr:$dst)]>; + [(any_store_64 (i64 (extractelt (v2i64 VR128:$src), + (iPTR 0))), addr:$dst)]>; } // ExeDomain, SchedRW // For disassembler only @@ -5280,8 +5280,8 @@ multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> { (ins i16mem:$dst, VR128:$src1, u8imm:$src2), !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), - [(store (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))), - addr:$dst)]>, Sched<[WriteVecExtractSt]>; + [(any_store_16 (i16 (trunc (X86pextrw (v8i16 VR128:$src1), timm:$src2))), + addr:$dst)]>, Sched<[WriteVecExtractSt]>; } let Predicates = [HasAVX, NoBWI] in diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll index 659cdec91d3e7..91c4d0a3d8c1c 100644 --- a/llvm/test/CodeGen/X86/atomic-load-store.ll +++ b/llvm/test/CodeGen/X86/atomic-load-store.ll @@ -353,30 +353,37 @@ define void @store_atomic_vec1_double_align(ptr %x, <1 x double> %v) nounwind { } define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) { -; CHECK-SSE-O3-LABEL: store_atomic_vec2_i8: -; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movd %xmm0, %eax -; CHECK-SSE-O3-NEXT: movw %ax, (%rdi) -; CHECK-SSE-O3-NEXT: retq +; CHECK-SSE2-O3-LABEL: store_atomic_vec2_i8: +; CHECK-SSE2-O3: # %bb.0: +; CHECK-SSE2-O3-NEXT: movd %xmm0, %eax +; CHECK-SSE2-O3-NEXT: movw %ax, (%rdi) +; CHECK-SSE2-O3-NEXT: retq +; +; CHECK-SSE4-O3-LABEL: store_atomic_vec2_i8: +; CHECK-SSE4-O3: # %bb.0: +; CHECK-SSE4-O3-NEXT: pextrw $0, %xmm0, (%rdi) +; CHECK-SSE4-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i8: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O3-NEXT: movw %ax, (%rdi) +; CHECK-AVX-O3-NEXT: vpextrw $0, %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; -; CHECK-SSE-O0-LABEL: store_atomic_vec2_i8: -; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movd %xmm0, %eax -; CHECK-SSE-O0-NEXT: # kill: def $ax killed $ax killed $eax -; CHECK-SSE-O0-NEXT: movw %ax, (%rdi) -; CHECK-SSE-O0-NEXT: retq +; CHECK-SSE2-O0-LABEL: store_atomic_vec2_i8: +; CHECK-SSE2-O0: # %bb.0: +; CHECK-SSE2-O0-NEXT: movd %xmm0, %eax +; CHECK-SSE2-O0-NEXT: # kill: def $ax killed $ax killed $eax +; CHECK-SSE2-O0-NEXT: movw %ax, (%rdi) +; CHECK-SSE2-O0-NEXT: retq +; +; CHECK-SSE4-O0-LABEL: store_atomic_vec2_i8: +; CHECK-SSE4-O0: # %bb.0: +; CHECK-SSE4-O0-NEXT: pextrw $0, %xmm0, (%rdi) +; CHECK-SSE4-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i8: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O0-NEXT: # kill: def $ax killed $ax killed $eax -; CHECK-AVX-O0-NEXT: movw %ax, (%rdi) +; CHECK-AVX-O0-NEXT: vpextrw $0, %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <2 x i8> %v, ptr %x release, align 4 ret void @@ -385,26 +392,22 @@ define void @store_atomic_vec2_i8(ptr %x, <2 x i8> %v) { define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) { ; CHECK-SSE-O3-LABEL: store_atomic_vec2_i16: ; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movd %xmm0, %eax -; CHECK-SSE-O3-NEXT: movl %eax, (%rdi) +; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) ; CHECK-SSE-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i16: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O3-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec2_i16: ; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movd %xmm0, %eax -; CHECK-SSE-O0-NEXT: movl %eax, (%rdi) +; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi) ; CHECK-SSE-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i16: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O0-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <2 x i16> %v, ptr %x release, align 4 ret void @@ -413,26 +416,22 @@ define void @store_atomic_vec2_i16(ptr %x, <2 x i16> %v) { define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) { ; CHECK-SSE-O3-LABEL: store_atomic_vec2_ptr270: ; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movq %xmm0, %rax -; CHECK-SSE-O3-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) ; CHECK-SSE-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_ptr270: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O3-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec2_ptr270: ; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movq %xmm0, %rax -; CHECK-SSE-O0-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) ; CHECK-SSE-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec2_ptr270: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O0-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <2 x ptr addrspace(270)> %v, ptr %x release, align 8 ret void @@ -441,26 +440,22 @@ define void @store_atomic_vec2_ptr270(ptr %x, <2 x ptr addrspace(270)> %v) { define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) { ; CHECK-SSE-O3-LABEL: store_atomic_vec2_i32_align: ; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movq %xmm0, %rax -; CHECK-SSE-O3-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) ; CHECK-SSE-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_i32_align: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O3-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec2_i32_align: ; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movq %xmm0, %rax -; CHECK-SSE-O0-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) ; CHECK-SSE-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec2_i32_align: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O0-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <2 x i32> %v, ptr %x release, align 8 ret void @@ -469,26 +464,22 @@ define void @store_atomic_vec2_i32_align(ptr %x, <2 x i32> %v) { define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) { ; CHECK-SSE-O3-LABEL: store_atomic_vec2_float_align: ; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movq %xmm0, %rax -; CHECK-SSE-O3-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) ; CHECK-SSE-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec2_float_align: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O3-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec2_float_align: ; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movq %xmm0, %rax -; CHECK-SSE-O0-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) ; CHECK-SSE-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec2_float_align: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O0-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <2 x float> %v, ptr %x release, align 8 ret void @@ -497,26 +488,22 @@ define void @store_atomic_vec2_float_align(ptr %x, <2 x float> %v) { define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind { ; CHECK-SSE-O3-LABEL: store_atomic_vec4_i8: ; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movd %xmm0, %eax -; CHECK-SSE-O3-NEXT: movl %eax, (%rdi) +; CHECK-SSE-O3-NEXT: movss %xmm0, (%rdi) ; CHECK-SSE-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec4_i8: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O3-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovss %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec4_i8: ; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movd %xmm0, %eax -; CHECK-SSE-O0-NEXT: movl %eax, (%rdi) +; CHECK-SSE-O0-NEXT: movd %xmm0, (%rdi) ; CHECK-SSE-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec4_i8: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovd %xmm0, %eax -; CHECK-AVX-O0-NEXT: movl %eax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovd %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <4 x i8> %v, ptr %x release, align 4 ret void @@ -525,26 +512,22 @@ define void @store_atomic_vec4_i8(ptr %x, <4 x i8> %v) nounwind { define void @store_atomic_vec4_i16(ptr %x, <4 x i16> %v) nounwind { ; CHECK-SSE-O3-LABEL: store_atomic_vec4_i16: ; CHECK-SSE-O3: # %bb.0: -; CHECK-SSE-O3-NEXT: movq %xmm0, %rax -; CHECK-SSE-O3-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O3-NEXT: movlps %xmm0, (%rdi) ; CHECK-SSE-O3-NEXT: retq ; ; CHECK-AVX-O3-LABEL: store_atomic_vec4_i16: ; CHECK-AVX-O3: # %bb.0: -; CHECK-AVX-O3-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O3-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O3-NEXT: vmovlps %xmm0, (%rdi) ; CHECK-AVX-O3-NEXT: retq ; ; CHECK-SSE-O0-LABEL: store_atomic_vec4_i16: ; CHECK-SSE-O0: # %bb.0: -; CHECK-SSE-O0-NEXT: movq %xmm0, %rax -; CHECK-SSE-O0-NEXT: movq %rax, (%rdi) +; CHECK-SSE-O0-NEXT: movq %xmm0, (%rdi) ; CHECK-SSE-O0-NEXT: retq ; ; CHECK-AVX-O0-LABEL: store_atomic_vec4_i16: ; CHECK-AVX-O0: # %bb.0: -; CHECK-AVX-O0-NEXT: vmovq %xmm0, %rax -; CHECK-AVX-O0-NEXT: movq %rax, (%rdi) +; CHECK-AVX-O0-NEXT: vmovq %xmm0, (%rdi) ; CHECK-AVX-O0-NEXT: retq store atomic <4 x i16> %v, ptr %x release, align 8 ret void diff --git a/llvm/test/CodeGen/X86/atomic-unordered.ll b/llvm/test/CodeGen/X86/atomic-unordered.ll index 7946204865ab6..edfdec37150c2 100644 --- a/llvm/test/CodeGen/X86/atomic-unordered.ll +++ b/llvm/test/CodeGen/X86/atomic-unordered.ll @@ -348,18 +348,16 @@ define void @store_i256(ptr %ptr, i256 %v) { define void @vec_store(ptr %p0, <2 x i32> %vec) { ; CHECK-O0-LABEL: vec_store: ; CHECK-O0: # %bb.0: -; CHECK-O0-NEXT: vmovd %xmm0, %ecx ; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax -; CHECK-O0-NEXT: movl %ecx, (%rdi) +; CHECK-O0-NEXT: vmovd %xmm0, (%rdi) ; CHECK-O0-NEXT: movl %eax, 4(%rdi) ; CHECK-O0-NEXT: retq ; ; CHECK-O3-LABEL: vec_store: ; CHECK-O3: # %bb.0: -; CHECK-O3-NEXT: vmovd %xmm0, %eax -; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx -; CHECK-O3-NEXT: movl %eax, (%rdi) -; CHECK-O3-NEXT: movl %ecx, 4(%rdi) +; CHECK-O3-NEXT: vextractps $1, %xmm0, %eax +; CHECK-O3-NEXT: vmovss %xmm0, (%rdi) +; CHECK-O3-NEXT: movl %eax, 4(%rdi) ; CHECK-O3-NEXT: retq %v1 = extractelement <2 x i32> %vec, i32 0 %v2 = extractelement <2 x i32> %vec, i32 1 @@ -373,18 +371,16 @@ define void @vec_store(ptr %p0, <2 x i32> %vec) { define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) { ; CHECK-O0-LABEL: vec_store_unaligned: ; CHECK-O0: # %bb.0: -; CHECK-O0-NEXT: vmovd %xmm0, %ecx ; CHECK-O0-NEXT: vpextrd $1, %xmm0, %eax -; CHECK-O0-NEXT: movl %ecx, (%rdi) +; CHECK-O0-NEXT: vmovd %xmm0, (%rdi) ; CHECK-O0-NEXT: movl %eax, 4(%rdi) ; CHECK-O0-NEXT: retq ; ; CHECK-O3-LABEL: vec_store_unaligned: ; CHECK-O3: # %bb.0: -; CHECK-O3-NEXT: vmovd %xmm0, %eax -; CHECK-O3-NEXT: vpextrd $1, %xmm0, %ecx -; CHECK-O3-NEXT: movl %eax, (%rdi) -; CHECK-O3-NEXT: movl %ecx, 4(%rdi) +; CHECK-O3-NEXT: vextractps $1, %xmm0, %eax +; CHECK-O3-NEXT: vmovss %xmm0, (%rdi) +; CHECK-O3-NEXT: movl %eax, 4(%rdi) ; CHECK-O3-NEXT: retq %v1 = extractelement <2 x i32> %vec, i32 0 %v2 = extractelement <2 x i32> %vec, i32 1 @@ -399,12 +395,17 @@ define void @vec_store_unaligned(ptr %p0, <2 x i32> %vec) { ; Legal if wider type is also atomic (TODO) ; Also, can avoid register move from xmm to eax (TODO) define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) { -; CHECK-LABEL: widen_broadcast2: -; CHECK: # %bb.0: -; CHECK-NEXT: vmovd %xmm0, %eax -; CHECK-NEXT: movl %eax, (%rdi) -; CHECK-NEXT: movl %eax, 4(%rdi) -; CHECK-NEXT: retq +; CHECK-O0-LABEL: widen_broadcast2: +; CHECK-O0: # %bb.0: +; CHECK-O0-NEXT: vmovd %xmm0, (%rdi) +; CHECK-O0-NEXT: vmovd %xmm0, 4(%rdi) +; CHECK-O0-NEXT: retq +; +; CHECK-O3-LABEL: widen_broadcast2: +; CHECK-O3: # %bb.0: +; CHECK-O3-NEXT: vmovss %xmm0, (%rdi) +; CHECK-O3-NEXT: vmovss %xmm0, 4(%rdi) +; CHECK-O3-NEXT: retq %v1 = extractelement <2 x i32> %vec, i32 0 %p1 = getelementptr i32, ptr %p0, i64 1 store atomic i32 %v1, ptr %p0 unordered, align 8 @@ -414,12 +415,17 @@ define void @widen_broadcast2(ptr %p0, <2 x i32> %vec) { ; Not legal to widen due to alignment restriction define void @widen_broadcast2_unaligned(ptr %p0, <2 x i32> %vec) { -; CHECK-LABEL: widen_broadcast2_unaligned: -; CHECK: # %bb.0: -; CHECK-NEXT: vmovd %xmm0, %eax -; CHECK-NEXT: movl %eax, (%rdi) -; CHECK-NEXT: movl %eax, 4(%rdi) -; CHECK-NEXT: retq +; CHECK-O0-LABEL: widen_broadcast2_unaligned: +; CHECK-O0: # %bb.0: +; CHECK-O0-NEXT: vmovd %xmm0, (%rdi) +; CHECK-O0-NEXT: vmovd %xmm0, 4(%rdi) +; CHECK-O0-NEXT: retq +; +; CHECK-O3-LABEL: widen_broadcast2_unaligned: +; CHECK-O3: # %bb.0: +; CHECK-O3-NEXT: vmovss %xmm0, (%rdi) +; CHECK-O3-NEXT: vmovss %xmm0, 4(%rdi) +; CHECK-O3-NEXT: retq %v1 = extractelement <2 x i32> %vec, i32 0 %p1 = getelementptr i32, ptr %p0, i64 1 store atomic i32 %v1, ptr %p0 unordered, align 4 _______________________________________________ llvm-branch-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
