Issue 242207
Summary [X86][VectorCombine] AVX-512 even-lane shuffle codegen regression after #229662 (foldShuffleOfAdjacentLoads)
Labels new issue
Assignees
Reporter eaeltsin
    ### Description

Commit cc7c71f048c083fbaec7581d94124559f69317f4 (reland PR #229662: `Reland "[VectorCombine] Combine adjacent loads feeding as shuffle operands"`) introduced a codegen regression on AVX-512 targets when two adjacent 256-bit vector loads feed an even-lane shuffle (`<0, 2, 4, ... 30>`) that has multiple subvector consumers.

`VectorCombine::foldShuffleOfAdjacentLoads` combines two `<16 x i16>` loads into a single `<32 x i16>` wide load and converts the two-source shuffle into a single-source `<32 x i16> -> <16 x i16>` shuffle. However, downstream X86 ISel / DAGCombiner fails to lower this 512-bit to 256-bit shuffle as a single truncation (`vpmovdw %zmm -> %ymm`) shared across consumers. Instead, subvector extracts are pushed into the shuffle, decomposing it independently for each use and producing 12 vector instructions (including `vpermw` and multiple `vpinsrw` instructions) instead of the previous 5.

On Skylake-SP, this roughly doubles port 5 shuffle pressure and caused an observed +35% latency regression on integer sequence encoding benchmarks.

### Minimal C++ Reproducer

```cpp
#include <immintrin.h>

__attribute__((target("avx512bw,avx512vl,popcnt"))) int EncodeBlock(
    const short* data, char* out, const char* shuffle_table) {
  const __m256i d0 = _mm256_loadu_si256((const __m256i*)data);
  const __m256i d1 = _mm256_loadu_si256((const __m256i*)(data + 16));
  const __m256i even_lanes = _mm256_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14, 16,
 18, 20, 22, 24, 26, 28, 30);
  const __m256i v = _mm256_permutex2var_epi16(d0, even_lanes, d1);
  const unsigned mask = _mm256_test_epi16_mask(v, v);
  const unsigned lo_mask = mask & 0xff;
  const unsigned hi_mask = mask >> 8;
  const __m128i lo = _mm_shuffle_epi8(
      _mm256_castsi256_si128(v),
 _mm_loadu_si128((const __m128i*)(shuffle_table + lo_mask * 16)));
  const __m128i hi = _mm_shuffle_epi8(
      _mm256_extracti128_si256(v, 1),
 _mm_loadu_si128((const __m128i*)(shuffle_table + hi_mask * 16)));
 *(unsigned short*)out = (unsigned short)mask;
 _mm_storeu_si128((__m128i*)(out + 2), lo);
  _mm_storeu_si128((__m128i*)(out + 2 + __builtin_popcount(lo_mask) * 2), hi);
  return 2 + __builtin_popcount(mask) * 2;
}
```

### Old code
```
_Z11EncodeBlockPKsPcPKc:
	vmovdqu64	(%rdi), %zmm0
	vpmovdw	%zmm0, %ymm0
	vptestmw	%ymm0, %ymm0, %k0
	kmovw	%k0, %eax
	movzbl	%al, %ecx
	popcntl	%ecx, %edi
	shll	$4, %ecx
	vpshufb	(%rdx,%rcx), %xmm0, %xmm1
	popcntl	%eax, %ecx
	shrl	$4, %eax
	andl	$-16, %eax
	vextracti128	$1, %ymm0, %xmm0
	vpshufb	(%rdx,%rax), %xmm0, %xmm0
	kmovw	%k0, (%rsi)
	vmovdqu	%xmm1, 2(%rsi)
	vmovdqu	%xmm0, 2(%rsi,%rdi,2)
	leal	2(,%rcx,2), %eax
	vzeroupper
	retq
```

### New code

```
.LCPI0_0:
	.short	0
	.short	2
	.short	4
	.short	6
	.short	8
	.short	5
	.short	6
	.short	7
_Z11EncodeBlockPKsPcPKc:
	vmovdqa	.LCPI0_0(%rip), %xmm0
	vpermw	(%rdi), %ymm0, %ymm0
	vmovdqu	16(%rdi), %xmm1
	vpextrw	$6, %xmm1, %ecx
	vpextrw	$4, %xmm1, %r8d
	vpextrw	$2, %xmm1, %r9d
	vmovdqu64	(%rdi), %zmm1
	vpmovdw	%zmm1, %ymm1
	vptestmw	%ymm1, %ymm1, %k0
	kmovw	%k0, %eax
	movzbl	%al, %r10d
	popcntl	%r10d, %r11d
	shll	$4, %r10d
	vpinsrw	$5, %r9d, %xmm0, %xmm0
	vpinsrw	$6, %r8d, %xmm0, %xmm0
	vpinsrw	$7, %ecx, %xmm0, %xmm0
	vpshufb	(%rdx,%r10), %xmm0, %xmm0
	popcntl	%eax, %ecx
	shrl	$4, %eax
	andl	$-16, %eax
	vmovdqu	32(%rdi), %ymm1
	vpmovdw	%ymm1, %xmm1
	vpshufb	(%rdx,%rax), %xmm1, %xmm1
	kmovw	%k0, (%rsi)
	vmovdqu	%xmm0, 2(%rsi)
	vmovdqu	%xmm1, 2(%rsi,%r11,2)
	leal	2(,%rcx,2), %eax
	vzeroupper
	retq
```

_______________________________________________
llvm-bugs mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-bugs

Reply via email to