Issue 243602
Summary [X86][AVX512BW] Wrong code: shuffle of zext <2 x i8> and zext <2 x i16> lowered to vpunpckldq + vpmovzxbd
Labels new issue
Assignees
Reporter r-a-y-z-e-r
    ## Summary

With AVX512BW enabled, a `shufflevector` that combines `zext <2 x i8> to <2 x i32>` with
`zext <2 x i16> to <2 x i32>` into an `<8 x i32>` is selected as `vpunpckldq` of the two narrow
sources followed by a single `vpmovzxbd`. That widens the 16-bit elements byte by byte, so the
second word's lane receives the high byte of the first word. `-mcpu=x86-64-v3` is correct;
`+avx512f`, `+avx512vl` alone are correct; adding `+avx512bw` makes it wrong.

Found with clang 21.1.8 (Ubuntu 21.1.8-6ubuntu1), x86_64-unknown-linux-gnu.

## Reduced reproducer (llc)

```llvm
declare <3 x i8> @llvm.masked.load.v3i8.p0(ptr, i32 immarg, <3 x i1>, <3 x i8>)

define <8 x i32> @f(ptr %a, ptr %b) {
  %bytes = call <3 x i8> @llvm.masked.load.v3i8.p0(
      ptr %a, i32 1, <3 x i1> <i1 true, i1 false, i1 true>, <3 x i8> poison)
  %pair = shufflevector <3 x i8> %bytes, <3 x i8> poison,
      <2 x i32> <i32 0, i32 2>
  %words = load <2 x i16>, ptr %b, align 1
  %x = zext <2 x i8> %pair to <2 x i32>
  %y = zext <2 x i16> %words to <2 x i32>
  %result = shufflevector <2 x i32> %x, <2 x i32> %y,
      <8 x i32> <i32 0, i32 poison, i32 1, i32 poison, i32 2, i32 3, i32 poison, i32 poison>
  ret <8 x i32> %result
}
```

`llc -O2 -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v4`:

```asm
movw $5, %ax
kmovd       %eax, %k1
vmovdqu8    (%rdi), %xmm0 {%k1} {z}
vmovd (%rsi), %xmm1
vpunpckldq  %xmm1, %xmm0, %xmm0
vpmovzxbd   %xmm0, %ymm0 # widens the two i16 elements as four bytes
retq
```

`-mcpu=x86-64-v3` (correct):

```asm
movzbl (%rdi), %eax
vmovd       %eax, %xmm0
vpinsrb     $1, 2(%rdi), %xmm0, %xmm0
vmovd       (%rsi), %xmm1
vpmovzxwd   %xmm1, %xmm1
vpmovzxbq %xmm0, %xmm0
vinserti128 $1, %xmm1, %ymm0, %ymm0
retq
```

For `a = {11, 99, 22}`, `b = {0, 2}` the defined lanes are:

| Lane | Expected | v3 | v4 |
|---|---:|---:|---:|
| 0 | 11 | 11 | 11 |
| 2 | 22 | 22 | 22 |
| 4 | 0 | 0 | 0 |
| 5 | 2 | 2 | **0** |

Sweeping `b = {0, n}` over all 65 536 values of `n`: v3 is correct for all, v4 is wrong for 65 535.

Feature bisection from x86-64-v3: `+avx512f` correct; `+avx512f,+avx512vl` correct;
`+avx512f,+avx512bw,+avx512vl` wrong; `+avx512f,+avx512bw,+evex512` (no VL) wrong.

## Original C reproducer

The pattern came from SLP-vectorizing eight adjacent 4-input LUT evaluations,
`v[o] = (C >> (v[i0] | v[i1] << 1 | v[i2] << 2 | v[i3] << 3)) & 1;` on `uint8_t *restrict v`,
with 32-bit constants. A self-contained exhaustive test over all 2^17 Boolean inputs:

```c
/* Eight adjacent 4-input LUT lines, f (32-bit constants) and fixed (64-bit
 * constants), checked against an optnone reference over all 2^17 inputs.
 * Build and run: clang -O3 -march=x86-64-v4 lut_shift.c -o lut_shift && ./lut_shift
 */
#include <stdio.h>
#include <stdint.h>
__attribute__((noinline)) void f(uint8_t *restrict v) {
 v[28417] = (65257u >> (v[31446] | v[26763] << 1 | v[26873] << 2 | v[24605] << 3)) & 1;
  v[28418] = (233u >> (v[26763] | v[26873] << 1 | v[24605] << 2)) & 1;
  v[28419] = (5632u >> (v[31852] | v[26658] << 1 | v[26875] << 2 | v[26876] << 3)) & 1;
  v[28420] = (96u >> (v[26658] | v[26875] << 1 | v[26876] << 2)) & 1;
  v[28421] = (45883u >> (v[31732] | v[23494] << 1 | v[26877] << 2 | v[26659] << 3)) & 1;
  v[28422] = (52273u >> (v[23494] | v[26877] << 1 | v[26878] << 2 | v[26659] << 3)) & 1;
  v[28423] = (5632u >> (v[31466] | v[26660] << 1 | v[26662] << 2 | v[26879] << 3)) & 1;
  v[28424] = (96u >> (v[26660] | v[26662] << 1 | v[26879] << 2)) & 1;
}
#include <stdio.h>
#include <stdint.h>
__attribute__((noinline)) void fixed(uint8_t *restrict v) {
  v[28417] = (65257ull >> (v[31446] | v[26763] << 1 | v[26873] << 2 | v[24605] << 3)) & 1;
  v[28418] = (233ull >> (v[26763] | v[26873] << 1 | v[24605] << 2)) & 1;
  v[28419] = (5632ull >> (v[31852] | v[26658] << 1 | v[26875] << 2 | v[26876] << 3)) & 1;
  v[28420] = (96ull >> (v[26658] | v[26875] << 1 | v[26876] << 2)) & 1;
  v[28421] = (45883ull >> (v[31732] | v[23494] << 1 | v[26877] << 2 | v[26659] << 3)) & 1;
  v[28422] = (52273ull >> (v[23494] | v[26877] << 1 | v[26878] << 2 | v[26659] << 3)) & 1;
  v[28423] = (5632ull >> (v[31466] | v[26660] << 1 | v[26662] << 2 | v[26879] << 3)) & 1;
  v[28424] = (96ull >> (v[26660] | v[26662] << 1 | v[26879] << 2)) & 1;
}
#include <stdio.h>
#include <stdint.h>
__attribute__((noinline,optnone)) void reference(uint8_t *restrict v) {
  v[28417] = (65257u >> (v[31446] | v[26763] << 1 | v[26873] << 2 | v[24605] << 3)) & 1;
  v[28418] = (233u >> (v[26763] | v[26873] << 1 | v[24605] << 2)) & 1;
  v[28419] = (5632u >> (v[31852] | v[26658] << 1 | v[26875] << 2 | v[26876] << 3)) & 1;
  v[28420] = (96u >> (v[26658] | v[26875] << 1 | v[26876] << 2)) & 1;
  v[28421] = (45883u >> (v[31732] | v[23494] << 1 | v[26877] << 2 | v[26659] << 3)) & 1;
  v[28422] = (52273u >> (v[23494] | v[26877] << 1 | v[26878] << 2 | v[26659] << 3)) & 1;
  v[28423] = (5632u >> (v[31466] | v[26660] << 1 | v[26662] << 2 | v[26879] << 3)) & 1;
  v[28424] = (96u >> (v[26660] | v[26662] << 1 | v[26879] << 2)) & 1;
}
int main(void) {uint8_t a[40000]={0},b[40000]={0},c[40000]={0};unsigned bad=0,fixed_bad=0;
for(unsigned bits=0;bits<(1u<<17);bits++) {
a[23494]=b[23494]=c[23494]=(bits>>0)&1;
a[24605]=b[24605]=c[24605]=(bits>>1)&1;
a[26658]=b[26658]=c[26658]=(bits>>2)&1;
a[26659]=b[26659]=c[26659]=(bits>>3)&1;
a[26660]=b[26660]=c[26660]=(bits>>4)&1;
a[26662]=b[26662]=c[26662]=(bits>>5)&1;
a[26763]=b[26763]=c[26763]=(bits>>6)&1;
a[26873]=b[26873]=c[26873]=(bits>>7)&1;
a[26875]=b[26875]=c[26875]=(bits>>8)&1;
a[26876]=b[26876]=c[26876]=(bits>>9)&1;
a[26877]=b[26877]=c[26877]=(bits>>10)&1;
a[26878]=b[26878]=c[26878]=(bits>>11)&1;
a[26879]=b[26879]=c[26879]=(bits>>12)&1;
a[31446]=b[31446]=c[31446]=(bits>>13)&1;
a[31466]=b[31466]=c[31466]=(bits>>14)&1;
a[31732]=b[31732]=c[31732]=(bits>>15)&1;
a[31852]=b[31852]=c[31852]=(bits>>16)&1;
f(a);fixed(b);reference(c);
bad+=(a[28417]!=c[28417] || a[28418]!=c[28418] || a[28419]!=c[28419] || a[28420]!=c[28420] || a[28421]!=c[28421] || a[28422]!=c[28422] || a[28423]!=c[28423] || a[28424]!=c[28424]);
fixed_bad+=(b[28417]!=c[28417] || b[28418]!=c[28418] || b[28419]!=c[28419] || b[28420]!=c[28420] || b[28421]!=c[28421] || b[28422]!=c[28422] || b[28423]!=c[28423] || b[28424]!=c[28424]);
}
printf("combinations=%u original_mismatches=%u fixed_mismatches=%u\n",1u<<17,bad,fixed_bad);return fixed_bad!=0;}
```

`clang -O3 -march=x86-64-v4 lut_shift.c && ./a.out` prints
`combinations=131072 original_mismatches=57344 fixed_mismatches=0` (expected 0 mismatches).
Correct with `-march=x86-64-v3`, with `-fno-slp-vectorize`, with gcc 15.2 at `-march=x86-64-v4`.
Still wrong with `-mprefer-vector-width=256`. The SLP-vectorized IR is correct when lowered for
x86-64-v3 and wrong for x86-64-v4; the wrong `VPUNPCKLDQZ128rr`/`VPMOVZXBDZ256rr` pair is already
present after x86-isel.

We ruled out a CPU erratum (found on an AMD Zen 4-class machine): an interpreter of the emitted
instructions following the Intel SDM reproduces the hardware's results on all 131 072 inputs, and
both disagree with the C source on the same 57 344.

_______________________________________________
llvm-bugs mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-bugs

Reply via email to