[Bug tree-optimization/127036] New: wrong code at -O{2,3} with -fno-vect-cost-model on x86_64-linux-gnu (generated code hangs)

[zhendong.su at inf dot ethz.ch via Gcc-bugs]

https://gcc.gnu.org/bugzilla/show_bug.cgi?id=127036

            Bug ID: 127036
           Summary: wrong code at -O{2,3} with -fno-vect-cost-model on
                    x86_64-linux-gnu (generated code hangs)
           Product: gcc
           Version: 16.0
            Status: UNCONFIRMED
          Severity: normal
          Priority: P3
         Component: tree-optimization
          Assignee: unassigned at gcc dot gnu.org
          Reporter: zhendong.su at inf dot ethz.ch
  Target Milestone: ---

Compiler Explorer: https://godbolt.org/z/9eK3oWMrd

Note:
- fails: 12.1 to trunk
- works: 11.5 and earlier

[707] % gcctk -v
Using built-in specs.
COLLECT_GCC=gcctk
COLLECT_LTO_WRAPPER=/local/home/suz/suz-local/software/local/gcc-trunk/bin/../libexec/gcc/x86_64-pc-linux-gnu/17.0.0/lto-wrapper
Target: x86_64-pc-linux-gnu
Configured with: ../gcc-trunk/configure --disable-bootstrap
--enable-checking=yes --prefix=/local/suz-local/software/local/gcc-trunk
--enable-sanitizers --enable-languages=c,c++,lto --disable-werror
--disable-multilib
Thread model: posix
Supported LTO compression algorithms: zlib
gcc version 17.0.0 20260823 (experimental) [master r17-1628-gd1c12591336] (GCC) 
[708] % 
[708] % gcctk -O2 small.c; ./a.out
[709] % 
[709] % gcctk -O2 -fno-vect-cost-model small.c
[710] % timeout -s 9 5 ./a.out
Killed
[711] % cat small.c
int a[256];
short e, k = -202, m = 108, n = -8451, s, t = -6, u, aa;
int f = -29, o, ab, ac = -248, ad, ae = -2147483648, af, ag, ah, aj, ak, al;
long p, q = -643472885350108251, v, am, an, ao, ap = -341041202314391927, aq,
        ar, as, at, au, av, aw, ax;
char r = -112;
long *w, *ay, *az;
short *x;
short **y;
long **z, **ba;
double bb;
int bc(int bd, int be) {
  for (int g = 0; g < 256; ++g) {
    unsigned c = g;
    for (int h = 0; h < 8; ++h)
      c = c & 1 ? c >> 1 ^ 3988292384 : c >> 1;
    a[g] = c;
  }
  unsigned i = bd;
  int j = be;
  for (int b = 0; b < 4; ++b) {
    char d = j;
    j >>= 8;
    i = i >> 8 ^ a[(i ^ d) & 255];
  }
  return i;
}
short bf(short bd, short be) {
  short l = bd + be;
  return l;
}
void bg(long bd) {
  if (bd)
  ai:
    goto ai;
}
int bh(long bd, char be, char bi, int bj, long bk, char bl, short bm) {
  z = &w;
  v = 242 - 196791;
  y = &x;
  *y = &n;
  *z = &v;
  o = -2063961555 + 242 - -670692255;
  p = ((long)((unsigned long)-175 << 16) >> 16) - -16 * v;
  *w = v - -8388800 * p;
  s = bm + bf(15627 - 25591 - 16, bm - -6) + *x;
  *x = bm - 6911;
  e = bm - (-64 ^ (bm - 32704));
  u = 32767 - 31215 - -e;
  x = &m;
  ab = bc(0, 108);
  ab = bc(ab, (long)((unsigned long)k << 48) >> 48);
  ab = bc(ab, 100);
  ab = bc(ab, (long)((unsigned long)n << 48) >> 48);
  ab = bc(ab, 247);
  ab = bc(ab, f);
  ab = bc(ab, o);
  ab = bc(ab, 0);
  ab = bc(ab, 188);
  ab = bc(ab, q);
  ab = bc(ab, 0);
  ab = bc(ab, 0);
  ab = bc(ab, 28);
  ab = bc(ab, (long)((unsigned long)r << 56) >> 56);
  ab = bc(ab, (long)((unsigned long)s << 48) >> 48);
  ab = bc(ab, (long)((unsigned long)t << 48) >> 48);
  ab = bc(ab, (long)((unsigned long)u << 48) >> 48);
  ab = bc(ab, v);
  ab = bc(ab, bd);
  ab = bc(ab, (long)((unsigned long)be << 56) >> 56);
  ab = bc(ab, bi);
  ab = bc(ab, bj);
  ab = bc(ab, bk);
  ab = bc(ab, bl);
  ab = bc(ab, bm);
  am = *w;
  ab = bc(ab, am);
  aa = *x;
  ab = bc(ab, aa);
  return ab;
}
int bn(int bd, short be) {
bo:
  bb = ae;
  ae = ae + 2147483647;
  if (be >= (30 << bd) - ae)
    goto bp;
  goto bo;
bp:
  ag = bc(ag, bb);
  ag = bc(ag, 0);
  ag = bc(ag, 0);
  ag = bc(ag, ao);
  ag = bc(ag, be);
  ag = bc(ag, af);
  ag = bc(ag, af);
  return ag;
}
int bq(long bd, long be, short bi, char bj) {
  az = &aq;
  ba = &ay;
  *ba = az;
  aq = 1789569705 - bd;
br:
  ah = (491516 - (unsigned long)ap) << 44 >> 44;
  ap = -1074593792;
  if (2147471022 + (long)ah != be - 2147483618)
    goto ai;
  goto br;
ai:
  ap = 1789569712 + 2 * aq;
  *az = -2130706432 - ap;
  ar = -1073741826 - ap;
  al = bh(ar + 6442450945, -61, 1, -101449852, 9223372032802865152, bj - 2, 1)
-
       525767262;
  av = (long)((unsigned long)aq << 16) >> 16;
  ax = 4186636 * av % 11863279;
  aw = ax + 9354052;
  bg(aw);
  aj = al - aq;
  az = &ap;
  *ay = bd + 2145386491;
  ar = -1076871164 + aq - aj;
  *ay = (long)-1072168945 - 2130182160 - ar;
  ap = -2129904 - 1048577 + ar;
  *az = (-1084981252 - *az + 1051754483) & ap;
  ar = 4849666 - be;
  as = *az;
  *ba = &as;
  aj = 1146881 - ah;
  ak = bc(0, ap);
  ak = bc(ak, aq);
  ak = bc(ak, ah);
  ak = bc(ak, aj);
  ak = bc(ak, ar);
  ak = bc(ak, bd);
  ak = bc(ak, 0);
  ak = bc(ak, be);
  ak = bc(ak, bi);
  ak = bc(ak, bj);
  at = *ay;
  ak = bc(ak, at);
  au = *az;
  ak = bc(ak, au);
  return ak;
}
int main() {
  ad = bq(1, 4294967299, 32637, 126) - 466268612;
  ac = bn(ac + ad, 26864);
  if (0 <= ac)
    bq(an, 9, 7, 6);
}