Hi! The ix86_get_mask_mode hook uses int mask for 512-bit vectors or 128/256-bit vectors with AVX512VL (that is correct), and only for V*[SD][IF]mode if not AVX512BW (also correct), but with AVX512BW it would stop checking the elem_size altogether and pretend the hw has masking support for V*TImode etc., which it doesn't. That can lead to various ICEs later on.
Fixed thusly, bootstrapped/regtested on x86_64-linux and i686-linux, ok for trunk? 2020-04-02 Jakub Jelinek <ja...@redhat.com> PR target/94438 * config/i386/i386.c (ix86_get_mask_mode): Only use int mask for elem_size 1, 2, 4 and 8. * gcc.target/i386/avx512bw-pr94438.c: New test. * gcc.target/i386/avx512vlbw-pr94438.c: New test. --- gcc/config/i386/i386.c.jj 2020-03-17 13:50:52.916933781 +0100 +++ gcc/config/i386/i386.c 2020-04-02 17:14:00.202672882 +0200 @@ -21771,7 +21771,9 @@ ix86_get_mask_mode (machine_mode data_mo if ((TARGET_AVX512F && vector_size == 64) || (TARGET_AVX512VL && (vector_size == 32 || vector_size == 16))) { - if (elem_size == 4 || elem_size == 8 || TARGET_AVX512BW) + if (elem_size == 4 + || elem_size == 8 + || (TARGET_AVX512BW && (elem_size == 1 || elem_size == 2))) return smallest_int_mode_for_size (nunits); } --- gcc/testsuite/gcc.target/i386/avx512bw-pr94438.c.jj 2020-04-02 17:18:37.374587069 +0200 +++ gcc/testsuite/gcc.target/i386/avx512bw-pr94438.c 2020-04-02 17:17:15.928787665 +0200 @@ -0,0 +1,13 @@ +/* PR target/94438 */ +/* { dg-do compile { target int128 } } */ +/* { dg-options "-mavx512bw" } */ + +typedef __attribute__ ((__vector_size__ (4 * sizeof (__int128)))) __int128 V; +void bar (V); + +void +foo (V w) +{ + V v = 0 <= (0 >= w); + bar (v); +} --- gcc/testsuite/gcc.target/i386/avx512vlbw-pr94438.c.jj 2020-04-02 17:18:08.272016069 +0200 +++ gcc/testsuite/gcc.target/i386/avx512vlbw-pr94438.c 2020-04-02 17:16:25.302533951 +0200 @@ -0,0 +1,13 @@ +/* PR target/94438 */ +/* { dg-do compile { target int128 } } */ +/* { dg-options "-mavx512bw -mavx512vl" } */ + +typedef __attribute__ ((__vector_size__ (sizeof (__int128)))) __int128 V; +void bar (V); + +void +foo (V w) +{ + V v = 0 <= (0 >= w); + bar (v); +} Jakub