> -----Original Message-----
> From: Richard Biener <[email protected]>
> Sent: Wednesday, August 12, 2026 3:58 PM
> To: [email protected]
> Cc: Liu, Hongtao <[email protected]>
> Subject: [PATCH] tree-optimization/126789 - MMX and SSE mask interaction
> with AVX512
>
> The following makes the vectorizer deal with the fact that the x86 backend can
> have a set of integer mode masks and a set of integer vector mode masks
> depending on active ISAs. To convert between both representations we have
> to use vcond_mask for integer mode to integer vector mode. For the case of
> V2SImode the backend currently misses patterns which the following adds.
> Now, when doing BB vectorization the vectorizer pattern recognition cannot
> know that we'll use V2SImode with a vector integer mask mode but it guesses
> V16SImode which would use DImode. So we cannot arrange for the mask
> representation conversion during pattern recognition, but it's easy to handle
> VECTOR_BOOLEAN_TYPE_P conversions with vcond_mask as option in
> vectorizable_conversion, so that's what the patch does.
>
> I don't have a testcase for the reverse which would need to use vcmp[_eq] to
> convert from integer vector mode to integer mode mask, so I'm not
> implementing that with this patch.
>
> Bootstrapped and tested on x86_64-unknown-linux-gnu. Are the x86 parts
> OK?
Ok for x86 parts.
>
> Thanks,
> Richard.
>
> PR tree-optimization/126789
> * config/i386/mmx.md (mmxxmmmode): Add V2SF and V2SI.
> (mmxxmmmodelower): Likewise.
> (vcond_mask_<mode>qi): New expander for V2SF and V2SI.
> * tree-vect-stmts.cc (vectorizable_conversion): Handle
> conversion via VEC_COND_EXPR.
> (supportable_indirect_convert_operation): For mask vector
> conversions to vector integer mode try using a VEC_COND_EXPR.
>
> * gcc.target/i386/vect-pr126789-2.c: New testcase.
> ---
> gcc/config/i386/mmx.md | 24 +++++++++-
> .../gcc.target/i386/vect-pr126789-2.c | 19 ++++++++
> gcc/tree-vect-stmts.cc | 48 +++++++++++++++++--
> 3 files changed, 84 insertions(+), 7 deletions(-) create mode 100644
> gcc/testsuite/gcc.target/i386/vect-pr126789-2.c
>
> diff --git a/gcc/config/i386/mmx.md b/gcc/config/i386/mmx.md index
> f61335af0b9..640b2c0e63f 100644
> --- a/gcc/config/i386/mmx.md
> +++ b/gcc/config/i386/mmx.md
> @@ -2085,11 +2085,13 @@
>
> (define_mode_attr mmxxmmmode
> [(V2HF "V8HF") (V2HI "V8HI") (V2BF "V8BF")
> - (V4HF "V8HF") (V4HI "V8HI") (V4BF "V8BF")])
> + (V4HF "V8HF") (V4HI "V8HI") (V4BF "V8BF")
> + (V2SF "V4SF") (V2SI "V4SI")])
>
> (define_mode_attr mmxxmmmodelower
> [(V2HF "v8hf") (V2HI "v8hi") (V2BF "v8bf")
> - (V4HF "v8hf") (V4HI "v8hi") (V4BF "v8bf")])
> + (V4HF "v8hf") (V4HI "v8hi") (V4BF "v8bf")
> + (V2SF "v4sf") (V2SI "v4si")])
>
> (define_expand "movd_<mode>_to_sse"
> [(set (match_operand:<mmxxmmmode> 0 "register_operand") @@ -2341,6
> +2343,24 @@
> DONE;
> })
>
> +(define_expand "vcond_mask_<mode>qi"
> + [(set (match_operand:V2FI 0 "register_operand")
> + (vec_merge:V2FI
> + (match_operand:V2FI 1 "register_operand")
> + (match_operand:V2FI 2 "register_operand")
> + (match_operand:QI 3 "register_operand")))]
> + "TARGET_MMX_WITH_SSE && TARGET_AVX512VL"
> +{
> + rtx op0 = gen_reg_rtx (<mmxxmmmode>mode);
> + operands[1] = lowpart_subreg (<mmxxmmmode>mode, operands[1],
> +<MODE>mode);
> + operands[2] = lowpart_subreg (<mmxxmmmode>mode, operands[2],
> +<MODE>mode);
> + emit_insn (gen_vcond_mask_<mmxxmmmodelower>qi (op0, operands[1],
> + operands[2], operands[3]));
> + emit_move_insn (operands[0],
> + lowpart_subreg (<MODE>mode, op0,
> <mmxxmmmode>mode));
> + DONE;
> +})
> +
> (define_expand "vec_cmpv2hfqi"
> [(set (match_operand:QI 0 "register_operand")
> (match_operator:QI 1 ""
> diff --git a/gcc/testsuite/gcc.target/i386/vect-pr126789-2.c
> b/gcc/testsuite/gcc.target/i386/vect-pr126789-2.c
> new file mode 100644
> index 00000000000..85d177bc3a4
> --- /dev/null
> +++ b/gcc/testsuite/gcc.target/i386/vect-pr126789-2.c
> @@ -0,0 +1,19 @@
> +/* { dg-do compile { target { ! ia32 } } } */
> +/* { dg-options "-O2 -mavx512vl -fno-vect-cost-model" } */
> +
> +int foo (double g, int f, double *r, int *s) {
> + int hu = 0;
> + bool test0 = r[0] < g;
> + bool test1 = r[1] < g;
> + bool test2 = s[0] < f;
> + bool test3 = s[1] < f;
> + hu += (test0 & test2) + (test1 & test3);
> + return hu;
> +}
> +
> +/* { dg-final { scan-assembler "vcmppd" } } */
> +/* That we use vpcmpgtd and not vpcmpd is because ix86_get_mask_mode
> + does not get us QImode for MMX modes. But we should be able to
> + inter-operate with mixed SSE/AVX512 masks and vectorize the
> +reduction. */
> +/* { dg-final { scan-assembler "vpcmpgtd" } } */
> diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc index
> 7ecbc8fb391..bbd510b3cb9 100644
> --- a/gcc/tree-vect-stmts.cc
> +++ b/gcc/tree-vect-stmts.cc
> @@ -5780,12 +5780,33 @@ vectorizable_conversion (vec_info *vinfo,
> vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> vop0 = new_temp;
> }
> - new_stmt = vect_gimple_build (vec_dest, code1, vop0);
> - new_temp = make_ssa_name (vec_dest, new_stmt);
> - gimple_set_lhs (new_stmt, new_temp);
> - vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> + if (code1 == COND_EXPR)
> + {
> + gcc_assert (!multi_step_cvt);
> + new_stmt
> + = gimple_build_assign (vec_dest, VEC_COND_EXPR, vop0,
> + build_minus_one_cst
> + (TREE_TYPE (vec_dest)),
> + build_zero_cst (TREE_TYPE (vec_dest)));
> + new_temp = make_ssa_name (vec_dest, new_stmt);
> + gimple_set_lhs (new_stmt, new_temp);
> + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> + tree new_temp2 = make_ssa_name (vectype_out);
> + new_stmt = gimple_build_assign (new_temp2,
> + build1 (VIEW_CONVERT_EXPR,
> + vectype_out, new_temp));
> + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
> + slp_node->push_vec_def (new_temp2);
> + }
> + else
> + {
> + new_stmt = vect_gimple_build (vec_dest, code1, vop0);
> + new_temp = make_ssa_name (vec_dest, new_stmt);
> + gimple_set_lhs (new_stmt, new_temp);
> + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi);
>
> - slp_node->push_vec_def (new_stmt);
> + slp_node->push_vec_def (new_stmt);
> + }
> }
> break;
>
> @@ -14578,6 +14599,23 @@ supportable_indirect_convert_operation
> (code_helper code,
> return true;
> }
>
> + /* For conversions between mask types where the destination has
> + a data mode attempt a vcond_mask conversion. */
> + if (VECTOR_BOOLEAN_TYPE_P (vectype_in)
> + && VECTOR_BOOLEAN_TYPE_P (vectype_out)
> + && GET_MODE_CLASS (TYPE_MODE (vectype_out)) ==
> MODE_VECTOR_INT)
> + {
> + tree scalar_datatype
> + = build_nonstandard_integer_type (element_precision (vectype_out),
> 0);
> + tree datatype_out = build_vector_type_for_mode (scalar_datatype,
> + TYPE_MODE
> (vectype_out));
> + if (expand_vec_cond_expr_p (datatype_out, vectype_in))
> + {
> + converts.safe_push (std::make_pair (datatype_out, COND_EXPR));
> + return true;
> + }
> + }
> +
> /* For conversions between float and integer types try whether
> we can use intermediate signed integer types to support the
> conversion. */
> --
> 2.51.0
