This patch adds the middle-end representation for conditional vector
comparisons.
It introduces IFN_COND_VEC_CMP and IFN_COND_LEN_VEC_CMP, together with the
cond_vec_cmp and cond_len_vec_cmp optabs. The IFNs take a comparison code,
a governing vector predicate, and the two vector operands being compared.
The LEN form also carries the usual vector length and bias operands used by
length-masked internal functions.
The support queries now validate the operands that the final expander needs:
the destination predicate, mask, comparison RTX, and the
LEN/BIAS operands for the length form. The generic documentation now
describes the new IFNs and their operands.
For example, this source:
void f (int *a, int *b, int *c, int *d)
{
for (int i = 0; i < 1024; ++i)
if (c[i] != 0 && a[i] < b[i])
d[i] = a[i];
}
The old GIMPLE/code generation carried this as separate compare and mask
operations, so the target had to rediscover that the second compare was
governed by the first mask:
mask0 = vect_c != 0;
mask1 = vect_a < vect_b;
mask2 = mask0 & mask1;
With this patch, e.g. SVE receives the governing
predicate directly and emits the second compare under that predicate:
cmpne p7.s, p7/z, z29.s, #0
ld1w z31.s, p7/z, [x0, x4, lsl 2]
ld1w z30.s, p7/z, [x1, x4, lsl 2]
cmplt p7.s, p7/z, z31.s, z30.s
st1w z31.s, p7, [x3, x4, lsl 2]
The change is mostly plumbing, but it keeps predicate-producing compares
visible long enough for the conditional compare IFN and AArch64 lowering
patches to select the intended target instructions.
This change is done late in isel rather than in the vectorizer because
various passes still understand VLA compares and we'd lose all optimizations
here if we picked a different representation too early.
However we need to do this in expand, so we can have CSE eliminate duplicate
compares. Without this we could only do it after combine after which we
have no more CSE. Additional side-effect is that we can now simplify many
of the SVE comparisons.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
gcc/ChangeLog:
* doc/generic.texi: Document IFN_COND_VEC_CMP and
IFN_COND_LEN_VEC_CMP.
* internal-fn.cc (internal_fn_mask_index): Handle IFN_COND_VEC_CMP
and IFN_COND_LEN_VEC_CMP.
(expand_cond_vec_cmp_1_p): New function.
(expand_cond_vec_cmp_p): Likewise.
(expand_COND_VEC_CMP): Likewise.
(expand_cond_len_vec_cmp_p): Likewise.
(expand_COND_LEN_VEC_CMP): Likewise.
* internal-fn.def (COND_VEC_CMP): New internal function.
(COND_LEN_VEC_CMP): Likewise.
* internal-fn.h (expand_COND_VEC_CMP): Declare.
(expand_COND_LEN_VEC_CMP): Likewise.
(expand_cond_vec_cmp_p): Likewise.
(expand_cond_len_vec_cmp_p): Likewise.
* optabs.def (cond_vec_cmp_optab): New optab.
(cond_len_vec_cmp_optab): Likewise.
---
diff --git a/gcc/doc/generic.texi b/gcc/doc/generic.texi
index
42a04e34eda4539f95d4e169f20e6ffbe03ea8a4..0e910ec5e731a4580cf81ed3b13ece1026a69b5a
100644
--- a/gcc/doc/generic.texi
+++ b/gcc/doc/generic.texi
@@ -1834,6 +1834,8 @@ a value from @code{enum annot_expr_kind}, the third is an
@code{INTEGER_CST}.
@tindex IFN_VEC_WIDEN_MINUS_EVEN
@tindex IFN_VEC_WIDEN_MINUS_ODD
@tindex IFN_VEC_TRUNC_ADD_HIGH
+@tindex IFN_COND_VEC_CMP
+@tindex IFN_COND_LEN_VEC_CMP
@tindex VEC_UNPACK_HI_EXPR
@tindex VEC_UNPACK_LO_EXPR
@tindex VEC_UNPACK_FLOAT_HI_EXPR
@@ -1974,6 +1976,19 @@ type.
This operation currently only used for early break result compression when the
result of a vector boolean can be represented as 0 or -1.
+@item IFN_COND_VEC_CMP
+@itemx IFN_COND_LEN_VEC_CMP
+These internal functions represent predicated vector comparisons. The first
+argument is an integer constant whose value is a @code{tree_code} comparison
+code. The next argument is the mask, followed by the two vector operands
+being compared. The result has the same vector boolean type as the mask;
+inactive elements are false.
+
+@code{IFN_COND_LEN_VEC_CMP} has two extra arguments, the vector length and
+bias, with the same meaning as for other length-masked internal functions.
+The functions are expanded through the @code{cond_vec_cmp} and
+@code{cond_len_vec_cmp} optabs.
+
@item VEC_UNPACK_HI_EXPR
@itemx VEC_UNPACK_LO_EXPR
These nodes represent unpacking of the high and low parts of the input vector,
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index
0030db0c29bf62c4cb4aebb250be9fb5e598dbfa..fbb9847b06054a39d5db7f26e293b5af909494cb
100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -5354,6 +5354,10 @@ internal_fn_mask_index (internal_fn fn)
case IFN_VCOND_MASK_LEN:
return 0;
+ case IFN_COND_VEC_CMP:
+ case IFN_COND_LEN_VEC_CMP:
+ return 1;
+
default:
return (conditional_internal_fn_code (fn) != ERROR_MARK
|| get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
@@ -5997,3 +6001,150 @@ expand_BITREVERSE (internal_fn fn, gcall *stmt)
if (expand_bitquery (fn, stmt))
expand_unary_optab_fn (fn, stmt, bitreverse_optab);
}
+
+static bool
+expand_cond_vec_cmp_1_p (optab optab, tree_code tcode, tree mask, tree op0,
+ tree len = NULL_TREE, tree bias = NULL_TREE)
+{
+ tree lhs_type = TREE_TYPE (mask);
+ tree op_type = TREE_TYPE (op0);
+ machine_mode mask_mode = TYPE_MODE (lhs_type);
+ machine_mode op_mode = TYPE_MODE (op_type);
+
+ gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
+
+ insn_code icode = convert_optab_handler (optab, op_mode, mask_mode);
+ if (icode == CODE_FOR_nothing)
+ return false;
+
+ enum rtx_code rcode = get_rtx_code_1 (tcode, TYPE_UNSIGNED (op_type));
+ if (rcode == UNKNOWN)
+ return false;
+
+ rtx target = alloca_raw_REG (mask_mode, LAST_VIRTUAL_REGISTER);
+ if (!insn_operand_matches (icode, 0, target))
+ return false;
+
+ rtx pred = alloca_raw_REG (mask_mode, LAST_VIRTUAL_REGISTER + 1);
+ if (!insn_operand_matches (icode, 2, pred))
+ return false;
+
+ rtx reg1 = alloca_raw_REG (op_mode, LAST_VIRTUAL_REGISTER + 2);
+ rtx reg2 = alloca_raw_REG (op_mode, LAST_VIRTUAL_REGISTER + 3);
+ rtx test = alloca_rtx_fmt_ee (rcode, mask_mode, reg1, reg2);
+ if (!insn_operand_matches (icode, 1, test))
+ return false;
+
+ if (len)
+ {
+ gcc_assert (bias);
+ machine_mode len_mode = insn_data[(int) icode].operand[5].mode;
+
+ if (len_mode == VOIDmode)
+ len_mode = TYPE_MODE (TREE_TYPE (len));
+ rtx len_rtx = alloca_raw_REG (len_mode, LAST_VIRTUAL_REGISTER + 4);
+ if (!insn_operand_matches (icode, 5, len_rtx))
+ return false;
+
+ if (!tree_fits_shwi_p (bias)
+ || !insn_operand_matches (icode, 6, GEN_INT (tree_to_shwi (bias))))
+ return false;
+ }
+
+ return true;
+}
+
+bool
+expand_cond_vec_cmp_p (tree_code tcode, tree mask, tree op0, tree /* op1 */)
+{
+ return expand_cond_vec_cmp_1_p (cond_vec_cmp_optab, tcode, mask, op0);
+}
+
+void
+expand_COND_VEC_CMP (internal_fn fn, gcall *stmt)
+{
+ gcc_assert (fn == IFN_COND_VEC_CMP);
+
+ tree lhs = gimple_call_lhs (stmt);
+ tree lhs_type = TREE_TYPE (lhs);
+ tree_code tcode = (tree_code) tree_to_uhwi (gimple_call_arg (stmt, 0));
+ tree mask = gimple_call_arg (stmt, 1);
+ tree op0 = gimple_call_arg (stmt, 2);
+ tree op1 = gimple_call_arg (stmt, 3);
+ tree op_type = TREE_TYPE (op0);
+ machine_mode mask_mode = TYPE_MODE (lhs_type);
+ machine_mode op_mode = TYPE_MODE (op_type);
+ bool unsignedp = TYPE_UNSIGNED (op_type);
+
+ gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
+
+ insn_code icode = convert_optab_handler (cond_vec_cmp_optab, op_mode,
+ mask_mode);
+ gcc_assert (icode != CODE_FOR_nothing);
+
+ rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ rtx mask_rtx = expand_normal (mask);
+ rtx comparison = vector_compare_rtx (mask_mode, tcode, op0, op1,
+ unsignedp, icode, 3);
+
+ class expand_operand ops[5];
+ create_call_lhs_operand (&ops[0], lhs_rtx, mask_mode);
+ create_fixed_operand (&ops[1], comparison);
+ create_input_operand (&ops[2], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
+ create_fixed_operand (&ops[3], XEXP (comparison, 0));
+ create_fixed_operand (&ops[4], XEXP (comparison, 1));
+ expand_insn (icode, 5, ops);
+ assign_call_lhs (lhs, lhs_rtx, &ops[0]);
+}
+
+bool
+expand_cond_len_vec_cmp_p (tree_code tcode, tree mask, tree op0, tree /* op1
*/,
+ tree len, tree bias)
+{
+ return expand_cond_vec_cmp_1_p (cond_len_vec_cmp_optab, tcode, mask, op0,
+ len, bias);
+}
+
+void
+expand_COND_LEN_VEC_CMP (internal_fn fn, gcall *stmt)
+{
+ gcc_assert (fn == IFN_COND_LEN_VEC_CMP);
+
+ tree lhs = gimple_call_lhs (stmt);
+ tree lhs_type = TREE_TYPE (lhs);
+ tree_code tcode = (tree_code) tree_to_uhwi (gimple_call_arg (stmt, 0));
+ tree mask = gimple_call_arg (stmt, 1);
+ tree op0 = gimple_call_arg (stmt, 2);
+ tree op1 = gimple_call_arg (stmt, 3);
+ tree len = gimple_call_arg (stmt, 4);
+ tree bias = gimple_call_arg (stmt, 5);
+ tree op_type = TREE_TYPE (op0);
+ machine_mode mask_mode = TYPE_MODE (lhs_type);
+ machine_mode op_mode = TYPE_MODE (op_type);
+ bool unsignedp = TYPE_UNSIGNED (op_type);
+
+ gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
+
+ insn_code icode = convert_optab_handler (cond_len_vec_cmp_optab, op_mode,
+ mask_mode);
+ gcc_assert (icode != CODE_FOR_nothing);
+
+ rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ rtx mask_rtx = expand_normal (mask);
+ rtx len_rtx = expand_normal (len);
+ rtx bias_rtx = expand_normal (bias);
+ rtx comparison = vector_compare_rtx (mask_mode, tcode, op0, op1,
+ unsignedp, icode, 3);
+
+ class expand_operand ops[7];
+ create_call_lhs_operand (&ops[0], lhs_rtx, mask_mode);
+ create_fixed_operand (&ops[1], comparison);
+ create_input_operand (&ops[2], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
+ create_fixed_operand (&ops[3], XEXP (comparison, 0));
+ create_fixed_operand (&ops[4], XEXP (comparison, 1));
+ create_convert_operand_from (&ops[5], len_rtx, TYPE_MODE (TREE_TYPE (len)),
+ TYPE_UNSIGNED (TREE_TYPE (len)));
+ create_input_operand (&ops[6], bias_rtx, QImode);
+ expand_insn (icode, 7, ops);
+ assign_call_lhs (lhs, lhs_rtx, &ops[0]);
+}
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index
8ec060415e60dfb6d073a7041c866a1fdf05e800..46879437c9345f09b9d239d4862a93baa4164d5f
100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -578,6 +578,10 @@ DEF_INTERNAL_FN (VA_ARG, ECF_NOTHROW | ECF_LEAF, NULL)
DEF_INTERNAL_FN (VEC_CONVERT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (RAWMEMCHR, ECF_PURE | ECF_LEAF | ECF_NOTHROW, NULL)
+/* Conditional vector comparisons. These use custom expansions. */
+DEF_INTERNAL_FN (COND_VEC_CMP, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (COND_LEN_VEC_CMP, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+
/* An unduplicable, uncombinable function. Generally used to preserve
a CFG property in the face of jump threading, tail merging or
other such optimizations. The first argument distinguishes
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index
30930f388b069ca98167f0911c62681143979327..ac3a35eef62663267dd7ed8c749d02154208486b
100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -288,6 +288,11 @@ extern void expand_PARITY (internal_fn, gcall *);
extern void expand_POPCOUNT (internal_fn, gcall *);
extern void expand_BSWAP (internal_fn, gcall *);
extern void expand_BITREVERSE (internal_fn, gcall *);
+extern void expand_COND_VEC_CMP (internal_fn, gcall *);
+extern void expand_COND_LEN_VEC_CMP (internal_fn, gcall *);
+
+extern bool expand_cond_vec_cmp_p (tree_code, tree, tree, tree);
+extern bool expand_cond_len_vec_cmp_p (tree_code, tree, tree, tree, tree,
tree);
extern bool vectorized_internal_fn_supported_p (internal_fn, tree);
diff --git a/gcc/optabs.def b/gcc/optabs.def
index
73571fe700c6cb73b35f7f25796bb1b2d090d244..d8df39a50ecca33e38c1178a8dbb9fa6ca9cf941
100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -99,6 +99,8 @@ OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b")
OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b")
OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b")
OPTAB_CD(vec_cmpeq_optab, "vec_cmpeq$a$b")
+OPTAB_CD(cond_vec_cmp_optab, "cond_vec_cmp$a$b")
+OPTAB_CD(cond_len_vec_cmp_optab, "cond_len_vec_cmp$a$b")
OPTAB_CD(maskload_optab, "maskload$a$b")
OPTAB_CD(maskstore_optab, "maskstore$a$b")
OPTAB_CD(mask_len_load_optab, "mask_len_load$a$b")
--
diff --git a/gcc/doc/generic.texi b/gcc/doc/generic.texi
index 42a04e34eda4539f95d4e169f20e6ffbe03ea8a4..0e910ec5e731a4580cf81ed3b13ece1026a69b5a 100644
--- a/gcc/doc/generic.texi
+++ b/gcc/doc/generic.texi
@@ -1834,6 +1834,8 @@ a value from @code{enum annot_expr_kind}, the third is an @code{INTEGER_CST}.
@tindex IFN_VEC_WIDEN_MINUS_EVEN
@tindex IFN_VEC_WIDEN_MINUS_ODD
@tindex IFN_VEC_TRUNC_ADD_HIGH
+@tindex IFN_COND_VEC_CMP
+@tindex IFN_COND_LEN_VEC_CMP
@tindex VEC_UNPACK_HI_EXPR
@tindex VEC_UNPACK_LO_EXPR
@tindex VEC_UNPACK_FLOAT_HI_EXPR
@@ -1974,6 +1976,19 @@ type.
This operation currently only used for early break result compression when the
result of a vector boolean can be represented as 0 or -1.
+@item IFN_COND_VEC_CMP
+@itemx IFN_COND_LEN_VEC_CMP
+These internal functions represent predicated vector comparisons. The first
+argument is an integer constant whose value is a @code{tree_code} comparison
+code. The next argument is the mask, followed by the two vector operands
+being compared. The result has the same vector boolean type as the mask;
+inactive elements are false.
+
+@code{IFN_COND_LEN_VEC_CMP} has two extra arguments, the vector length and
+bias, with the same meaning as for other length-masked internal functions.
+The functions are expanded through the @code{cond_vec_cmp} and
+@code{cond_len_vec_cmp} optabs.
+
@item VEC_UNPACK_HI_EXPR
@itemx VEC_UNPACK_LO_EXPR
These nodes represent unpacking of the high and low parts of the input vector,
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 0030db0c29bf62c4cb4aebb250be9fb5e598dbfa..fbb9847b06054a39d5db7f26e293b5af909494cb 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -5354,6 +5354,10 @@ internal_fn_mask_index (internal_fn fn)
case IFN_VCOND_MASK_LEN:
return 0;
+ case IFN_COND_VEC_CMP:
+ case IFN_COND_LEN_VEC_CMP:
+ return 1;
+
default:
return (conditional_internal_fn_code (fn) != ERROR_MARK
|| get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
@@ -5997,3 +6001,150 @@ expand_BITREVERSE (internal_fn fn, gcall *stmt)
if (expand_bitquery (fn, stmt))
expand_unary_optab_fn (fn, stmt, bitreverse_optab);
}
+
+static bool
+expand_cond_vec_cmp_1_p (optab optab, tree_code tcode, tree mask, tree op0,
+ tree len = NULL_TREE, tree bias = NULL_TREE)
+{
+ tree lhs_type = TREE_TYPE (mask);
+ tree op_type = TREE_TYPE (op0);
+ machine_mode mask_mode = TYPE_MODE (lhs_type);
+ machine_mode op_mode = TYPE_MODE (op_type);
+
+ gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
+
+ insn_code icode = convert_optab_handler (optab, op_mode, mask_mode);
+ if (icode == CODE_FOR_nothing)
+ return false;
+
+ enum rtx_code rcode = get_rtx_code_1 (tcode, TYPE_UNSIGNED (op_type));
+ if (rcode == UNKNOWN)
+ return false;
+
+ rtx target = alloca_raw_REG (mask_mode, LAST_VIRTUAL_REGISTER);
+ if (!insn_operand_matches (icode, 0, target))
+ return false;
+
+ rtx pred = alloca_raw_REG (mask_mode, LAST_VIRTUAL_REGISTER + 1);
+ if (!insn_operand_matches (icode, 2, pred))
+ return false;
+
+ rtx reg1 = alloca_raw_REG (op_mode, LAST_VIRTUAL_REGISTER + 2);
+ rtx reg2 = alloca_raw_REG (op_mode, LAST_VIRTUAL_REGISTER + 3);
+ rtx test = alloca_rtx_fmt_ee (rcode, mask_mode, reg1, reg2);
+ if (!insn_operand_matches (icode, 1, test))
+ return false;
+
+ if (len)
+ {
+ gcc_assert (bias);
+ machine_mode len_mode = insn_data[(int) icode].operand[5].mode;
+
+ if (len_mode == VOIDmode)
+ len_mode = TYPE_MODE (TREE_TYPE (len));
+ rtx len_rtx = alloca_raw_REG (len_mode, LAST_VIRTUAL_REGISTER + 4);
+ if (!insn_operand_matches (icode, 5, len_rtx))
+ return false;
+
+ if (!tree_fits_shwi_p (bias)
+ || !insn_operand_matches (icode, 6, GEN_INT (tree_to_shwi (bias))))
+ return false;
+ }
+
+ return true;
+}
+
+bool
+expand_cond_vec_cmp_p (tree_code tcode, tree mask, tree op0, tree /* op1 */)
+{
+ return expand_cond_vec_cmp_1_p (cond_vec_cmp_optab, tcode, mask, op0);
+}
+
+void
+expand_COND_VEC_CMP (internal_fn fn, gcall *stmt)
+{
+ gcc_assert (fn == IFN_COND_VEC_CMP);
+
+ tree lhs = gimple_call_lhs (stmt);
+ tree lhs_type = TREE_TYPE (lhs);
+ tree_code tcode = (tree_code) tree_to_uhwi (gimple_call_arg (stmt, 0));
+ tree mask = gimple_call_arg (stmt, 1);
+ tree op0 = gimple_call_arg (stmt, 2);
+ tree op1 = gimple_call_arg (stmt, 3);
+ tree op_type = TREE_TYPE (op0);
+ machine_mode mask_mode = TYPE_MODE (lhs_type);
+ machine_mode op_mode = TYPE_MODE (op_type);
+ bool unsignedp = TYPE_UNSIGNED (op_type);
+
+ gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
+
+ insn_code icode = convert_optab_handler (cond_vec_cmp_optab, op_mode,
+ mask_mode);
+ gcc_assert (icode != CODE_FOR_nothing);
+
+ rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ rtx mask_rtx = expand_normal (mask);
+ rtx comparison = vector_compare_rtx (mask_mode, tcode, op0, op1,
+ unsignedp, icode, 3);
+
+ class expand_operand ops[5];
+ create_call_lhs_operand (&ops[0], lhs_rtx, mask_mode);
+ create_fixed_operand (&ops[1], comparison);
+ create_input_operand (&ops[2], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
+ create_fixed_operand (&ops[3], XEXP (comparison, 0));
+ create_fixed_operand (&ops[4], XEXP (comparison, 1));
+ expand_insn (icode, 5, ops);
+ assign_call_lhs (lhs, lhs_rtx, &ops[0]);
+}
+
+bool
+expand_cond_len_vec_cmp_p (tree_code tcode, tree mask, tree op0, tree /* op1 */,
+ tree len, tree bias)
+{
+ return expand_cond_vec_cmp_1_p (cond_len_vec_cmp_optab, tcode, mask, op0,
+ len, bias);
+}
+
+void
+expand_COND_LEN_VEC_CMP (internal_fn fn, gcall *stmt)
+{
+ gcc_assert (fn == IFN_COND_LEN_VEC_CMP);
+
+ tree lhs = gimple_call_lhs (stmt);
+ tree lhs_type = TREE_TYPE (lhs);
+ tree_code tcode = (tree_code) tree_to_uhwi (gimple_call_arg (stmt, 0));
+ tree mask = gimple_call_arg (stmt, 1);
+ tree op0 = gimple_call_arg (stmt, 2);
+ tree op1 = gimple_call_arg (stmt, 3);
+ tree len = gimple_call_arg (stmt, 4);
+ tree bias = gimple_call_arg (stmt, 5);
+ tree op_type = TREE_TYPE (op0);
+ machine_mode mask_mode = TYPE_MODE (lhs_type);
+ machine_mode op_mode = TYPE_MODE (op_type);
+ bool unsignedp = TYPE_UNSIGNED (op_type);
+
+ gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
+
+ insn_code icode = convert_optab_handler (cond_len_vec_cmp_optab, op_mode,
+ mask_mode);
+ gcc_assert (icode != CODE_FOR_nothing);
+
+ rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
+ rtx mask_rtx = expand_normal (mask);
+ rtx len_rtx = expand_normal (len);
+ rtx bias_rtx = expand_normal (bias);
+ rtx comparison = vector_compare_rtx (mask_mode, tcode, op0, op1,
+ unsignedp, icode, 3);
+
+ class expand_operand ops[7];
+ create_call_lhs_operand (&ops[0], lhs_rtx, mask_mode);
+ create_fixed_operand (&ops[1], comparison);
+ create_input_operand (&ops[2], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
+ create_fixed_operand (&ops[3], XEXP (comparison, 0));
+ create_fixed_operand (&ops[4], XEXP (comparison, 1));
+ create_convert_operand_from (&ops[5], len_rtx, TYPE_MODE (TREE_TYPE (len)),
+ TYPE_UNSIGNED (TREE_TYPE (len)));
+ create_input_operand (&ops[6], bias_rtx, QImode);
+ expand_insn (icode, 7, ops);
+ assign_call_lhs (lhs, lhs_rtx, &ops[0]);
+}
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index 8ec060415e60dfb6d073a7041c866a1fdf05e800..46879437c9345f09b9d239d4862a93baa4164d5f 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -578,6 +578,10 @@ DEF_INTERNAL_FN (VA_ARG, ECF_NOTHROW | ECF_LEAF, NULL)
DEF_INTERNAL_FN (VEC_CONVERT, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (RAWMEMCHR, ECF_PURE | ECF_LEAF | ECF_NOTHROW, NULL)
+/* Conditional vector comparisons. These use custom expansions. */
+DEF_INTERNAL_FN (COND_VEC_CMP, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+DEF_INTERNAL_FN (COND_LEN_VEC_CMP, ECF_CONST | ECF_LEAF | ECF_NOTHROW, NULL)
+
/* An unduplicable, uncombinable function. Generally used to preserve
a CFG property in the face of jump threading, tail merging or
other such optimizations. The first argument distinguishes
diff --git a/gcc/internal-fn.h b/gcc/internal-fn.h
index 30930f388b069ca98167f0911c62681143979327..ac3a35eef62663267dd7ed8c749d02154208486b 100644
--- a/gcc/internal-fn.h
+++ b/gcc/internal-fn.h
@@ -288,6 +288,11 @@ extern void expand_PARITY (internal_fn, gcall *);
extern void expand_POPCOUNT (internal_fn, gcall *);
extern void expand_BSWAP (internal_fn, gcall *);
extern void expand_BITREVERSE (internal_fn, gcall *);
+extern void expand_COND_VEC_CMP (internal_fn, gcall *);
+extern void expand_COND_LEN_VEC_CMP (internal_fn, gcall *);
+
+extern bool expand_cond_vec_cmp_p (tree_code, tree, tree, tree);
+extern bool expand_cond_len_vec_cmp_p (tree_code, tree, tree, tree, tree, tree);
extern bool vectorized_internal_fn_supported_p (internal_fn, tree);
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 73571fe700c6cb73b35f7f25796bb1b2d090d244..d8df39a50ecca33e38c1178a8dbb9fa6ca9cf941 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -99,6 +99,8 @@ OPTAB_CD(vcond_mask_optab, "vcond_mask_$a$b")
OPTAB_CD(vec_cmp_optab, "vec_cmp$a$b")
OPTAB_CD(vec_cmpu_optab, "vec_cmpu$a$b")
OPTAB_CD(vec_cmpeq_optab, "vec_cmpeq$a$b")
+OPTAB_CD(cond_vec_cmp_optab, "cond_vec_cmp$a$b")
+OPTAB_CD(cond_len_vec_cmp_optab, "cond_len_vec_cmp$a$b")
OPTAB_CD(maskload_optab, "maskload$a$b")
OPTAB_CD(maskstore_optab, "maskstore$a$b")
OPTAB_CD(mask_len_load_optab, "mask_len_load$a$b")
