This patch refactors the AArch64 SVE vector-compare helpers so that callers
can supply the predicate that should govern the comparison. Existing callers
keep their current all-true behaviour, but the helpers no longer assume all
lanes are active.
That is a small mechanical change, but it is the backend hook needed by the
following conditional vector compare implementation. The middle end can now
pass the mask captured by COND_VEC_CMP or COND_LEN_VEC_CMP all the way to the
target compare expander.
For example:
void f (int *__restrict a, int *__restrict b, int *__restrict c,
int *__restrict d)
{
for (int i = 0; i < 1024; ++i)
if (c[i] != 0 && a[i] < b[i])
d[i] = a[i];
}
Old GIMPLE/code generation carried this as separate compare and mask
operations, so the target had to rediscover that the second compare was
governed by the first mask:
mask0 = vect_c != 0;
mask1 = vect_a < vect_b;
mask2 = mask0 & mask1;
With this branch and aarch64-none-elf-gcc, SVE receives the governing
predicate directly and emits the second compare under that predicate:
cmpne p7.s, p7/z, z29.s, #0
ld1w z31.s, p7/z, [x0, x4, lsl 2]
ld1w z30.s, p7/z, [x1, x4, lsl 2]
cmplt p7.s, p7/z, z31.s, z30.s
st1w z31.s, p7, [x3, x4, lsl 2]
The key point in the new code is the second compare: it is explicitly
performed under p7 rather than under an implicit ptrue. That is exactly the
information carried by the conditional compare IFN.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
Thanks,
Tamar
gcc/ChangeLog:
* config/aarch64/aarch64-protos.h: Update SVE compare prototypes.
(aarch64_expand_sve_vec_cmp_int): Add governing predicate parameter.
(aarch64_expand_sve_vec_cmp_float): Likewise.
* config/aarch64/aarch64.cc (aarch64_sve_emit_int_cmp): Accept a
governing predicate.
(aarch64_convert_sve_data_to_pred): Pass the predicate through.
(aarch64_expand_sve_vec_cmp_int): Likewise.
(aarch64_expand_sve_vec_cmp_float): Likewise.
---
diff --git a/gcc/config/aarch64/aarch64-protos.h
b/gcc/config/aarch64/aarch64-protos.h
index
4d882ce90ae2ea50b9b33a380af7a6f1d0bc27a5..4d38e65f9c41d7e201f8300893552640e19e7942
100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1136,8 +1136,10 @@ bool aarch64_gen_adjusted_ldpstp (rtx *, bool,
machine_mode, RTX_CODE);
void aarch64_finish_ldpstp_peephole (rtx *, bool,
enum rtx_code = (enum rtx_code)0);
-void aarch64_expand_sve_vec_cmp_int (rtx, rtx_code, rtx, rtx);
-void aarch64_expand_sve_vec_cmp_float (rtx, rtx_code, rtx, rtx);
+void aarch64_expand_sve_vec_cmp_int (rtx, rtx_code, rtx, rtx,
+ rtx = NULL_RTX);
+void aarch64_expand_sve_vec_cmp_float (rtx, rtx_code, rtx, rtx,
+ rtx = NULL_RTX);
bool aarch64_prepare_sve_int_fma (rtx *, rtx_code);
bool aarch64_prepare_sve_cond_int_fma (rtx *, rtx_code);
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index
b5239681addf09ab245ba885fff58cf61eef4f0e..dfb045b2afe71ea5fc8837760ebabebebffd6909
100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -4337,7 +4337,8 @@ aarch64_sve_emit_masked_fp_pred (machine_mode data_mode,
rtx pred)
static rtx
aarch64_sve_emit_int_cmp (rtx target, machine_mode pred_mode, rtx_code cmp,
- machine_mode data_mode, rtx op1, rtx op2)
+ machine_mode data_mode, rtx op1, rtx op2,
+ rtx pred)
{
auto src_pred_mode = aarch64_sve_pred_mode (data_mode);
insn_code icode;
@@ -4347,8 +4348,13 @@ aarch64_sve_emit_int_cmp (rtx target, machine_mode
pred_mode, rtx_code cmp,
icode = code_for_aarch64_pred_cmp_acle (cmp, data_mode);
expand_operand ops[5];
create_output_operand (&ops[0], target, pred_mode);
- create_input_operand (&ops[1], CONSTM1_RTX (src_pred_mode), src_pred_mode);
- create_integer_operand (&ops[2], SVE_KNOWN_PTRUE);
+ bool known_ptrue_p = !pred;
+ if (known_ptrue_p)
+ pred = CONSTM1_RTX (src_pred_mode);
+ create_input_operand (&ops[1], pred, src_pred_mode);
+ create_integer_operand (&ops[2], (known_ptrue_p
+ ? SVE_KNOWN_PTRUE
+ : SVE_MAYBE_NOT_PTRUE));
create_input_operand (&ops[3], op1, data_mode);
create_input_operand (&ops[4], op2, data_mode);
expand_insn (icode, 5, ops);
@@ -4363,7 +4369,7 @@ aarch64_convert_sve_data_to_pred (rtx target, rtx src)
{
machine_mode src_mode = GET_MODE (src);
return aarch64_sve_emit_int_cmp (target, VNx16BImode, NE, src_mode,
- src, CONST0_RTX (src_mode));
+ src, CONST0_RTX (src_mode), NULL_RTX);
}
/* Return the assembly token for svprfop value PRFOP. */
@@ -28997,15 +29003,19 @@ aarch64_reverse_mask (machine_mode mode, unsigned int
nunits)
/* Expand an SVE integer comparison using the SVE equivalent of:
- (set TARGET (CODE OP0 OP1)). */
+ (set TARGET (CODE OP0 OP1)).
+
+ If PRED is nonnull, use it as the governing predicate instead of an
+ all-true predicate. */
void
-aarch64_expand_sve_vec_cmp_int (rtx target, rtx_code code, rtx op0, rtx op1)
+aarch64_expand_sve_vec_cmp_int (rtx target, rtx_code code, rtx op0, rtx op1,
+ rtx pred)
{
machine_mode pred_mode = GET_MODE (target);
machine_mode data_mode = GET_MODE (op0);
rtx res = aarch64_sve_emit_int_cmp (target, pred_mode, code, data_mode,
- op0, op1);
+ op0, op1, pred);
if (!rtx_equal_p (target, res))
emit_move_insn (target, res);
}
@@ -29097,14 +29107,19 @@ aarch64_emit_sve_invert_fp_cond (rtx target, rtx_code
code, rtx pred,
(set TARGET (CODE OP0 OP1)). */
void
-aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
+aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1,
+ rtx pred)
{
machine_mode data_mode = GET_MODE (op0);
- rtx pred = aarch64_sve_fp_pred (data_mode, nullptr);
+ bool pred_supplied_p = pred != NULL_RTX;
+ if (!pred)
+ pred = aarch64_sve_fp_pred (data_mode, nullptr);
/* The governing and destination modes. */
machine_mode pred_mode = GET_MODE (pred);
machine_mode target_mode = GET_MODE (target);
+ if (!aarch64_simd_reg_or_zero (op1, data_mode))
+ op1 = force_reg (data_mode, op1);
/* For partial vector modes, the choice of predicate mode depends
on whether we need to suppress exceptions for inactive elements.
@@ -29117,7 +29132,7 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code
code, rtx op0, rtx op1)
If we don't need to suppress exceptions, the predicate mode matches
the container size, PRED_MODE == TARGET_MODE, and the predicate is
thus a ptrue wrt both TARGET_MODE and PRED_MODE. */
- bool known_ptrue_p = pred_mode == target_mode;
+ bool known_ptrue_p = !pred_supplied_p && pred_mode == target_mode;
switch (code)
{
case UNORDERED:
@@ -29163,7 +29178,7 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code
code, rtx op0, rtx op1)
aarch64_emit_sve_fp_cond (uo_tmp, UNORDERED,
pred, known_ptrue_p, op0, op1);
- /* Invert the result. Governered by PRED so that we only
+ /* Invert the result. Governed by PRED so that we only
flip the active bits. */
rtx ordered = gen_reg_rtx (pred_mode);
uo_tmp = gen_lowpart (pred_mode, uo_tmp);
--
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index 4d882ce90ae2ea50b9b33a380af7a6f1d0bc27a5..4d38e65f9c41d7e201f8300893552640e19e7942 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1136,8 +1136,10 @@ bool aarch64_gen_adjusted_ldpstp (rtx *, bool, machine_mode, RTX_CODE);
void aarch64_finish_ldpstp_peephole (rtx *, bool,
enum rtx_code = (enum rtx_code)0);
-void aarch64_expand_sve_vec_cmp_int (rtx, rtx_code, rtx, rtx);
-void aarch64_expand_sve_vec_cmp_float (rtx, rtx_code, rtx, rtx);
+void aarch64_expand_sve_vec_cmp_int (rtx, rtx_code, rtx, rtx,
+ rtx = NULL_RTX);
+void aarch64_expand_sve_vec_cmp_float (rtx, rtx_code, rtx, rtx,
+ rtx = NULL_RTX);
bool aarch64_prepare_sve_int_fma (rtx *, rtx_code);
bool aarch64_prepare_sve_cond_int_fma (rtx *, rtx_code);
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index b5239681addf09ab245ba885fff58cf61eef4f0e..dfb045b2afe71ea5fc8837760ebabebebffd6909 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -4337,7 +4337,8 @@ aarch64_sve_emit_masked_fp_pred (machine_mode data_mode, rtx pred)
static rtx
aarch64_sve_emit_int_cmp (rtx target, machine_mode pred_mode, rtx_code cmp,
- machine_mode data_mode, rtx op1, rtx op2)
+ machine_mode data_mode, rtx op1, rtx op2,
+ rtx pred)
{
auto src_pred_mode = aarch64_sve_pred_mode (data_mode);
insn_code icode;
@@ -4347,8 +4348,13 @@ aarch64_sve_emit_int_cmp (rtx target, machine_mode pred_mode, rtx_code cmp,
icode = code_for_aarch64_pred_cmp_acle (cmp, data_mode);
expand_operand ops[5];
create_output_operand (&ops[0], target, pred_mode);
- create_input_operand (&ops[1], CONSTM1_RTX (src_pred_mode), src_pred_mode);
- create_integer_operand (&ops[2], SVE_KNOWN_PTRUE);
+ bool known_ptrue_p = !pred;
+ if (known_ptrue_p)
+ pred = CONSTM1_RTX (src_pred_mode);
+ create_input_operand (&ops[1], pred, src_pred_mode);
+ create_integer_operand (&ops[2], (known_ptrue_p
+ ? SVE_KNOWN_PTRUE
+ : SVE_MAYBE_NOT_PTRUE));
create_input_operand (&ops[3], op1, data_mode);
create_input_operand (&ops[4], op2, data_mode);
expand_insn (icode, 5, ops);
@@ -4363,7 +4369,7 @@ aarch64_convert_sve_data_to_pred (rtx target, rtx src)
{
machine_mode src_mode = GET_MODE (src);
return aarch64_sve_emit_int_cmp (target, VNx16BImode, NE, src_mode,
- src, CONST0_RTX (src_mode));
+ src, CONST0_RTX (src_mode), NULL_RTX);
}
/* Return the assembly token for svprfop value PRFOP. */
@@ -28997,15 +29003,19 @@ aarch64_reverse_mask (machine_mode mode, unsigned int nunits)
/* Expand an SVE integer comparison using the SVE equivalent of:
- (set TARGET (CODE OP0 OP1)). */
+ (set TARGET (CODE OP0 OP1)).
+
+ If PRED is nonnull, use it as the governing predicate instead of an
+ all-true predicate. */
void
-aarch64_expand_sve_vec_cmp_int (rtx target, rtx_code code, rtx op0, rtx op1)
+aarch64_expand_sve_vec_cmp_int (rtx target, rtx_code code, rtx op0, rtx op1,
+ rtx pred)
{
machine_mode pred_mode = GET_MODE (target);
machine_mode data_mode = GET_MODE (op0);
rtx res = aarch64_sve_emit_int_cmp (target, pred_mode, code, data_mode,
- op0, op1);
+ op0, op1, pred);
if (!rtx_equal_p (target, res))
emit_move_insn (target, res);
}
@@ -29097,14 +29107,19 @@ aarch64_emit_sve_invert_fp_cond (rtx target, rtx_code code, rtx pred,
(set TARGET (CODE OP0 OP1)). */
void
-aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
+aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1,
+ rtx pred)
{
machine_mode data_mode = GET_MODE (op0);
- rtx pred = aarch64_sve_fp_pred (data_mode, nullptr);
+ bool pred_supplied_p = pred != NULL_RTX;
+ if (!pred)
+ pred = aarch64_sve_fp_pred (data_mode, nullptr);
/* The governing and destination modes. */
machine_mode pred_mode = GET_MODE (pred);
machine_mode target_mode = GET_MODE (target);
+ if (!aarch64_simd_reg_or_zero (op1, data_mode))
+ op1 = force_reg (data_mode, op1);
/* For partial vector modes, the choice of predicate mode depends
on whether we need to suppress exceptions for inactive elements.
@@ -29117,7 +29132,7 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
If we don't need to suppress exceptions, the predicate mode matches
the container size, PRED_MODE == TARGET_MODE, and the predicate is
thus a ptrue wrt both TARGET_MODE and PRED_MODE. */
- bool known_ptrue_p = pred_mode == target_mode;
+ bool known_ptrue_p = !pred_supplied_p && pred_mode == target_mode;
switch (code)
{
case UNORDERED:
@@ -29163,7 +29178,7 @@ aarch64_expand_sve_vec_cmp_float (rtx target, rtx_code code, rtx op0, rtx op1)
aarch64_emit_sve_fp_cond (uo_tmp, UNORDERED,
pred, known_ptrue_p, op0, op1);
- /* Invert the result. Governered by PRED so that we only
+ /* Invert the result. Governed by PRED so that we only
flip the active bits. */
rtx ordered = gen_reg_rtx (pred_mode);
uo_tmp = gen_lowpart (pred_mode, uo_tmp);
