This builds upon my previous patch for compare-elim [1]
[1] https://gcc.gnu.org/pipermail/gcc-patches/2026-September/731176.html
Compare-elim can already merge a flag-setting operation with an earlier
value-producing instruction when the target can recognize the final combined
parallel form. This patch extends the same idea to PTEST-like operations
whose flags are only used by equality comparisons against zero.
In SVE, PTEST derives NZCV from predicate bits selected by a governing
predicate.
The important distinction is that a full PTEST condition code carries more
information than a zero/nonzero test. If later code only branches using EQ or
NE
on a comparison with 0 then only the Z flag is meaningful. In that case the
earlier
predicate-producing instruction can be used instead of the PTEST flags and the
PTEST can be eliminated. In such cases it only needs to provide an equivalent
zero/nonzero answer.
However the SVE architecture definition for values of predicates (i.e. BImode)
indicates that only the bottom bit of the lane is checked. That means that e.g.
on
a VNx16BImode vector boolean for an HI element data vector, both 0x00 and 0x10
indicate that the lane is inactive and e.g. that no match is found so the Z
flag is
set.
But the PTEST can re-shape the CC flag. and so if the PTEST's GP is .b then
`0x10` is re-interpreted as `0x1, 0x0` and so the Z flag is cleared as there is
now
and active lane.
What makes the elimination possible is that even though the architecture allows
this flexibility, the individual instructions explicitly say which bits they
set.
For instance an SVE Integer comparison explicitly says the lanes are 0x1 and 0x0
zero extended.
The generic pass therefore does not encode any target specific information. We
query the target asking if for a given comparison which flags are relevant using
the existing select_cc_mode target hook and rewrite the instruction into one
setting only the returned flag and perform recog. Should the target deem the
replacement to be valid and recog succeeds we accept the new shape.
This delegate the responsibility to perform the legality checks to the target.
In RTL terms, the interesting shape is:
(set (reg:MODE PRED) ...)
(set (reg:CC_NZC CC)
(unspec:CC_NZC [... (reg:MODE PRED)] UNSPEC_PTEST))
(set (...) (ne/eq (...) (reg:CC_NZC CC) (const_int 0)))
When the flags users are all EQ/NE, compare-elim can try to fold the first two
instructions into:
(parallel [
(set (reg:CC_Z CC)
(unspec:CC_Z [... <producer expression>] UNSPEC_PTEST))
(set (reg:MODE PRED) <producer expression>)
])
For AArch64 this lets the following target patch remove PTESTs after a
predicate-producing SVE instructions, while still rejecting cases where FIRST
or LAST semantics need the full condition-code result or where the semantics
or use of the predicate makes the elimination invalid.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
OK for master?
Thanks,
Tamar
gcc/ChangeLog
* compare-elim.cc (find_comparison_dom_walker::before_dom_children):
Prefer the tested value setter as the producer for non-COMPARE flags
sets.
(comparison_producer_set): New function.
(finish_eliminate_compare): Likewise.
(try_eliminate_compare_with_flags_src): Likewise.
(try_eliminate_compare): Use comparison_producer_set. Try direct
elimination before rebuilding the flags source for zero-only tests.
---
diff --git a/gcc/compare-elim.cc b/gcc/compare-elim.cc
index
0f10f6874490f8c3c08949a8a7512eb6ffc287db..1855604cd2c39c6e2afebb0e6c1bd74a2c4280ef
100644
--- a/gcc/compare-elim.cc
+++ b/gcc/compare-elim.cc
@@ -359,6 +359,27 @@ can_eliminate_compare (rtx compare, rtx eh_note, struct
comparison *cmp)
return true;
}
+/* Return the value-producing set in INSN, if it has one of the forms that
+ compare-elim knows how to merge with a later flags set. */
+
+static rtx
+comparison_producer_set (rtx_insn *insn)
+{
+ rtx pat = PATTERN (insn);
+
+ if (GET_CODE (pat) == SET)
+ return pat;
+
+ if (GET_CODE (pat) == PARALLEL && XVECLEN (pat, 0) == 2)
+ {
+ rtx set = XVECEXP (pat, 0, 0);
+ if (GET_CODE (set) == SET)
+ return set;
+ }
+
+ return NULL_RTX;
+}
+
/* Identify comparison instructions within BB. If the flags from the last
compare in the BB is live at the end of the block, install the compare
in BB->AUX. Called via dom_walker.walk (). */
@@ -445,6 +466,17 @@ find_comparison_dom_walker::before_dom_children
(basic_block bb)
last_cmp->prev_clobber = last_clobber;
last_cmp->flags_src = src;
last_cmp->orig_mode = GET_MODE (src);
+ if (GET_CODE (src) == UNSPEC && XVECLEN (src, 0) > 0)
+ {
+ rtx tested_value = XVECEXP (src, 0, XVECLEN (src, 0) - 1);
+ if (REG_P (tested_value) && last_setter[REGNO (tested_value)])
+ {
+ rtx_insn *last_insn = last_setter[REGNO (tested_value)];
+ rtx set = comparison_producer_set (last_insn);
+ if (set && rtx_equal_p (SET_DEST (set), tested_value))
+ last_cmp->prev_clobber = last_insn;
+ }
+ }
all_compares.safe_push (last_cmp);
last_clobber = NULL;
@@ -718,6 +750,99 @@ try_validate_parallel (rtx set_a, rtx set_b)
return par;
}
+/* Finish eliminating CMP by replacing INSN with a parallel containing
+ SET_A and SET_B. */
+
+static bool
+finish_eliminate_compare (struct comparison *cmp, rtx_insn *insn,
+ rtx set_a, rtx set_b)
+{
+ rtvec v = rtvec_alloc (2);
+ RTVEC_ELT (v, 0) = set_a;
+ RTVEC_ELT (v, 1) = set_b;
+
+ rtx pat = gen_rtx_PARALLEL (VOIDmode, v);
+
+ /* Succeed if the new instruction is valid. Note that we may have started
+ a change group within maybe_select_cc_mode, therefore we must continue.
*/
+ validate_change (insn, &PATTERN (insn), pat, true);
+
+ if (!apply_change_group ())
+ return false;
+
+ /* Success. Delete the compare insn... */
+ delete_insn (cmp->insn);
+
+ /* ... and any notes that are now invalid due to multiple sets. */
+ rtx x = find_regno_note (insn, REG_UNUSED, targetm.flags_regnum);
+ if (x)
+ remove_note (insn, x);
+ x = find_reg_note (insn, REG_EQUAL, NULL);
+ if (x)
+ remove_note (insn, x);
+ x = find_reg_note (insn, REG_EQUIV, NULL);
+ if (x)
+ remove_note (insn, x);
+
+ return true;
+}
+
+/* Try to eliminate the flags-setting instruction CMP->INSN by using
+ FLAGS_SRC as the source of the flags set. FLAGS_SET is the original
+ set of the flags register and VALUE_SET is the value-producing set in
+ INSN. */
+
+static bool
+try_eliminate_compare_with_flags_src (struct comparison *cmp, rtx_insn *insn,
+ rtx flags_set, rtx value_set,
+ rtx flags_src)
+{
+ if (GET_CODE (flags_src) != UNSPEC)
+ return false;
+
+ int len = XVECLEN (flags_src, 0);
+ if (len == 0
+ || !rtx_equal_p (XVECEXP (flags_src, 0, len - 1),
+ SET_DEST (value_set)))
+ return false;
+
+ /* Preserve the target-specific operands that describe how FLAGS_SRC tests
+ its final operand. Only replace that final operand, then let recog
+ decide whether the combined form is valid. */
+ rtvec vec = rtvec_alloc (len);
+ for (int i = 0; i < len; ++i)
+ RTVEC_ELT (vec, i) = copy_rtx (XVECEXP (flags_src, 0, i));
+
+ RTVEC_ELT (vec, len - 1) = copy_rtx (SET_SRC (value_set));
+
+ rtx tested_value = RTVEC_ELT (vec, len - 1);
+ /* The recorded users only compare the flags register against zero.
+ Ask the target which CC mode, if any, represents those uses when
+ applied to the value tested by FLAGS_SRC. */
+ rtx flags = maybe_select_cc_mode (cmp, tested_value,
+ CONST0_RTX (GET_MODE (tested_value)));
+ if (!flags)
+ flags = copy_rtx (SET_DEST (flags_set));
+
+ rtx new_flags_src = gen_rtx_UNSPEC (GET_MODE (flags), vec,
+ XINT (flags_src, 1));
+ if (modified_between_p (new_flags_src, insn, cmp->insn))
+ {
+ cancel_changes (0);
+ return false;
+ }
+
+ rtx x = copy_rtx (value_set);
+ rtx y = gen_rtx_SET (flags, new_flags_src);
+ if (try_validate_parallel (y, x))
+ return finish_eliminate_compare (cmp, insn, y, x);
+ if (try_validate_parallel (x, y))
+ return finish_eliminate_compare (cmp, insn, x, y);
+
+ cancel_changes (0);
+ return false;
+}
+
/* For a comparison instruction described by CMP check if it compares a
register with zero i.e. it is of the form CC := CMP R1, 0.
If it is, find the instruction defining R1 (say I1) and try to create a
@@ -806,13 +931,11 @@ try_eliminate_compare (struct comparison *cmp)
if (!cmp->flags_src && try_merge_compare (cmp))
return true;
- /* We must have found an interesting "clobber" preceding the compare. */
+ /* We must have found an interesting producer preceding the compare. */
if (cmp->prev_clobber == NULL)
return false;
rtx_insn *insn = cmp->prev_clobber;
- rtx x;
- rtx y;
if (cmp->flags_src)
{
@@ -823,22 +946,25 @@ try_eliminate_compare (struct comparison *cmp)
rtx flags_set = simple_regno_set (PATTERN (cmp->insn),
targetm.flags_regnum);
- rtx clobber_pat = PATTERN (insn);
- if (!flags_set
- || GET_CODE (clobber_pat) != PARALLEL
- || XVECLEN (clobber_pat, 0) != 2)
+ rtx value_set = comparison_producer_set (insn);
+ if (!flags_set || !value_set)
return false;
- rtx value_set = XVECEXP (clobber_pat, 0, 0);
- if (GET_CODE (value_set) != SET
- || side_effects_p (SET_SRC (value_set))
+ if (side_effects_p (SET_SRC (value_set))
|| side_effects_p (SET_SRC (flags_set))
- || modified_between_p (SET_SRC (flags_set), insn, cmp->insn))
+ || modified_between_p (SET_SRC (value_set), insn, cmp->insn))
return false;
- x = copy_rtx (value_set);
- y = copy_rtx (flags_set);
- goto finalize;
+ if (!modified_between_p (SET_SRC (flags_set), insn, cmp->insn)
+ && finish_eliminate_compare (cmp, insn, copy_rtx (flags_set),
+ copy_rtx (value_set)))
+ return true;
+
+ if (try_eliminate_compare_with_flags_src (cmp, insn, flags_set,
+ value_set, cmp->flags_src))
+ return true;
+
+ return false;
}
/* Verify that IN_A is not clobbered in between CMP and PREV_CLOBBER.
@@ -881,7 +1007,7 @@ try_eliminate_compare (struct comparison *cmp)
Validate that PREV_CLOBBER itself does in fact refer to IN_A. Do
recall that we've already validated the shape of PREV_CLOBBER. */
- x = XVECEXP (PATTERN (insn), 0, 0);
+ rtx x = XVECEXP (PATTERN (insn), 0, 0);
if (rtx_equal_p (SET_DEST (x), in_a))
cmp_a = SET_SRC (x);
@@ -934,45 +1060,16 @@ try_eliminate_compare (struct comparison *cmp)
flags = gen_rtx_REG (cmp->orig_mode, targetm.flags_regnum);
/* Generate a new comparison for installation in the setter. */
- y = cmp->not_in_a
- ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
- : copy_rtx (cmp_a);
+ rtx y = cmp->not_in_a
+ ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
+ : copy_rtx (cmp_a);
y = gen_rtx_COMPARE (GET_MODE (flags), y, copy_rtx (cmp_b));
y = gen_rtx_SET (flags, y);
- finalize:
/* Canonicalize instruction to:
[(set (reg:CCM) (compare:CCM (operation) (immediate)))
(set (reg) (operation)] */
-
- rtvec v = rtvec_alloc (2);
- RTVEC_ELT (v, 0) = y;
- RTVEC_ELT (v, 1) = x;
-
- rtx pat = gen_rtx_PARALLEL (VOIDmode, v);
-
- /* Succeed if the new instruction is valid. Note that we may have started
- a change group within maybe_select_cc_mode, therefore we must continue. */
- validate_change (insn, &PATTERN (insn), pat, true);
-
- if (!apply_change_group ())
- return false;
-
- /* Success. Delete the compare insn... */
- delete_insn (cmp->insn);
-
- /* ... and any notes that are now invalid due to multiple sets. */
- x = find_regno_note (insn, REG_UNUSED, targetm.flags_regnum);
- if (x)
- remove_note (insn, x);
- x = find_reg_note (insn, REG_EQUAL, NULL);
- if (x)
- remove_note (insn, x);
- x = find_reg_note (insn, REG_EQUIV, NULL);
- if (x)
- remove_note (insn, x);
-
- return true;
+ return finish_eliminate_compare (cmp, insn, y, x);
}
/* Main entry point to the pass. */
--
diff --git a/gcc/compare-elim.cc b/gcc/compare-elim.cc
index 0f10f6874490f8c3c08949a8a7512eb6ffc287db..1855604cd2c39c6e2afebb0e6c1bd74a2c4280ef 100644
--- a/gcc/compare-elim.cc
+++ b/gcc/compare-elim.cc
@@ -359,6 +359,27 @@ can_eliminate_compare (rtx compare, rtx eh_note, struct comparison *cmp)
return true;
}
+/* Return the value-producing set in INSN, if it has one of the forms that
+ compare-elim knows how to merge with a later flags set. */
+
+static rtx
+comparison_producer_set (rtx_insn *insn)
+{
+ rtx pat = PATTERN (insn);
+
+ if (GET_CODE (pat) == SET)
+ return pat;
+
+ if (GET_CODE (pat) == PARALLEL && XVECLEN (pat, 0) == 2)
+ {
+ rtx set = XVECEXP (pat, 0, 0);
+ if (GET_CODE (set) == SET)
+ return set;
+ }
+
+ return NULL_RTX;
+}
+
/* Identify comparison instructions within BB. If the flags from the last
compare in the BB is live at the end of the block, install the compare
in BB->AUX. Called via dom_walker.walk (). */
@@ -445,6 +466,17 @@ find_comparison_dom_walker::before_dom_children (basic_block bb)
last_cmp->prev_clobber = last_clobber;
last_cmp->flags_src = src;
last_cmp->orig_mode = GET_MODE (src);
+ if (GET_CODE (src) == UNSPEC && XVECLEN (src, 0) > 0)
+ {
+ rtx tested_value = XVECEXP (src, 0, XVECLEN (src, 0) - 1);
+ if (REG_P (tested_value) && last_setter[REGNO (tested_value)])
+ {
+ rtx_insn *last_insn = last_setter[REGNO (tested_value)];
+ rtx set = comparison_producer_set (last_insn);
+ if (set && rtx_equal_p (SET_DEST (set), tested_value))
+ last_cmp->prev_clobber = last_insn;
+ }
+ }
all_compares.safe_push (last_cmp);
last_clobber = NULL;
@@ -718,6 +750,99 @@ try_validate_parallel (rtx set_a, rtx set_b)
return par;
}
+/* Finish eliminating CMP by replacing INSN with a parallel containing
+ SET_A and SET_B. */
+
+static bool
+finish_eliminate_compare (struct comparison *cmp, rtx_insn *insn,
+ rtx set_a, rtx set_b)
+{
+ rtvec v = rtvec_alloc (2);
+ RTVEC_ELT (v, 0) = set_a;
+ RTVEC_ELT (v, 1) = set_b;
+
+ rtx pat = gen_rtx_PARALLEL (VOIDmode, v);
+
+ /* Succeed if the new instruction is valid. Note that we may have started
+ a change group within maybe_select_cc_mode, therefore we must continue. */
+ validate_change (insn, &PATTERN (insn), pat, true);
+
+ if (!apply_change_group ())
+ return false;
+
+ /* Success. Delete the compare insn... */
+ delete_insn (cmp->insn);
+
+ /* ... and any notes that are now invalid due to multiple sets. */
+ rtx x = find_regno_note (insn, REG_UNUSED, targetm.flags_regnum);
+ if (x)
+ remove_note (insn, x);
+ x = find_reg_note (insn, REG_EQUAL, NULL);
+ if (x)
+ remove_note (insn, x);
+ x = find_reg_note (insn, REG_EQUIV, NULL);
+ if (x)
+ remove_note (insn, x);
+
+ return true;
+}
+
+/* Try to eliminate the flags-setting instruction CMP->INSN by using
+ FLAGS_SRC as the source of the flags set. FLAGS_SET is the original
+ set of the flags register and VALUE_SET is the value-producing set in
+ INSN. */
+
+static bool
+try_eliminate_compare_with_flags_src (struct comparison *cmp, rtx_insn *insn,
+ rtx flags_set, rtx value_set,
+ rtx flags_src)
+{
+ if (GET_CODE (flags_src) != UNSPEC)
+ return false;
+
+ int len = XVECLEN (flags_src, 0);
+ if (len == 0
+ || !rtx_equal_p (XVECEXP (flags_src, 0, len - 1),
+ SET_DEST (value_set)))
+ return false;
+
+ /* Preserve the target-specific operands that describe how FLAGS_SRC tests
+ its final operand. Only replace that final operand, then let recog
+ decide whether the combined form is valid. */
+ rtvec vec = rtvec_alloc (len);
+ for (int i = 0; i < len; ++i)
+ RTVEC_ELT (vec, i) = copy_rtx (XVECEXP (flags_src, 0, i));
+
+ RTVEC_ELT (vec, len - 1) = copy_rtx (SET_SRC (value_set));
+
+ rtx tested_value = RTVEC_ELT (vec, len - 1);
+ /* The recorded users only compare the flags register against zero.
+ Ask the target which CC mode, if any, represents those uses when
+ applied to the value tested by FLAGS_SRC. */
+ rtx flags = maybe_select_cc_mode (cmp, tested_value,
+ CONST0_RTX (GET_MODE (tested_value)));
+ if (!flags)
+ flags = copy_rtx (SET_DEST (flags_set));
+
+ rtx new_flags_src = gen_rtx_UNSPEC (GET_MODE (flags), vec,
+ XINT (flags_src, 1));
+ if (modified_between_p (new_flags_src, insn, cmp->insn))
+ {
+ cancel_changes (0);
+ return false;
+ }
+
+ rtx x = copy_rtx (value_set);
+ rtx y = gen_rtx_SET (flags, new_flags_src);
+ if (try_validate_parallel (y, x))
+ return finish_eliminate_compare (cmp, insn, y, x);
+ if (try_validate_parallel (x, y))
+ return finish_eliminate_compare (cmp, insn, x, y);
+
+ cancel_changes (0);
+ return false;
+}
+
/* For a comparison instruction described by CMP check if it compares a
register with zero i.e. it is of the form CC := CMP R1, 0.
If it is, find the instruction defining R1 (say I1) and try to create a
@@ -806,13 +931,11 @@ try_eliminate_compare (struct comparison *cmp)
if (!cmp->flags_src && try_merge_compare (cmp))
return true;
- /* We must have found an interesting "clobber" preceding the compare. */
+ /* We must have found an interesting producer preceding the compare. */
if (cmp->prev_clobber == NULL)
return false;
rtx_insn *insn = cmp->prev_clobber;
- rtx x;
- rtx y;
if (cmp->flags_src)
{
@@ -823,22 +946,25 @@ try_eliminate_compare (struct comparison *cmp)
rtx flags_set = simple_regno_set (PATTERN (cmp->insn),
targetm.flags_regnum);
- rtx clobber_pat = PATTERN (insn);
- if (!flags_set
- || GET_CODE (clobber_pat) != PARALLEL
- || XVECLEN (clobber_pat, 0) != 2)
+ rtx value_set = comparison_producer_set (insn);
+ if (!flags_set || !value_set)
return false;
- rtx value_set = XVECEXP (clobber_pat, 0, 0);
- if (GET_CODE (value_set) != SET
- || side_effects_p (SET_SRC (value_set))
+ if (side_effects_p (SET_SRC (value_set))
|| side_effects_p (SET_SRC (flags_set))
- || modified_between_p (SET_SRC (flags_set), insn, cmp->insn))
+ || modified_between_p (SET_SRC (value_set), insn, cmp->insn))
return false;
- x = copy_rtx (value_set);
- y = copy_rtx (flags_set);
- goto finalize;
+ if (!modified_between_p (SET_SRC (flags_set), insn, cmp->insn)
+ && finish_eliminate_compare (cmp, insn, copy_rtx (flags_set),
+ copy_rtx (value_set)))
+ return true;
+
+ if (try_eliminate_compare_with_flags_src (cmp, insn, flags_set,
+ value_set, cmp->flags_src))
+ return true;
+
+ return false;
}
/* Verify that IN_A is not clobbered in between CMP and PREV_CLOBBER.
@@ -881,7 +1007,7 @@ try_eliminate_compare (struct comparison *cmp)
Validate that PREV_CLOBBER itself does in fact refer to IN_A. Do
recall that we've already validated the shape of PREV_CLOBBER. */
- x = XVECEXP (PATTERN (insn), 0, 0);
+ rtx x = XVECEXP (PATTERN (insn), 0, 0);
if (rtx_equal_p (SET_DEST (x), in_a))
cmp_a = SET_SRC (x);
@@ -934,45 +1060,16 @@ try_eliminate_compare (struct comparison *cmp)
flags = gen_rtx_REG (cmp->orig_mode, targetm.flags_regnum);
/* Generate a new comparison for installation in the setter. */
- y = cmp->not_in_a
- ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
- : copy_rtx (cmp_a);
+ rtx y = cmp->not_in_a
+ ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
+ : copy_rtx (cmp_a);
y = gen_rtx_COMPARE (GET_MODE (flags), y, copy_rtx (cmp_b));
y = gen_rtx_SET (flags, y);
- finalize:
/* Canonicalize instruction to:
[(set (reg:CCM) (compare:CCM (operation) (immediate)))
(set (reg) (operation)] */
-
- rtvec v = rtvec_alloc (2);
- RTVEC_ELT (v, 0) = y;
- RTVEC_ELT (v, 1) = x;
-
- rtx pat = gen_rtx_PARALLEL (VOIDmode, v);
-
- /* Succeed if the new instruction is valid. Note that we may have started
- a change group within maybe_select_cc_mode, therefore we must continue. */
- validate_change (insn, &PATTERN (insn), pat, true);
-
- if (!apply_change_group ())
- return false;
-
- /* Success. Delete the compare insn... */
- delete_insn (cmp->insn);
-
- /* ... and any notes that are now invalid due to multiple sets. */
- x = find_regno_note (insn, REG_UNUSED, targetm.flags_regnum);
- if (x)
- remove_note (insn, x);
- x = find_reg_note (insn, REG_EQUAL, NULL);
- if (x)
- remove_note (insn, x);
- x = find_reg_note (insn, REG_EQUIV, NULL);
- if (x)
- remove_note (insn, x);
-
- return true;
+ return finish_eliminate_compare (cmp, insn, y, x);
}
/* Main entry point to the pass. */
