This patch extends compare-elim so it can recognise target-specific
flag-setting operations, not only explicit COMPARE rtxes.
AArch64 SVE expansion can produce two equivalent vector predicate comparisons
for one flag consuming instruction: one comparison produces the predicate value
that is needed later, while another comparison exists only to produce the
condition flags for the branch. The flags-producing operation is represented
using target-specific RTL rather than a plain COMPARE, so the old compare-elim
logic did not see the duplication.
This happens because SVE compares are used 3 ways for the same instruction:
1. Only the flags are interesting (used for control flow)
2. Only the result are interesting (normal vectorized data compares)
3. Both are interesting (for instance early break).
With the changes to early break we now produce 1 and 2 more readily, for
instance
The example:
int any_gt (signed char *a, signed char *b, int n)
{
for (int i = 0; i < n; ++i)
if (a[i] > b[i])
return i;
return 0;
}
compiled with -O3 -march=armv8-a+sve
With cmpelim disabled, the SVE loop body contains two identical compares: one
for the live predicate value and one for the branch flags:
ld1b z30.b, p7/z, [x0, x3]
ld1b z31.b, p7/z, [x1, x3]
cmpgt p15.b, p7/z, z30.b, z31.b
cmpgt p7.b, p7/z, z30.b, z31.b
b.none .L5
With this patch, the duplicate compare is removed and the branch uses the
flags from the remaining SVE compare:
ld1b z30.b, p7/z, [x0, x3]
ld1b z31.b, p7/z, [x1, x3]
cmpgt p15.b, p7/z, z30.b, z31.b
b.none .L5
The patch teaches compare-elim to record non-COMPARE flag setters by looking
at the target flags_regnum. Once the instruction has been identified the
subsequent flows are the same.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
-O3, profiled, profiled -O3 and -O3 -LTO bootstrap tested and no issues.
Also SPECCPU 2017 and SPECCPU 2026 tested.
OK for master?
Thanks,
Tamar
gcc/ChangeLog:
* compare-elim.cc (struct comparison): Track non-COMPARE flag setters.
(conforming_flags_set): Accept matching clobbers from flag setters.
(find_comparison_dom_walker::before_dom_children): Record them.
(try_validate_parallel): Validate merged flag-setting patterns.
(try_eliminate_compare): Eliminate redundant compares after recog.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c: New test.
* gcc.target/aarch64/sve/cmpelim-sve-ptest.c: New test.
---
diff --git a/gcc/compare-elim.cc b/gcc/compare-elim.cc
index
6218324e1ab6c1e3358c2eb0ad351bbe5e9dc574..0f10f6874490f8c3c08949a8a7512eb6ffc287db
100644
--- a/gcc/compare-elim.cc
+++ b/gcc/compare-elim.cc
@@ -102,6 +102,9 @@ struct comparison
constants. */
rtx in_a, in_b;
+ /* The source of a non-COMPARE set of the flags register. */
+ rtx flags_src;
+
/* The REG_EH_REGION of the comparison. */
rtx eh_note;
@@ -221,6 +224,31 @@ arithmetic_flags_clobber_p (rtx_insn *insn)
return false;
}
+/* Look for a non-COMPARE set of the flags register. This is used for
+ target-specific flag-setting operations that can be paired with a
+ previous flags clobber by recog. */
+
+static rtx
+conforming_flags_set (rtx_insn *insn)
+{
+ rtx set = single_set (insn);
+ if (set == NULL)
+ return NULL_RTX;
+
+ rtx dest = SET_DEST (set);
+ if (!REG_P (dest) || REGNO (dest) != targetm.flags_regnum)
+ return NULL_RTX;
+
+ rtx src = SET_SRC (set);
+ if (GET_CODE (src) == COMPARE)
+ return NULL_RTX;
+
+ if (reg_referenced_p (dest, src))
+ return NULL_RTX;
+
+ return src;
+}
+
/* Look for uses of FLAGS in INSN. If we find one we can analyze, record
it in CMP; otherwise indicate that we've missed a use. */
@@ -410,6 +438,18 @@ find_comparison_dom_walker::before_dom_children
(basic_block bb)
last_clobber = NULL;
last_cmp_valid = true;
}
+ else if ((src = conforming_flags_set (insn)))
+ {
+ last_cmp = XCNEW (struct comparison);
+ last_cmp->insn = insn;
+ last_cmp->prev_clobber = last_clobber;
+ last_cmp->flags_src = src;
+ last_cmp->orig_mode = GET_MODE (src);
+ all_compares.safe_push (last_cmp);
+
+ last_clobber = NULL;
+ last_cmp_valid = false;
+ }
else
{
@@ -675,7 +715,7 @@ try_validate_parallel (rtx set_a, rtx set_b)
SET_PREV_INSN (insn) = NULL_RTX;
SET_NEXT_INSN (insn) = NULL_RTX;
INSN_LOCATION (insn) = 0;
- return insn;
+ return par;
}
/* For a comparison instruction described by CMP check if it compares a
@@ -763,13 +803,44 @@ try_eliminate_compare (struct comparison *cmp)
{
rtx flags, in_a, in_b, cmp_a, cmp_b;
- if (try_merge_compare (cmp))
+ if (!cmp->flags_src && try_merge_compare (cmp))
return true;
/* We must have found an interesting "clobber" preceding the compare. */
if (cmp->prev_clobber == NULL)
return false;
+ rtx_insn *insn = cmp->prev_clobber;
+ rtx x;
+ rtx y;
+
+ if (cmp->flags_src)
+ {
+ if (cmp->missing_uses
+ || cmp->n_uses == 0
+ || !can_merge_compare_into_arith (cmp->insn, insn))
+ return false;
+
+ rtx flags_set = simple_regno_set (PATTERN (cmp->insn),
+ targetm.flags_regnum);
+ rtx clobber_pat = PATTERN (insn);
+ if (!flags_set
+ || GET_CODE (clobber_pat) != PARALLEL
+ || XVECLEN (clobber_pat, 0) != 2)
+ return false;
+
+ rtx value_set = XVECEXP (clobber_pat, 0, 0);
+ if (GET_CODE (value_set) != SET
+ || side_effects_p (SET_SRC (value_set))
+ || side_effects_p (SET_SRC (flags_set))
+ || modified_between_p (SET_SRC (flags_set), insn, cmp->insn))
+ return false;
+
+ x = copy_rtx (value_set);
+ y = copy_rtx (flags_set);
+ goto finalize;
+ }
+
/* Verify that IN_A is not clobbered in between CMP and PREV_CLOBBER.
Given that this target requires this pass, we can assume that most
insns do clobber the flags, and so the distance between the compare
@@ -809,9 +880,8 @@ try_eliminate_compare (struct comparison *cmp)
/* We've reached PREV_CLOBBER without finding a modification of IN_A.
Validate that PREV_CLOBBER itself does in fact refer to IN_A. Do
recall that we've already validated the shape of PREV_CLOBBER. */
- rtx_insn *insn = cmp->prev_clobber;
- rtx x = XVECEXP (PATTERN (insn), 0, 0);
+ x = XVECEXP (PATTERN (insn), 0, 0);
if (rtx_equal_p (SET_DEST (x), in_a))
cmp_a = SET_SRC (x);
@@ -864,12 +934,13 @@ try_eliminate_compare (struct comparison *cmp)
flags = gen_rtx_REG (cmp->orig_mode, targetm.flags_regnum);
/* Generate a new comparison for installation in the setter. */
- rtx y = cmp->not_in_a
- ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
- : copy_rtx (cmp_a);
+ y = cmp->not_in_a
+ ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
+ : copy_rtx (cmp_a);
y = gen_rtx_COMPARE (GET_MODE (flags), y, copy_rtx (cmp_b));
y = gen_rtx_SET (flags, y);
+ finalize:
/* Canonicalize instruction to:
[(set (reg:CCM) (compare:CCM (operation) (immediate)))
(set (reg) (operation)] */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c
b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c
new file mode 100644
index
0000000000000000000000000000000000000000..8e590d6171a3be4d4d32deb7b657d64f293983bb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O3 -march=armv8-a+sve -mautovec-preference=sve-only --param
vect-epilogues-nomask=0 -fdisable-rtl-cmpelim" } */
+
+int
+any_gt1 (signed char *restrict a, signed char *restrict b, int n)
+{
+ for (int i = 0; i < n; ++i)
+ if (a[i] > b[i])
+ return i;
+ return 0;
+}
+
+/* { dg-prune-output "disable pass rtl-cmpelim" } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-9]+\.b, p[0-9]+/z,
z[0-9]+\.b, z[0-9]+\.b} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest.c
b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest.c
new file mode 100644
index
0000000000000000000000000000000000000000..79263044fb7bf255eea4765aa5315e597641734a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest.c
@@ -0,0 +1,14 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O3 -march=armv8-a+sve -mautovec-preference=sve-only --param
vect-epilogues-nomask=0" } */
+
+int
+any_gt1 (signed char *restrict a, signed char *restrict b, int n)
+{
+ for (int i = 0; i < n; ++i)
+ if (a[i] > b[i])
+ return i;
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-9]+\.b, p[0-9]+/z,
z[0-9]+\.b, z[0-9]+\.b} 1 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
--
diff --git a/gcc/compare-elim.cc b/gcc/compare-elim.cc
index 6218324e1ab6c1e3358c2eb0ad351bbe5e9dc574..0f10f6874490f8c3c08949a8a7512eb6ffc287db 100644
--- a/gcc/compare-elim.cc
+++ b/gcc/compare-elim.cc
@@ -102,6 +102,9 @@ struct comparison
constants. */
rtx in_a, in_b;
+ /* The source of a non-COMPARE set of the flags register. */
+ rtx flags_src;
+
/* The REG_EH_REGION of the comparison. */
rtx eh_note;
@@ -221,6 +224,31 @@ arithmetic_flags_clobber_p (rtx_insn *insn)
return false;
}
+/* Look for a non-COMPARE set of the flags register. This is used for
+ target-specific flag-setting operations that can be paired with a
+ previous flags clobber by recog. */
+
+static rtx
+conforming_flags_set (rtx_insn *insn)
+{
+ rtx set = single_set (insn);
+ if (set == NULL)
+ return NULL_RTX;
+
+ rtx dest = SET_DEST (set);
+ if (!REG_P (dest) || REGNO (dest) != targetm.flags_regnum)
+ return NULL_RTX;
+
+ rtx src = SET_SRC (set);
+ if (GET_CODE (src) == COMPARE)
+ return NULL_RTX;
+
+ if (reg_referenced_p (dest, src))
+ return NULL_RTX;
+
+ return src;
+}
+
/* Look for uses of FLAGS in INSN. If we find one we can analyze, record
it in CMP; otherwise indicate that we've missed a use. */
@@ -410,6 +438,18 @@ find_comparison_dom_walker::before_dom_children (basic_block bb)
last_clobber = NULL;
last_cmp_valid = true;
}
+ else if ((src = conforming_flags_set (insn)))
+ {
+ last_cmp = XCNEW (struct comparison);
+ last_cmp->insn = insn;
+ last_cmp->prev_clobber = last_clobber;
+ last_cmp->flags_src = src;
+ last_cmp->orig_mode = GET_MODE (src);
+ all_compares.safe_push (last_cmp);
+
+ last_clobber = NULL;
+ last_cmp_valid = false;
+ }
else
{
@@ -675,7 +715,7 @@ try_validate_parallel (rtx set_a, rtx set_b)
SET_PREV_INSN (insn) = NULL_RTX;
SET_NEXT_INSN (insn) = NULL_RTX;
INSN_LOCATION (insn) = 0;
- return insn;
+ return par;
}
/* For a comparison instruction described by CMP check if it compares a
@@ -763,13 +803,44 @@ try_eliminate_compare (struct comparison *cmp)
{
rtx flags, in_a, in_b, cmp_a, cmp_b;
- if (try_merge_compare (cmp))
+ if (!cmp->flags_src && try_merge_compare (cmp))
return true;
/* We must have found an interesting "clobber" preceding the compare. */
if (cmp->prev_clobber == NULL)
return false;
+ rtx_insn *insn = cmp->prev_clobber;
+ rtx x;
+ rtx y;
+
+ if (cmp->flags_src)
+ {
+ if (cmp->missing_uses
+ || cmp->n_uses == 0
+ || !can_merge_compare_into_arith (cmp->insn, insn))
+ return false;
+
+ rtx flags_set = simple_regno_set (PATTERN (cmp->insn),
+ targetm.flags_regnum);
+ rtx clobber_pat = PATTERN (insn);
+ if (!flags_set
+ || GET_CODE (clobber_pat) != PARALLEL
+ || XVECLEN (clobber_pat, 0) != 2)
+ return false;
+
+ rtx value_set = XVECEXP (clobber_pat, 0, 0);
+ if (GET_CODE (value_set) != SET
+ || side_effects_p (SET_SRC (value_set))
+ || side_effects_p (SET_SRC (flags_set))
+ || modified_between_p (SET_SRC (flags_set), insn, cmp->insn))
+ return false;
+
+ x = copy_rtx (value_set);
+ y = copy_rtx (flags_set);
+ goto finalize;
+ }
+
/* Verify that IN_A is not clobbered in between CMP and PREV_CLOBBER.
Given that this target requires this pass, we can assume that most
insns do clobber the flags, and so the distance between the compare
@@ -809,9 +880,8 @@ try_eliminate_compare (struct comparison *cmp)
/* We've reached PREV_CLOBBER without finding a modification of IN_A.
Validate that PREV_CLOBBER itself does in fact refer to IN_A. Do
recall that we've already validated the shape of PREV_CLOBBER. */
- rtx_insn *insn = cmp->prev_clobber;
- rtx x = XVECEXP (PATTERN (insn), 0, 0);
+ x = XVECEXP (PATTERN (insn), 0, 0);
if (rtx_equal_p (SET_DEST (x), in_a))
cmp_a = SET_SRC (x);
@@ -864,12 +934,13 @@ try_eliminate_compare (struct comparison *cmp)
flags = gen_rtx_REG (cmp->orig_mode, targetm.flags_regnum);
/* Generate a new comparison for installation in the setter. */
- rtx y = cmp->not_in_a
- ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
- : copy_rtx (cmp_a);
+ y = cmp->not_in_a
+ ? gen_rtx_NOT (GET_MODE (cmp_a), copy_rtx (cmp_a))
+ : copy_rtx (cmp_a);
y = gen_rtx_COMPARE (GET_MODE (flags), y, copy_rtx (cmp_b));
y = gen_rtx_SET (flags, y);
+ finalize:
/* Canonicalize instruction to:
[(set (reg:CCM) (compare:CCM (operation) (immediate)))
(set (reg) (operation)] */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c
new file mode 100644
index 0000000000000000000000000000000000000000..8e590d6171a3be4d4d32deb7b657d64f293983bb
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest-2.c
@@ -0,0 +1,14 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O3 -march=armv8-a+sve -mautovec-preference=sve-only --param vect-epilogues-nomask=0 -fdisable-rtl-cmpelim" } */
+
+int
+any_gt1 (signed char *restrict a, signed char *restrict b, int n)
+{
+ for (int i = 0; i < n; ++i)
+ if (a[i] > b[i])
+ return i;
+ return 0;
+}
+
+/* { dg-prune-output "disable pass rtl-cmpelim" } */
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-9]+\.b, p[0-9]+/z, z[0-9]+\.b, z[0-9]+\.b} 2 } } */
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest.c b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest.c
new file mode 100644
index 0000000000000000000000000000000000000000..79263044fb7bf255eea4765aa5315e597641734a
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/cmpelim-sve-ptest.c
@@ -0,0 +1,14 @@
+/* { dg-do compile { target aarch64_asm_sve_ok } } */
+/* { dg-options "-O3 -march=armv8-a+sve -mautovec-preference=sve-only --param vect-epilogues-nomask=0" } */
+
+int
+any_gt1 (signed char *restrict a, signed char *restrict b, int n)
+{
+ for (int i = 0; i < n; ++i)
+ if (a[i] > b[i])
+ return i;
+ return 0;
+}
+
+/* { dg-final { scan-assembler-times {\tcmpgt\tp[0-9]+\.b, p[0-9]+/z, z[0-9]+\.b, z[0-9]+\.b} 1 } } */
+/* { dg-final { scan-assembler-not {\tptest\t} } } */
