The example loop
#define N 512
#define START 1
#define END 505
int x[N] __attribute__((aligned(32)));
int __attribute__((noipa))
foo (void)
{
for (int *p = x + START; p < x + END; ++p)
if (*p == 0)
return START;
return -1;
}
shows that when we vectorize we generate the following pre-header when doing
peeling for alignment:
_22 = (unsigned long) &MEM <int[512]> [(void *)&x + 4B];
_23 = _22 & POLY_INT_CST [15, 16];
_24 = _23 >> 2;
_26 = 1 - _24;
_27 = _26 * 4;
vectp_x.6_25 = &x + _27;
_36 = 504 + _24;
max_mask_37 = .WHILE_ULT (0, _36, { 0, ... });
_38 = .WHILE_ULT (0, _24, { 0, ... });
_39 = ~_38;
_40 = max_mask_37 & _39;
notice how _22 and and the actual vector pointer vectp_x.6_25 start at the
same offset but have a different base. This happens because in
vect_create_addr_base_for_vector_ref we fold the offset into the base very
early and so we can't form or share the address computation anymore with the
scalar address which needs to be there for the alignment checks.
The patch delays this and forms an explicit base + offset and keeping base as
a separate value. This allows VN at the end of vect to share the computations
and we get
_22 = (unsigned long) &MEM <int[512]> [(void *)&x + 4B];
_23 = _22 & POLY_INT_CST [15, 16];
_24 = _23 >> 2;
_27 = _24 * 4;
_28 = -_27;
vectp_x.6_26 = &MEM <int[512]> [(void *)&x + 4B] + _28;
_37 = 504 + _24;
max_mask_38 = .WHILE_ULT (0, _37, { 0, ... });
_39 = .WHILE_ULT (0, _24, { 0, ... });
_40 = ~_39;
_41 = max_mask_38 & _40;
Notice how they now share the same base address. This drops two instructions
off the loop pre-header:
from
foo:
cntb x0
sub x3, x0, #1
cmp x0, 4096
and x0, x0, x3
ccmp x0, 0, 0, ls
bne .L7
adrp x4, .LANCHOR0
add x4, x4, :lo12:.LANCHOR0
add x1, x4, 4
mov w2, 1
and x1, x1, x3
mov w0, 0
cntw x3
lsr x1, x1, 2
whilelo p15.s, xzr, x1
sub x2, x2, x1
add x1, x1, 504
whilelo p7.s, xzr, x1
not p7.b, p7/z, p15.b
add x2, x4, x2, lsl 2
b .L4
to
foo:
cntb x0
sub x1, x0, #1
cmp x0, 4096
and x0, x0, x1
adrp x2, .LANCHOR0
ccmp x0, 0, 0, ls
add x2, x2, :lo12:.LANCHOR0
bne .L7
add x2, x2, 4
mov w0, 0
and x1, x2, x1
cntw x3
sub x2, x2, x1
lsr x1, x1, 2
whilelo p15.s, xzr, x1
add x1, x1, 504
whilelo p7.s, xzr, x1
not p7.b, p7/z, p15.b
b .L4
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
* tree-vect-data-refs.cc (vect_create_addr_base_for_vector_ref):
Restructure computations to force separate base.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/vect-early-break-cbranch_18.c: New test.
---
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
new file mode 100644
index
0000000000000000000000000000000000000000..9129f82cffe5e0028bbd6b7f0a9ae434388331e3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -msve-vector-bits=256 -mautovec-preference=sve-only
-fdump-tree-vect-details" } */
+
+#define N 512
+#define START 1
+#define END 505
+
+int x[N] __attribute__((aligned(32)));
+
+int __attribute__((noipa))
+foo (void)
+{
+ for (int *p = x + START; p < x + END; ++p)
+ if (*p == 0)
+ return START;
+
+ return -1;
+}
+
+/* { dg-final { scan-tree-dump "Alignment of access forced using peeling"
"vect" } } */
+/* { dg-final { scan-tree-dump-times {Value numbering stmt =
vectp_x\.[0-9]+_[0-9]+ = &x \+ 4;} 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times {Value numbering stmt =
vectp_x\.[0-9]+_[0-9]+ = vectp_x\.[0-9]+_[0-9]+ \+ 18446744073709551612;} 1
"vect" } } */
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index
ab850de024bf41ea7635ce7381bcc6acd8fe0c28..8c71865aa0b1c2db7f0b0167dc7ed1db6c6da260
100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -5729,7 +5729,13 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo,
stmt_vec_info stmt_info,
innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
tree data_ref_base = unshare_expr (drb->base_address);
- tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
+ tree vector_offset = NULL_TREE;
+ if (loop_vinfo && dr_info->offset)
+ vector_offset = unshare_expr (dr_info->offset);
+ tree base_offset = unshare_expr (vector_offset
+ ? drb->offset
+ : get_dr_vinfo_offset (vinfo, dr_info,
+ true));
tree init = unshare_expr (drb->init);
if (loop_vinfo)
@@ -5767,7 +5773,27 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo,
stmt_vec_info stmt_info,
vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
- addr_base = force_gimple_operand (addr_base, &seq, true, dest);
+
+ /* Keep vectorizer-added offsets separate from the original scalar access
+ address. Forming "base + scalar offset" first gives the target a better
+ chance of sharing it with other address calculations, such as the
+ misalignment check used for masked alignment peeling. */
+ if (vector_offset)
+ {
+ tree scalar_dest = vect_get_new_vect_var (vect_ptr_type,
+ vect_pointer_var, base_name);
+ gimple_seq addr_seq = NULL;
+ addr_base = force_gimple_operand (addr_base, &addr_seq, true,
+ scalar_dest);
+ gimple_seq_add_seq (&seq, addr_seq);
+ addr_base = fold_build_pointer_plus (addr_base,
+ fold_convert (sizetype,
+ vector_offset));
+ }
+
+ gimple_seq addr_seq = NULL;
+ addr_base = force_gimple_operand (addr_base, &addr_seq, true, dest);
+ gimple_seq_add_seq (&seq, addr_seq);
gimple_seq_add_seq (new_stmt_list, seq);
if (TREE_CODE (addr_base) == SSA_NAME
--
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
new file mode 100644
index 0000000000000000000000000000000000000000..9129f82cffe5e0028bbd6b7f0a9ae434388331e3
--- /dev/null
+++ b/gcc/testsuite/gcc.target/aarch64/sve/vect-early-break-cbranch_18.c
@@ -0,0 +1,22 @@
+/* { dg-do compile } */
+/* { dg-options "-Ofast -msve-vector-bits=256 -mautovec-preference=sve-only -fdump-tree-vect-details" } */
+
+#define N 512
+#define START 1
+#define END 505
+
+int x[N] __attribute__((aligned(32)));
+
+int __attribute__((noipa))
+foo (void)
+{
+ for (int *p = x + START; p < x + END; ++p)
+ if (*p == 0)
+ return START;
+
+ return -1;
+}
+
+/* { dg-final { scan-tree-dump "Alignment of access forced using peeling" "vect" } } */
+/* { dg-final { scan-tree-dump-times {Value numbering stmt = vectp_x\.[0-9]+_[0-9]+ = &x \+ 4;} 1 "vect" } } */
+/* { dg-final { scan-tree-dump-times {Value numbering stmt = vectp_x\.[0-9]+_[0-9]+ = vectp_x\.[0-9]+_[0-9]+ \+ 18446744073709551612;} 1 "vect" } } */
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index ab850de024bf41ea7635ce7381bcc6acd8fe0c28..8c71865aa0b1c2db7f0b0167dc7ed1db6c6da260 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -5729,7 +5729,13 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
tree data_ref_base = unshare_expr (drb->base_address);
- tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
+ tree vector_offset = NULL_TREE;
+ if (loop_vinfo && dr_info->offset)
+ vector_offset = unshare_expr (dr_info->offset);
+ tree base_offset = unshare_expr (vector_offset
+ ? drb->offset
+ : get_dr_vinfo_offset (vinfo, dr_info,
+ true));
tree init = unshare_expr (drb->init);
if (loop_vinfo)
@@ -5767,7 +5773,27 @@ vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
- addr_base = force_gimple_operand (addr_base, &seq, true, dest);
+
+ /* Keep vectorizer-added offsets separate from the original scalar access
+ address. Forming "base + scalar offset" first gives the target a better
+ chance of sharing it with other address calculations, such as the
+ misalignment check used for masked alignment peeling. */
+ if (vector_offset)
+ {
+ tree scalar_dest = vect_get_new_vect_var (vect_ptr_type,
+ vect_pointer_var, base_name);
+ gimple_seq addr_seq = NULL;
+ addr_base = force_gimple_operand (addr_base, &addr_seq, true,
+ scalar_dest);
+ gimple_seq_add_seq (&seq, addr_seq);
+ addr_base = fold_build_pointer_plus (addr_base,
+ fold_convert (sizetype,
+ vector_offset));
+ }
+
+ gimple_seq addr_seq = NULL;
+ addr_base = force_gimple_operand (addr_base, &addr_seq, true, dest);
+ gimple_seq_add_seq (&seq, addr_seq);
gimple_seq_add_seq (new_stmt_list, seq);
if (TREE_CODE (addr_base) == SSA_NAME
