Consider the following dot-product computation:
uint32_t
tcp_checksum(int n, uint8_t* data)
{
uint32_t sum = 0;
for (int i=0; i<n; i+=1)
sum += data[i] * data[i];
return sum;
}
At present, following vectorization and the subsequent optimization
passes, we will end up with the following GIMPLE code:
vect__1 = .MASK_LOAD (vectp_data, 8B, loop_mask_1, { 0, ... });
masked_op1_1 = VEC_COND_EXPR <loop_mask_1, vect__1, { 0, ... }>;
vect_patt_1 = DOT_PROD_EXPR <vect__1, masked_op1_1, vectt_sum_1>;
By looking at the input data going into the VEC_COND_EXPR, its mask
and its else value, we can walk back up the USE-DEF chain to see
whether the source of the input data shares the same mask and else
values. If so, we can safely remove the VEC_COND_EXPR statement from
the cfg, thus resulting in the more optimal variant:
vect__1 = .MASK_LOAD (vectp_data, 8B, loop_mask_1, { 0, ... });
vect_patt_1 = DOT_PROD_EXPR <vect__1, vect__1, vectt_sum_1>;
Regression testes on AArch64 & x86_64, no new regressions.
PR tree-optimization/111770
gcc/ChangeLog:
* gimple-isel.cc (stmt_is_any_masked_load_p): New.
(stmt_is_vec_cond_expr_zeroing): Likewise.
(redundant_condexpr_removal): Likewise.
(pass_gimple_isel::execute): Add call to
`redundant_condexpr_removal'.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-cond-dot.c: New.
---
gcc/gimple-isel.cc | 78 +++++++++++++++++++++++
gcc/testsuite/gcc.dg/vect/vect-cond-dot.c | 21 ++++++
2 files changed, 99 insertions(+)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-cond-dot.c
diff --git a/gcc/gimple-isel.cc b/gcc/gimple-isel.cc
index b193e27b183..aa020f402e0 100644
--- a/gcc/gimple-isel.cc
+++ b/gcc/gimple-isel.cc
@@ -1343,6 +1343,82 @@ gimple_isel_builtin_call (gcall *call,
gimple_stmt_iterator *gsi)
}
}
+static internal_fn
+stmt_is_any_masked_load_p (gimple *stmt)
+{
+ if (stmt && is_gimple_call (stmt) && gimple_call_internal_p (stmt))
+ {
+ internal_fn ifn = gimple_call_internal_fn (stmt);
+ if (internal_load_fn_p (gimple_call_internal_fn (stmt))
+ && internal_fn_mask_index (ifn) >= 0)
+ return ifn;
+ }
+ return IFN_LAST;
+}
+
+/* Ensure we have a statement of the form:
+ VEC_COND_EXPR <mask, then_val, { 0, ... }>
+ If the else value of STMT is a zero initializer, return the then value,
+ returning NULL_TREE otherwise. */
+static tree
+stmt_is_vec_cond_expr_zeroing (gimple *stmt)
+{
+ if (is_gimple_assign (stmt))
+ {
+ enum tree_code code = gimple_assign_rhs_code (stmt);
+ if (code == VEC_COND_EXPR)
+ {
+ tree then_val = gimple_assign_rhs2 (stmt);
+ tree else_val = gimple_assign_rhs3 (stmt);
+ if (initializer_zerop (else_val))
+ return then_val;
+ }
+ }
+ return NULL_TREE;
+}
+
+ /* Remove redundant VEC_COND_EXPR statments.
+ Consider the following set of statements:
+
+ vect__1 = .MASK_LOAD (vectp_data, 8B, loop_mask_1, { 0, ... });
+ masked_op1_1 = VEC_COND_EXPR <loop_mask_1, vect__1, { 0, ... }>;
+ vect_patt_1 = DOT_PROD_EXPR <vect__1, masked_op1_1 vect
+
+ By looking at the input data going into the VEC_COND_EXPR, we can walk
+ back up the USE-DEF chain to see whether the source of the input data
+ shares the same mask and else values. If so, we can safely remove the
+ VEC_COND_EXPR statement from the cfg. */
+static void
+redundant_condexpr_removal (gimple_stmt_iterator &gsi)
+{
+
+ gimple *vcond_stmt = *gsi;
+ tree then_val = stmt_is_vec_cond_expr_zeroing (vcond_stmt);
+ if (then_val && TREE_CODE (then_val) == SSA_NAME)
+ {
+ gimple *then_defn = SSA_NAME_DEF_STMT (then_val);
+ internal_fn then_ifn = stmt_is_any_masked_load_p (then_defn);
+ if (then_ifn != IFN_LAST)
+ {
+ tree vcond_mask, load_mask, load_else;
+ vcond_mask = gimple_assign_rhs1 (vcond_stmt);
+ load_mask = gimple_call_arg (then_defn,
+ internal_fn_mask_index (then_ifn));
+ load_else = gimple_call_arg (then_defn,
+ internal_fn_else_index (then_ifn));
+ if (vcond_mask == load_mask && initializer_zerop (load_else))
+ {
+ tree dest = gimple_assign_lhs (gsi_stmt (gsi));
+ replace_uses_by (dest, then_val);
+ gimple *def_stmt = gsi_stmt (gsi);
+ gsi_remove (&gsi, true);
+ release_defs (def_stmt);
+ }
+ }
+ }
+}
+
+
/* Iterate all gimple statements and perform pre RTL expansion
GIMPLE massaging to improve instruction selection. */
@@ -1360,6 +1436,8 @@ pass_gimple_isel::execute (struct function *fun)
/* Give the target first try at replacing the instruction. */
cfg_changed |= targetm.instruction_selection (fun, &gsi);
+ redundant_condexpr_removal (gsi);
+
/* Pre-expand VEC_COND_EXPRs to .VCOND* internal function
calls mapping to supported optabs. */
gimple *g = gimple_expand_vec_cond_expr (&gsi);
diff --git a/gcc/testsuite/gcc.dg/vect/vect-cond-dot.c
b/gcc/testsuite/gcc.dg/vect/vect-cond-dot.c
new file mode 100644
index 00000000000..7b6d920c9e5
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-cond-dot.c
@@ -0,0 +1,21 @@
+/* { dg-do compile } */
+/* { dg-additional-options "-fdump-tree-optimized" } */
+/* { dg-require-effective-target vect_masked_load } */
+#include <stdint.h>
+
+#define CHECK_DOT(IN, OUT) \
+OUT check_dot_##OUT(int n, IN* data) { \
+ OUT sum = 0; \
+ for (int i=0; i<n; i+=1) { \
+ sum += data[i] * data[i]; \
+ } \
+ return sum; \
+}
+
+CHECK_DOT (uint8_t, uint32_t);
+CHECK_DOT (int8_t, int32_t);
+CHECK_DOT (int16_t, int64_t);
+
+/* { dg-final { scan-tree-dump-times {vectorized 1 loops} 3 "vect" } } */
+/* { dg-final { scan-tree-dump-times {[\n\r]\s*(vect__[^ \t=]+) = \.MASK_LOAD
\([a-zA-Z0-9_.]+, [0-9]+B, (loop_mask_[0-9]+), \{ 0, \.\.\.
\}\);.*[\n\r]\s+masked_op[^ \t=]+ = VEC_COND_EXPR <\2, \1, \{ 0, \.\.\. \}} 9
"vect" { target aarch64*-*-* } } } */
+/* { dg-final { scan-tree-dump-not {VEC_COND_EXPR} "optimized" } } */
--
2.43.0
