From: Ezra Sitorus <[email protected]>
Currently, vect_recog_dot_prod_pattern can only match sums.
This patch looks for:
DPROD = X w* Y; #widen mult
sum_1 = sum_0 - DPROD; #subtraction
and replaces it with:
sum_1 = 0 - sum_0;
sum_2 = DOT_PROD<X, Y, sum_1>;
sum_3 = 0 - sum2;
It searches for a similar widening pattern but with subtraction instead.
In particular, for loops such as:
int acc, *a, *b, *c, *d;
for (int i = 0; i < N; i++)
acc += a[i] * b[i] - c[i] * d[i];
This would ideally get transformed into something like this:
acc_1 = DOT_PROD<c[i], d[i], acc_0>
acc_2 = 0 - acc_2;
acc_3 = DOT_PROD<a[i], b[i], acc_2>
acc_4 = 0 - acc_3;
A possible patch could be done later on to combine
vect_recog_dot_prod_pattern and vect_recog_neg_dot_prod_pattern.
However, since they result in different gimple emission then perhaps
it's not as trivial as it seems.
Note that there is another match.pd patch to aid in this as well.
gcc/ChangeLog:
* tree-vect-patterns.cc (vect_recog_neg_dot_prod_pattern): Add
new vect pattern.
(vect_vect_recog_func_ptrs): Add vect_recog_neg_dot_prod_pattern.
gcc/testsuite/ChangeLog:
* gcc.dg/vect/vect-reduc-neg-dot-1.c: Add new test.
* gcc.dg/vect/vect-reduc-neg-dot-2.c: Likewise.
---
.../gcc.dg/vect/vect-reduc-neg-dot-1.c | 46 +++++++
.../gcc.dg/vect/vect-reduc-neg-dot-2.c | 45 +++++++
gcc/tree-vect-patterns.cc | 124 ++++++++++++++++++
3 files changed, 215 insertions(+)
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-1.c
create mode 100644 gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-2.c
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-1.c
b/gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-1.c
new file mode 100644
index 000000000000..17a2e1513294
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-1.c
@@ -0,0 +1,46 @@
+/* { dg-require-effective-target vect_int } */
+
+#include "tree-vect.h"
+
+#define N 50
+
+#ifndef SIGNEDNESS_1
+#define SIGNEDNESS_1 signed
+#define SIGNEDNESS_2 signed
+#define SIGNEDNESS_3 signed
+#endif
+
+SIGNEDNESS_1 int __attribute__ ((noipa))
+f (SIGNEDNESS_1 short res, SIGNEDNESS_3 char *restrict a,
+ SIGNEDNESS_3 char *restrict b)
+{
+ for (__INTPTR_TYPE__ i = 0; i < N; ++i)
+ {
+ res -= (SIGNEDNESS_2 short) (a[i] * b[i]);
+ }
+ return res;
+}
+
+#define BASE ((SIGNEDNESS_3 short) -1 < 0 ? -126 : 4)
+#define OFFSET 20
+
+int
+main (void)
+{
+ check_vect ();
+
+ SIGNEDNESS_3 char a[N], b[N];
+ SIGNEDNESS_1 short expected = 0x1234;
+ for (int i = 0; i < N; ++i)
+ {
+ a[i] = BASE + i * 5;
+ b[i] = BASE + OFFSET + i * 4;
+ asm volatile ("" ::: "memory");
+ expected -= (SIGNEDNESS_2 short) (a[i] * b[i]);
+ }
+ if (f (0x1234, a, b) != expected)
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump "vect_recog_neg_dot_prod_pattern: detected"
"vect" } } */
+/* { dg-final { scan-tree-dump-times "vectorized 1 loop" 1 "vect" { target
vect_sdot_qi } } } */
diff --git a/gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-2.c
b/gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-2.c
new file mode 100644
index 000000000000..e6250d455284
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/vect/vect-reduc-neg-dot-2.c
@@ -0,0 +1,45 @@
+/* { dg-require-effective-target vect_int } */
+
+#include "tree-vect.h"
+
+#define N 50
+
+#ifndef SIGNEDNESS_1
+#define SIGNEDNESS_1 signed
+#define SIGNEDNESS_2 signed
+#define SIGNEDNESS_3 signed
+#endif
+
+SIGNEDNESS_1 char __attribute__ ((noipa))
+f (SIGNEDNESS_1 char res, SIGNEDNESS_3 char *restrict a,
+ SIGNEDNESS_3 char *restrict b)
+{
+ for (__INTPTR_TYPE__ i = 0; i < N; ++i)
+ {
+ res -= (SIGNEDNESS_2 char) (a[i] * b[i]);
+ }
+ return res;
+}
+
+#define BASE ((SIGNEDNESS_3 char) -1 < 0 ? -126 : 4)
+#define OFFSET 20
+
+int
+main (void)
+{
+ check_vect ();
+
+ SIGNEDNESS_3 char a[N], b[N];
+ SIGNEDNESS_1 char expected = 0x12;
+ for (char i = 0; i < N; ++i)
+ {
+ a[i] = BASE + i * 5;
+ b[i] = BASE + OFFSET + i * 4;
+ asm volatile ("" ::: "memory");
+ expected -= (SIGNEDNESS_2 int) (a[i] * b[i]);
+ }
+ if (f (0x12, a, b) != expected)
+ __builtin_abort ();
+}
+
+/* { dg-final { scan-tree-dump-not "vect_recog_neg_dot_prod_pattern: detected"
"vect" } } */
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 5fcc37f053c7..ea75ec7fece7 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -1348,6 +1348,129 @@ vect_recog_dot_prod_pattern (vec_info *vinfo,
return pattern_stmt;
}
+static gimple *
+vect_recog_neg_dot_prod_pattern (vec_info *vinfo,
+ stmt_vec_info stmt_vinfo, tree *type_out)
+{
+ tree oprnd0, oprnd1;
+ gimple *last_stmt = stmt_vinfo->stmt;
+ tree type, half_type;
+ gimple *neg_stmt, *dot_stmt, *pattern_stmt;
+ tree neg, dot, var;
+
+ /* Look for the following pattern
+ DX = (TYPE1) X;
+ DY = (TYPE1) Y;
+ DPROD = DX * DY;
+ DDPROD = (TYPE2) DPROD;
+ sum_1 = sum_0 - DDPROD;
+ In which
+ - DX is double the size of X
+ - DY is double the size of Y
+ - DX, DY, DPROD all have the same type but the sign
+ between X, Y and DPROD can differ.
+ - sum is the same size of DPROD or bigger
+ - sum has been recognized as a reduction variable.
+
+ This is equivalent to:
+ DPROD = X w* Y; #widen mult
+ sum_1 = sum_0 - DPROD; #subtraction
+
+ Replace with:
+ neg = 0 - sum_0;
+ prod = DOT_PROD<X, Y, neg>;
+ sum_1 = 0 - prod;
+
+ i.e -(X*Y + (-SUM)) = SUM - X*Y
+ */
+
+ /* Starting from LAST_STMT, follow the defs of its uses in search
+ of the above pattern. */
+
+ if (!vect_reassociating_reduction_p (vinfo, stmt_vinfo, MINUS_EXPR,
+ &oprnd0, &oprnd1))
+ return NULL;
+
+ type = TREE_TYPE (gimple_get_lhs (last_stmt));
+
+ vect_unpromoted_value unprom_mult;
+ oprnd1 = vect_look_through_possible_promotion (vinfo, oprnd1, &unprom_mult);
+
+ /* So far so good. Since last_stmt was detected as a (subtraction)
reduction,
+ we know that oprnd0 is the reduction variable (defined by a loop-header
+ phi), and oprnd1 is an ssa-name defined by a stmt in the loop body.
+ Left to check that oprnd1 is defined by a (widen_)mult_expr. */
+ if (!oprnd1)
+ return NULL;
+
+ stmt_vec_info mult_vinfo = vect_get_internal_def (vinfo, oprnd1);
+ if (!mult_vinfo)
+ return NULL;
+
+ /* FORNOW. Can continue analyzing the def-use chain when this stmt in a phi
+ inside the loop (in case we are analyzing an outer-loop). */
+ vect_unpromoted_value unprom0[2];
+ enum optab_subtype subtype = optab_vector;
+ if (!vect_widened_op_tree (vinfo, mult_vinfo, MULT_EXPR, WIDEN_MULT_EXPR,
+ false, 2, unprom0, &half_type, &subtype))
+ return NULL;
+
+ /* If there are two widening operations, make sure they agree on the sign
+ of the extension. The result of an optab_vector_mixed_sign operation
+ is signed; otherwise, the result has the same sign as the operands. */
+ if (TYPE_PRECISION (unprom_mult.type) != TYPE_PRECISION (type)
+ && (subtype == optab_vector_mixed_sign
+ ? TYPE_UNSIGNED (unprom_mult.type)
+ : TYPE_SIGN (unprom_mult.type) != TYPE_SIGN (half_type)))
+ return NULL;
+
+ vect_pattern_detected ("vect_recog_neg_dot_prod_pattern", last_stmt);
+
+ /* If the inputs have mixed signs, canonicalize on using the signed
+ input type for analysis. This also helps when emulating mixed-sign
+ operations using signed operations. */
+ if (subtype == optab_vector_mixed_sign)
+ half_type = signed_type_for (half_type);
+
+ tree half_vectype;
+ if (!vect_supportable_conv_optab_p (vinfo, type, DOT_PROD_EXPR, half_type,
+ type_out, &half_vectype, subtype))
+ {
+ /* We can emulate a mixed-sign dot-product using a sequence of
+ signed dot-products; see vect_emulate_mixed_dot_prod for details. */
+ if (subtype != optab_vector_mixed_sign
+ || !vect_supportable_conv_optab_p (vinfo, signed_type_for (type),
+ DOT_PROD_EXPR, half_type,
+ type_out, &half_vectype,
+ optab_vector))
+ return NULL;
+
+ *type_out = signed_or_unsigned_type_for (TYPE_UNSIGNED (type),
+ *type_out);
+ }
+
+ /* Get the inputs in the appropriate types. */
+ tree mult_oprnd[2];
+ vect_convert_inputs (vinfo, stmt_vinfo, 2, mult_oprnd, half_type,
+ unprom0, half_vectype, subtype);
+
+ tree zero = build_zero_cst (type);
+ neg = vect_recog_temp_ssa_var (type, NULL);
+ neg_stmt = gimple_build_assign (neg, MINUS_EXPR, zero, oprnd0);
+ append_pattern_def_seq (vinfo, stmt_vinfo, neg_stmt, *type_out);
+ gimple_set_location (neg_stmt, gimple_location (last_stmt));
+
+ dot = vect_recog_temp_ssa_var (type, NULL);
+ dot_stmt = gimple_build_assign (dot, DOT_PROD_EXPR,
+ mult_oprnd[0], mult_oprnd[1], neg);
+ append_pattern_def_seq (vinfo, stmt_vinfo, dot_stmt, *type_out);
+ gimple_set_location (dot_stmt, gimple_location (last_stmt));
+
+ var = vect_recog_temp_ssa_var (type, NULL);
+ pattern_stmt = gimple_build_assign (var, MINUS_EXPR, zero, dot);
+
+ return pattern_stmt;
+}
/* Function vect_recog_sad_pattern
@@ -7537,6 +7660,7 @@ static vect_recog_func vect_vect_recog_func_ptrs[] = {
{ vect_recog_mulhs_pattern, "mult_high" },
{ vect_recog_cast_forwprop_pattern, "cast_forwprop" },
{ vect_recog_widen_mult_pattern, "widen_mult" },
+ { vect_recog_neg_dot_prod_pattern, "neg_dot_prod" },
{ vect_recog_dot_prod_pattern, "dot_prod" },
{ vect_recog_sad_pattern, "sad" },
{ vect_recog_widen_sum_pattern, "widen_sum" },
--
2.53.0
