From: Stefan Schulze Frielinghaus <[email protected]>
The defined-at-zero value may differ between scalar and vector modes.
Even if the bit precision of a scalar and a vector lane coincide, the
defined-at-zero value may differ. Therefore, during vect-patterns,
consider the defined-at-zero value for the natural vector mode instead
of a scalar mode since this is closer to the final vector mode.
However, since in theory an at-zero value could differ for the natural
and final vector mode, do a final check during vectorizable_call.
gcc/ChangeLog:
* config/i386/i386.h (CTZ_DEFINED_VALUE_AT_ZERO): Utilize
GET_MODE_UNIT_BITSIZE instead of GET_MODE_BITSIZE in order to
deal with vector modes.
* config/rs6000/rs6000.h (CLZ_DEFINED_VALUE_AT_ZERO): Ditto.
(CTZ_DEFINED_VALUE_AT_ZERO):
* tree-vect-patterns.cc (vect_recog_ctz_ffs_pattern): Use
defined zero value for vector modes.
(vect_recog_popcount_clz_ctz_ffs_pattern): Test for a defined
zero value for vector modes.
* tree-vect-stmts.cc (vectorizable_call): Bail out if natural
and final vector mode do not coincide w.r.t. defined-at-zero
value.
---
gcc/config/i386/i386.h | 4 ++--
gcc/config/rs6000/rs6000.h | 4 ++--
gcc/tree-vect-patterns.cc | 22 ++++++++++------------
gcc/tree-vect-stmts.cc | 13 +++++++++++++
4 files changed, 27 insertions(+), 16 deletions(-)
diff --git a/gcc/config/i386/i386.h b/gcc/config/i386/i386.h
index 47725c56090..35563a6144a 100644
--- a/gcc/config/i386/i386.h
+++ b/gcc/config/i386/i386.h
@@ -3060,9 +3060,9 @@ extern void debug_dispatch_window (int);
/* The value at zero is only defined for the BMI instructions
LZCNT and TZCNT, not the BSR/BSF insns in the original isa. */
#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
- ((VALUE) = GET_MODE_BITSIZE (MODE), TARGET_BMI ? 2 : 0)
+ ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), TARGET_BMI ? 2 : 0)
#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
- ((VALUE) = GET_MODE_BITSIZE (MODE), TARGET_LZCNT ? 2 : 0)
+ ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), TARGET_LZCNT ? 2 : 0)
/* Flags returned by ix86_get_callcvt (). */
diff --git a/gcc/config/rs6000/rs6000.h b/gcc/config/rs6000/rs6000.h
index bcd4c39d24d..98952f11189 100644
--- a/gcc/config/rs6000/rs6000.h
+++ b/gcc/config/rs6000/rs6000.h
@@ -1749,14 +1749,14 @@ typedef struct rs6000_args
/* The cntlzw and cntlzd instructions return 32 and 64 for input of zero. */
#define CLZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
- ((VALUE) = GET_MODE_BITSIZE (MODE), 2)
+ ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), 2)
/* The CTZ patterns that are implemented in terms of CLZ return -1 for input of
zero. The hardware instructions added in Power9 and the sequences using
popcount return 32 or 64. */
#define CTZ_DEFINED_VALUE_AT_ZERO(MODE, VALUE) \
(TARGET_CTZ || TARGET_POPCNTD
\
- ? ((VALUE) = GET_MODE_BITSIZE (MODE), 2) \
+ ? ((VALUE) = GET_MODE_UNIT_BITSIZE (MODE), 2) \
: ((VALUE) = -1, 2))
/* Specify the machine mode that pointers have.
diff --git a/gcc/tree-vect-patterns.cc b/gcc/tree-vect-patterns.cc
index 232c9620bc0..e9d2870feb4 100644
--- a/gcc/tree-vect-patterns.cc
+++ b/gcc/tree-vect-patterns.cc
@@ -1942,7 +1942,7 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo,
stmt_vec_info stmt_vinfo,
{
ifnnew = IFN_CTZ;
defined_at_zero_new
- = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
val_new) == 2;
}
else if (direct_internal_fn_supported_p (IFN_CLZ, vec_rhs_type,
@@ -1950,7 +1950,7 @@ vect_recog_ctz_ffs_pattern (vec_info *vinfo,
stmt_vec_info stmt_vinfo,
{
ifnnew = IFN_CLZ;
defined_at_zero_new
- = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (rhs_type),
+ = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_rhs_type),
val_new) == 2;
}
if ((ifnnew == IFN_LAST
@@ -2243,6 +2243,10 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
if (TYPE_PRECISION (unprom_diff.type) != TYPE_PRECISION (lhs_type))
return NULL;
+ vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
+ if (!vec_type || !VECTOR_MODE_P (TYPE_MODE (vec_type)))
+ return NULL;
+
/* Also A should be unsigned or same precision as temp_in, otherwise
different builtins/internal functions have different behaviors. */
if (TYPE_PRECISION (unprom_diff.type)
@@ -2272,8 +2276,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
int val1, val2;
val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
int d2
- = CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
- val2);
+ = CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
if (d2 != 2 || val1 != val2 + addend)
return NULL;
}
@@ -2288,8 +2291,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
int val1, val2;
val1 = tree_to_shwi (gimple_call_arg (call_stmt, 1));
int d2
- = CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
- val2);
+ = CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type), val2);
if (d2 != 2 || val1 != val2)
return NULL;
}
@@ -2301,11 +2303,7 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info *vinfo,
gcc_unreachable ();
}
- vec_type = get_vectype_for_scalar_type (vinfo, lhs_type);
/* Do it only if the backend has popcount<vector_mode>2 etc. pattern. */
- if (!vec_type)
- return NULL;
-
bool supported
= direct_internal_fn_supported_p (ifn, vec_type, OPTIMIZE_FOR_SPEED);
/* We cannot rely on the reduction check in vect_recog_ctz_ffs_pattern
@@ -2358,11 +2356,11 @@ vect_recog_popcount_clz_ctz_ffs_pattern (vec_info
*vinfo,
tree arg2 = NULL_TREE;
int val;
if (ifn == IFN_CLZ
- && CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
+ && CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
val) == 2)
arg2 = build_int_cst (integer_type_node, val);
else if (ifn == IFN_CTZ
- && CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_INT_TYPE_MODE (lhs_type),
+ && CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vec_type),
val) == 2)
arg2 = build_int_cst (integer_type_node, val);
if (arg2)
diff --git a/gcc/tree-vect-stmts.cc b/gcc/tree-vect-stmts.cc
index 18d0dd8ff3a..554b5410bf8 100644
--- a/gcc/tree-vect-stmts.cc
+++ b/gcc/tree-vect-stmts.cc
@@ -3549,6 +3549,19 @@ vectorizable_call (vec_info *vinfo,
return false;
}
+ /* During vect-patterns we remembered the defined-at-zero value in the
+ optional second argument for the natural vector mode. Now, once we have
+ the final vector mode, verify that they coincide. If not, bail out. */
+ if (clz_ctz_arg1)
+ {
+ int val;
+ int d = (cfn == CFN_CLZ
+ ? CLZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val)
+ : CTZ_DEFINED_VALUE_AT_ZERO (TYPE_MODE (vectype_in), val));
+ if (d != 2 || val != tree_to_shwi (clz_ctz_arg1))
+ return false;
+ }
+
if (VECTOR_BOOLEAN_TYPE_P (vectype_out)
!= VECTOR_BOOLEAN_TYPE_P (vectype_in))
{
--
2.55.0
