The 3rd patch in this series will modify these functions, but I have to re-order
and extract some shared functionality to avoid duplicate and having to forward
declare stuff.
I thus separated out the refactoring bit from the parts that use and change
these functions.
This patch is NFC.
Bootstrapped Regtested on aarch64-none-linux-gnu and no issues.
Any comments? otherwise will commit end of week.
Thanks,
Tamar
gcc/ChangeLog:
* config/aarch64/aarch64-acle-builtins.cc
(gimple_folder::fold_to_vl_pred): Refactor folding logic into
aarch64_fold_sve_ptrue_vl.
* config/aarch64/aarch64-protos.h (aarch64_fold_sve_ptrue_vl): New.
* config/aarch64/aarch64.cc (aarch64_instruction_selection): Move down.
(aarch64_fold_sve_ptrue_vl): New.
---
diff --git a/gcc/config/aarch64/aarch64-acle-builtins.cc
b/gcc/config/aarch64/aarch64-acle-builtins.cc
index
426316001ced864081e798b50c7dcb3281a77834..e64c6f00f91717151be0f4d092ca4d5e1123f061
100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.cc
+++ b/gcc/config/aarch64/aarch64-acle-builtins.cc
@@ -2983,22 +2983,9 @@ gimple *
gimple_folder::fold_to_vl_pred (unsigned int vl)
{
tree vectype = TREE_TYPE (lhs);
- tree element_type = TREE_TYPE (vectype);
- tree minus_one = build_all_ones_cst (element_type);
- tree zero = build_zero_cst (element_type);
unsigned int element_bytes = type_suffix (0).element_bytes;
-
- /* Construct COUNT elements that contain the ptrue followed by
- a repeating sequence of COUNT elements. */
- unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype));
- gcc_assert (vl * element_bytes <= count);
- tree_vector_builder builder (vectype, count, 2);
- for (unsigned int i = 0; i < count * 2; ++i)
- {
- bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes;
- builder.quick_push (bit ? minus_one : zero);
- }
- return gimple_build_assign (lhs, builder.build ());
+ tree pred = aarch64_fold_sve_ptrue_vl (vectype, vl, element_bytes);
+ return gimple_build_assign (lhs, pred);
}
/* Try to fold the call to a constant, given that, for integers, the call
diff --git a/gcc/config/aarch64/aarch64-protos.h
b/gcc/config/aarch64/aarch64-protos.h
index
5580f3148d6db754adad6804b506f5a0fe3ecf66..f9943f506c12aec81b3119ceac5d7cafd223dfea
100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1035,6 +1035,7 @@ unsigned aarch64_debugger_regno (unsigned);
unsigned aarch64_trampoline_size (void);
void aarch64_asm_output_labelref (FILE *, const char *);
void aarch64_cpu_cpp_builtins (cpp_reader *);
+tree aarch64_fold_sve_ptrue_vl (tree, unsigned int, unsigned int);
const char * aarch64_gen_far_branch (rtx *, int, const char *, const char *);
const char * aarch64_output_probe_stack_range (rtx, rtx);
const char * aarch64_output_probe_sve_stack_clash (rtx, rtx, rtx, rtx);
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index
72822ca621f52afee144bffa76bc75aac28adae1..bcd143ac06cc1b3674d05dd087874d9b435cae06
100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -63,6 +63,7 @@
#include "gimple-iterator.h"
#include "gimple-fold.h"
#include "tree-vectorizer.h"
+#include "tree-vector-builder.h"
#include "aarch64-cost-tables.h"
#include "dumpfile.h"
#include "builtins.h"
@@ -2311,72 +2312,6 @@ aarch64_try_widen_mult_by_pow2 (const gassign *convert,
return true;
}
-/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
- change generic sequences to a form AArch64 has an easier time expanding
- instructions for. It's not supposed to be used for generic rewriting that
- all targets would benefit from. */
-
-static bool
-aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
-{
- auto stmt = gsi_stmt (*gsi);
- gassign *assign = dyn_cast<gassign *> (stmt);
-
- if (!assign)
- return false;
-
- if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign))
- && aarch64_try_widen_mult_by_pow2 (assign, gsi))
- return true;
-
- /* Convert
- p == q ? s1 : s2;
- to
- p != q ? s2 : s1;
- where p and q are svbool_t expr. Due to the absence of predicate
- comparison instructions, we use bitwise xor for checking inequality.
- Transforming == to != avoids an extra bitwise inversion to the xor. */
- if (gimple_assign_rhs_code (assign) != VEC_COND_EXPR)
- return false;
-
- tree lhs = gimple_assign_lhs (assign);
- tree rhs1 = gimple_assign_rhs1 (assign);
- tree rhs2 = gimple_assign_rhs2 (assign);
- tree rhs3 = gimple_assign_rhs3 (assign);
-
- if (TREE_CODE (rhs1) != SSA_NAME || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE
(rhs1)))
- return false;
-
- gassign *da = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (rhs1));
-
- if (!da)
- return false;
-
- if (gimple_assign_rhs_code (da) != EQ_EXPR)
- return false;
-
- tree eqa = gimple_assign_rhs1 (da);
- tree eqb = gimple_assign_rhs2 (da);
-
- if (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqa))
- || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqb)))
- return false;
-
- tree ne_expr_var = create_tmp_var (TREE_TYPE (rhs1));
- gimple *ne_stmt = gimple_build_assign (ne_expr_var, NE_EXPR, eqa, eqb);
- gsi_safe_insert_before (gsi, ne_stmt);
-
- gimple *g = gimple_build_call_internal (IFN_VCOND_MASK, 3,
- ne_expr_var, rhs3, rhs2);
- if (!g)
- return false;
-
- gimple_set_lhs (g, lhs);
- gsi_replace (gsi, g, false);
-
- return true;
-}
-
/* Implement TARGET_MAX_NOCE_IFCVT_SEQ_COST. If an explicit max was set then
honor it, otherwise apply a tuning specific scale to branch costs. */
@@ -4482,6 +4417,97 @@ aarch64_fold_sve_cnt_pat (aarch64_svpattern pattern,
unsigned int nelts_per_vq)
return -1;
}
+/* Build a predicate of type VECTYPE in which the first VL elements of size
+ ELEMENT_BYTES are set and the rest are clear. */
+
+tree
+aarch64_fold_sve_ptrue_vl (tree vectype, unsigned int vl,
+ unsigned int element_bytes)
+{
+ tree element_type = TREE_TYPE (vectype);
+ tree minus_one = build_all_ones_cst (element_type);
+ tree zero = build_zero_cst (element_type);
+
+ /* Construct COUNT elements that contain the ptrue followed by
+ a repeating sequence of COUNT elements. */
+ unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype));
+ count = MAX (count, vl * element_bytes);
+ count = pow2p_hwi (count) ? count : 1U << ceil_log2 (count);
+ tree_vector_builder builder (vectype, count, 2);
+ for (unsigned int i = 0; i < count * 2; ++i)
+ {
+ bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes;
+ builder.quick_push (bit ? minus_one : zero);
+ }
+ return builder.build ();
+}
+
+/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
+ change generic sequences to a form AArch64 has an easier time expanding
+ instructions for. It's not supposed to be used for generic rewriting that
+ all targets would benefit from. */
+
+static bool
+aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
+{
+ auto stmt = gsi_stmt (*gsi);
+ gassign *assign = dyn_cast<gassign *> (stmt);
+
+ if (!assign)
+ return false;
+
+ if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign))
+ && aarch64_try_widen_mult_by_pow2 (assign, gsi))
+ return true;
+
+ /* Convert
+ p == q ? s1 : s2;
+ to
+ p != q ? s2 : s1;
+ where p and q are svbool_t expr. Due to the absence of predicate
+ comparison instructions, we use bitwise xor for checking inequality.
+ Transforming == to != avoids an extra bitwise inversion to the xor. */
+ if (gimple_assign_rhs_code (assign) != VEC_COND_EXPR)
+ return false;
+
+ tree lhs = gimple_assign_lhs (assign);
+ tree rhs1 = gimple_assign_rhs1 (assign);
+ tree rhs2 = gimple_assign_rhs2 (assign);
+ tree rhs3 = gimple_assign_rhs3 (assign);
+
+ if (TREE_CODE (rhs1) != SSA_NAME || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE
(rhs1)))
+ return false;
+
+ gassign *da = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (rhs1));
+
+ if (!da)
+ return false;
+
+ if (gimple_assign_rhs_code (da) != EQ_EXPR)
+ return false;
+
+ tree eqa = gimple_assign_rhs1 (da);
+ tree eqb = gimple_assign_rhs2 (da);
+
+ if (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqa))
+ || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqb)))
+ return false;
+
+ tree ne_expr_var = create_tmp_var (TREE_TYPE (rhs1));
+ gimple *ne_stmt = gimple_build_assign (ne_expr_var, NE_EXPR, eqa, eqb);
+ gsi_safe_insert_before (gsi, ne_stmt);
+
+ gimple *g = gimple_build_call_internal (IFN_VCOND_MASK, 3,
+ ne_expr_var, rhs3, rhs2);
+ if (!g)
+ return false;
+
+ gimple_set_lhs (g, lhs);
+ gsi_replace (gsi, g, false);
+
+ return true;
+}
+
/* Return true if a single CNT[BHWD] instruction can multiply FACTOR
by the number of 128-bit quadwords in an SVE vector. */
--
diff --git a/gcc/config/aarch64/aarch64-acle-builtins.cc b/gcc/config/aarch64/aarch64-acle-builtins.cc
index 426316001ced864081e798b50c7dcb3281a77834..e64c6f00f91717151be0f4d092ca4d5e1123f061 100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.cc
+++ b/gcc/config/aarch64/aarch64-acle-builtins.cc
@@ -2983,22 +2983,9 @@ gimple *
gimple_folder::fold_to_vl_pred (unsigned int vl)
{
tree vectype = TREE_TYPE (lhs);
- tree element_type = TREE_TYPE (vectype);
- tree minus_one = build_all_ones_cst (element_type);
- tree zero = build_zero_cst (element_type);
unsigned int element_bytes = type_suffix (0).element_bytes;
-
- /* Construct COUNT elements that contain the ptrue followed by
- a repeating sequence of COUNT elements. */
- unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype));
- gcc_assert (vl * element_bytes <= count);
- tree_vector_builder builder (vectype, count, 2);
- for (unsigned int i = 0; i < count * 2; ++i)
- {
- bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes;
- builder.quick_push (bit ? minus_one : zero);
- }
- return gimple_build_assign (lhs, builder.build ());
+ tree pred = aarch64_fold_sve_ptrue_vl (vectype, vl, element_bytes);
+ return gimple_build_assign (lhs, pred);
}
/* Try to fold the call to a constant, given that, for integers, the call
diff --git a/gcc/config/aarch64/aarch64-protos.h b/gcc/config/aarch64/aarch64-protos.h
index 5580f3148d6db754adad6804b506f5a0fe3ecf66..f9943f506c12aec81b3119ceac5d7cafd223dfea 100644
--- a/gcc/config/aarch64/aarch64-protos.h
+++ b/gcc/config/aarch64/aarch64-protos.h
@@ -1035,6 +1035,7 @@ unsigned aarch64_debugger_regno (unsigned);
unsigned aarch64_trampoline_size (void);
void aarch64_asm_output_labelref (FILE *, const char *);
void aarch64_cpu_cpp_builtins (cpp_reader *);
+tree aarch64_fold_sve_ptrue_vl (tree, unsigned int, unsigned int);
const char * aarch64_gen_far_branch (rtx *, int, const char *, const char *);
const char * aarch64_output_probe_stack_range (rtx, rtx);
const char * aarch64_output_probe_sve_stack_clash (rtx, rtx, rtx, rtx);
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index 72822ca621f52afee144bffa76bc75aac28adae1..bcd143ac06cc1b3674d05dd087874d9b435cae06 100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -63,6 +63,7 @@
#include "gimple-iterator.h"
#include "gimple-fold.h"
#include "tree-vectorizer.h"
+#include "tree-vector-builder.h"
#include "aarch64-cost-tables.h"
#include "dumpfile.h"
#include "builtins.h"
@@ -2311,72 +2312,6 @@ aarch64_try_widen_mult_by_pow2 (const gassign *convert,
return true;
}
-/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
- change generic sequences to a form AArch64 has an easier time expanding
- instructions for. It's not supposed to be used for generic rewriting that
- all targets would benefit from. */
-
-static bool
-aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
-{
- auto stmt = gsi_stmt (*gsi);
- gassign *assign = dyn_cast<gassign *> (stmt);
-
- if (!assign)
- return false;
-
- if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign))
- && aarch64_try_widen_mult_by_pow2 (assign, gsi))
- return true;
-
- /* Convert
- p == q ? s1 : s2;
- to
- p != q ? s2 : s1;
- where p and q are svbool_t expr. Due to the absence of predicate
- comparison instructions, we use bitwise xor for checking inequality.
- Transforming == to != avoids an extra bitwise inversion to the xor. */
- if (gimple_assign_rhs_code (assign) != VEC_COND_EXPR)
- return false;
-
- tree lhs = gimple_assign_lhs (assign);
- tree rhs1 = gimple_assign_rhs1 (assign);
- tree rhs2 = gimple_assign_rhs2 (assign);
- tree rhs3 = gimple_assign_rhs3 (assign);
-
- if (TREE_CODE (rhs1) != SSA_NAME || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1)))
- return false;
-
- gassign *da = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (rhs1));
-
- if (!da)
- return false;
-
- if (gimple_assign_rhs_code (da) != EQ_EXPR)
- return false;
-
- tree eqa = gimple_assign_rhs1 (da);
- tree eqb = gimple_assign_rhs2 (da);
-
- if (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqa))
- || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqb)))
- return false;
-
- tree ne_expr_var = create_tmp_var (TREE_TYPE (rhs1));
- gimple *ne_stmt = gimple_build_assign (ne_expr_var, NE_EXPR, eqa, eqb);
- gsi_safe_insert_before (gsi, ne_stmt);
-
- gimple *g = gimple_build_call_internal (IFN_VCOND_MASK, 3,
- ne_expr_var, rhs3, rhs2);
- if (!g)
- return false;
-
- gimple_set_lhs (g, lhs);
- gsi_replace (gsi, g, false);
-
- return true;
-}
-
/* Implement TARGET_MAX_NOCE_IFCVT_SEQ_COST. If an explicit max was set then
honor it, otherwise apply a tuning specific scale to branch costs. */
@@ -4482,6 +4417,97 @@ aarch64_fold_sve_cnt_pat (aarch64_svpattern pattern, unsigned int nelts_per_vq)
return -1;
}
+/* Build a predicate of type VECTYPE in which the first VL elements of size
+ ELEMENT_BYTES are set and the rest are clear. */
+
+tree
+aarch64_fold_sve_ptrue_vl (tree vectype, unsigned int vl,
+ unsigned int element_bytes)
+{
+ tree element_type = TREE_TYPE (vectype);
+ tree minus_one = build_all_ones_cst (element_type);
+ tree zero = build_zero_cst (element_type);
+
+ /* Construct COUNT elements that contain the ptrue followed by
+ a repeating sequence of COUNT elements. */
+ unsigned int count = constant_lower_bound (TYPE_VECTOR_SUBPARTS (vectype));
+ count = MAX (count, vl * element_bytes);
+ count = pow2p_hwi (count) ? count : 1U << ceil_log2 (count);
+ tree_vector_builder builder (vectype, count, 2);
+ for (unsigned int i = 0; i < count * 2; ++i)
+ {
+ bool bit = (i & (element_bytes - 1)) == 0 && i < vl * element_bytes;
+ builder.quick_push (bit ? minus_one : zero);
+ }
+ return builder.build ();
+}
+
+/* Implement TARGET_INSTRUCTION_SELECTION. The target hook is used to
+ change generic sequences to a form AArch64 has an easier time expanding
+ instructions for. It's not supposed to be used for generic rewriting that
+ all targets would benefit from. */
+
+static bool
+aarch64_instruction_selection (function * /* fun */, gimple_stmt_iterator *gsi)
+{
+ auto stmt = gsi_stmt (*gsi);
+ gassign *assign = dyn_cast<gassign *> (stmt);
+
+ if (!assign)
+ return false;
+
+ if (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign))
+ && aarch64_try_widen_mult_by_pow2 (assign, gsi))
+ return true;
+
+ /* Convert
+ p == q ? s1 : s2;
+ to
+ p != q ? s2 : s1;
+ where p and q are svbool_t expr. Due to the absence of predicate
+ comparison instructions, we use bitwise xor for checking inequality.
+ Transforming == to != avoids an extra bitwise inversion to the xor. */
+ if (gimple_assign_rhs_code (assign) != VEC_COND_EXPR)
+ return false;
+
+ tree lhs = gimple_assign_lhs (assign);
+ tree rhs1 = gimple_assign_rhs1 (assign);
+ tree rhs2 = gimple_assign_rhs2 (assign);
+ tree rhs3 = gimple_assign_rhs3 (assign);
+
+ if (TREE_CODE (rhs1) != SSA_NAME || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (rhs1)))
+ return false;
+
+ gassign *da = dyn_cast<gassign *> (SSA_NAME_DEF_STMT (rhs1));
+
+ if (!da)
+ return false;
+
+ if (gimple_assign_rhs_code (da) != EQ_EXPR)
+ return false;
+
+ tree eqa = gimple_assign_rhs1 (da);
+ tree eqb = gimple_assign_rhs2 (da);
+
+ if (!VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqa))
+ || !VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (eqb)))
+ return false;
+
+ tree ne_expr_var = create_tmp_var (TREE_TYPE (rhs1));
+ gimple *ne_stmt = gimple_build_assign (ne_expr_var, NE_EXPR, eqa, eqb);
+ gsi_safe_insert_before (gsi, ne_stmt);
+
+ gimple *g = gimple_build_call_internal (IFN_VCOND_MASK, 3,
+ ne_expr_var, rhs3, rhs2);
+ if (!g)
+ return false;
+
+ gimple_set_lhs (g, lhs);
+ gsi_replace (gsi, g, false);
+
+ return true;
+}
+
/* Return true if a single CNT[BHWD] instruction can multiply FACTOR
by the number of 128-bit quadwords in an SVE vector. */
