https://gcc.gnu.org/g:492bf4d89a727e24057048e774bfb2df6454c7c9
commit r16-9271-g492bf4d89a727e24057048e774bfb2df6454c7c9 Author: Richard Biener <[email protected]> Date: Fri Dec 19 09:15:58 2025 +0100 Add tree_to_vec_perm_indices The following adds a function like tree_to_vec_perm_builder but for building vec_perm_indices directly from all operands of a VEC_PERM_EXPR. This avoids errors such as shown in PR123175 and allows us to extract information from the permuted inputs to implement vec_shl/vec_shr recognition in can_vec_perm_const_p, making that fully cover what vector lowering will assess. * vec-perm-indices.h (vec_perm_indices::new_vector): New overload. (vec_perm_indices::input_bitwise_zero_p): New method. (vec_perm_indices::m_input0_bitwise_zero_p, vec_perm_indices::m_input1_bitwise_zero_p): New members. (vec_perm_indices::vec_perm_indices): Adjust. (tree_to_vec_perm_indices): Declare. * vec-perm-indices.cc (vec_perm_indices::new_vector): New overload. (vec_perm_indices::new_expanded_vector): Adjust. (tree_to_vec_perm_indices): New function. * optabs-query.cc (can_vec_perm_const_p): Handle permute patterns mapping to vec_shl/vec_shr. * tree-vect-generic.cc (lower_vec_perm): Move vec_shl/vec_shr detection to can_vec_perm_const_p and simplify. (cherry picked from commit 86cdbd105172ac2c5ee93e1494d51b80354dbc97) Diff: --- gcc/optabs-query.cc | 53 ++++++++++++++++++++++++++++++++++++----- gcc/tree-vect-generic.cc | 62 +++--------------------------------------------- gcc/vec-perm-indices.cc | 25 +++++++++++++++++++ gcc/vec-perm-indices.h | 21 ++++++++++++++-- 4 files changed, 94 insertions(+), 67 deletions(-) diff --git a/gcc/optabs-query.cc b/gcc/optabs-query.cc index 40eaa3b85701..0d734de18b40 100644 --- a/gcc/optabs-query.cc +++ b/gcc/optabs-query.cc @@ -415,12 +415,7 @@ can_vec_perm_var_p (machine_mode mode) /* Return true if the target directly supports VEC_PERM_EXPRs on vectors of mode OP_MODE and result vector of mode MODE using the selector SEL. ALLOW_VARIABLE_P is true if it is acceptable to force the selector into a - register and use a variable permute (if the target supports that). - - Note that additional permutations representing whole-vector shifts may - also be handled via the vec_shr or vec_shl optab, but only where the - second input vector is entirely constant zeroes; this case is not dealt - with here. */ + register and use a variable permute (if the target supports that). */ bool can_vec_perm_const_p (machine_mode mode, machine_mode op_mode, @@ -465,6 +460,52 @@ can_vec_perm_const_p (machine_mode mode, machine_mode op_mode, into integer operations. */ } + unsigned elements; + if (mode == op_mode + && GET_MODE_NUNITS (mode).is_constant (&elements)) + { + if (sel.input_bitwise_zero_p (0) + && can_implement_p (vec_shl_optab, mode)) + { + unsigned int first = 0, i; + for (i = 0; i < elements; ++i) + if (known_eq (poly_uint64 (sel[i]), elements)) + { + if (i == 0 || first) + break; + first = i; + } + else if (first + ? maybe_ne (poly_uint64 (sel[i]), + elements + i - first) + : maybe_ge (poly_uint64 (sel[i]), elements)) + break; + if (first && i == elements) + return true; + } + if (sel.input_bitwise_zero_p (1) + && maybe_ne (sel[0], 0) + && known_lt (sel[0], elements) + && can_implement_p (vec_shr_optab, mode)) + { + if (sel.series_p (0, 1, sel[0], 1)) + return true; + unsigned i; + for (i = 1; i < elements; ++i) + { + poly_uint64 actual = sel[i]; + poly_uint64 expected = i + sel[0]; + /* Indices into the second vector are all equivalent. */ + if (maybe_lt (actual, elements) + ? maybe_ne (actual, expected) + : maybe_lt (expected, elements)) + break; + } + if (i == elements) + return true; + } + } + return false; } diff --git a/gcc/tree-vect-generic.cc b/gcc/tree-vect-generic.cc index 77066bbb2e1b..f25d207b35b9 100644 --- a/gcc/tree-vect-generic.cc +++ b/gcc/tree-vect-generic.cc @@ -1640,76 +1640,20 @@ lower_vec_perm (gimple_stmt_iterator *gsi) mask = gimple_assign_rhs1 (def_stmt); } - vec_perm_builder sel_int; + vec_perm_indices indices; if (TREE_CODE (mask) == VECTOR_CST - && tree_to_vec_perm_builder (&sel_int, mask)) + && tree_to_vec_perm_indices (&indices, vec0, vec1, mask)) { - vec_perm_indices indices (sel_int, 2, in_elements); machine_mode vmode = TYPE_MODE (vect_type); tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt)); machine_mode lhs_mode = TYPE_MODE (lhs_type); - if (can_vec_perm_const_p (lhs_mode, vmode, indices)) + if (can_vec_perm_const_p (lhs_mode, vmode, indices, true)) { gimple_assign_set_rhs3 (stmt, mask); update_stmt (stmt); return; } - /* Also detect vec_shr pattern - VEC_PERM_EXPR with zero - vector as VEC1 and a right element shift MASK. */ - if (can_implement_p (vec_shr_optab, TYPE_MODE (vect_type)) - && TREE_CODE (vec1) == VECTOR_CST - && initializer_zerop (vec1) - && maybe_ne (indices[0], 0) - && known_lt (poly_uint64 (indices[0]), elements)) - { - bool ok_p = indices.series_p (0, 1, indices[0], 1); - if (!ok_p) - { - for (i = 1; i < elements; ++i) - { - poly_uint64 actual = indices[i]; - poly_uint64 expected = i + indices[0]; - /* Indices into the second vector are all equivalent. */ - if (maybe_lt (actual, elements) - ? maybe_ne (actual, expected) - : maybe_lt (expected, elements)) - break; - } - ok_p = i == elements; - } - if (ok_p) - { - gimple_assign_set_rhs3 (stmt, mask); - update_stmt (stmt); - return; - } - } - /* And similarly vec_shl pattern. */ - if (can_implement_p (vec_shl_optab, TYPE_MODE (vect_type)) - && TREE_CODE (vec0) == VECTOR_CST - && initializer_zerop (vec0)) - { - unsigned int first = 0; - for (i = 0; i < elements; ++i) - if (known_eq (poly_uint64 (indices[i]), elements)) - { - if (i == 0 || first) - break; - first = i; - } - else if (first - ? maybe_ne (poly_uint64 (indices[i]), - elements + i - first) - : maybe_ge (poly_uint64 (indices[i]), elements)) - break; - if (first && i == elements) - { - gimple_assign_set_rhs3 (stmt, mask); - update_stmt (stmt); - return; - } - } } else if (can_vec_perm_var_p (TYPE_MODE (vect_type))) return; diff --git a/gcc/vec-perm-indices.cc b/gcc/vec-perm-indices.cc index 5b54a84caeea..6bb2f39ff912 100644 --- a/gcc/vec-perm-indices.cc +++ b/gcc/vec-perm-indices.cc @@ -31,6 +31,7 @@ along with GCC; see the file COPYING3. If not see #include "selftest.h" #include "rtx-vector-builder.h" + /* Switch to a new permutation vector that selects between NINPUTS vector inputs that have NELTS_PER_INPUT elements each. Take the elements of the new permutation vector from ELEMENTS, clamping each one to be in range. */ @@ -38,10 +39,14 @@ along with GCC; see the file COPYING3. If not see void vec_perm_indices::new_vector (const vec_perm_builder &elements, unsigned int ninputs, + bool input0_bitwise_zero_p, + bool input1_bitwise_zero_p, poly_uint64 nelts_per_input) { m_ninputs = ninputs; m_nelts_per_input = nelts_per_input; + m_input0_bitwise_zero_p = input0_bitwise_zero_p; + m_input1_bitwise_zero_p = input1_bitwise_zero_p; /* If the vector has a constant number of elements, expand the encoding and clamp each element. E.g. { 0, 2, 4, ... } might wrap halfway if there is only one vector input, and we want @@ -87,6 +92,8 @@ vec_perm_indices::new_expanded_vector (const vec_perm_indices &orig, unsigned int factor) { m_ninputs = orig.m_ninputs; + m_input0_bitwise_zero_p = orig.m_input0_bitwise_zero_p; + m_input1_bitwise_zero_p = orig.m_input1_bitwise_zero_p; m_nelts_per_input = orig.m_nelts_per_input * factor; m_encoding.new_vector (orig.m_encoding.full_nelts () * factor, orig.m_encoding.npatterns () * factor, @@ -301,6 +308,24 @@ tree_to_vec_perm_builder (vec_perm_builder *builder, tree cst) return true; } +/* Try to read the contents of VECTOR_CST PERM_CST as a constant permutation + vector permuting OP0 and OP1. Return true and populate INDICES on success, + otherwise return false without modifying INDICES. */ + +bool +tree_to_vec_perm_indices (vec_perm_indices *indices, tree op0, tree op1, + tree perm_cst) +{ + vec_perm_builder builder; + if (!tree_to_vec_perm_builder (&builder, perm_cst)) + return false; + indices->new_vector (builder, op0 == op1 ? 1 : 2, + initializer_zerop (op0), initializer_zerop (op1), + TYPE_VECTOR_SUBPARTS (TREE_TYPE (op0))); + return true; +} + + /* Return a VECTOR_CST of type TYPE for the permutation vector in INDICES. */ tree diff --git a/gcc/vec-perm-indices.h b/gcc/vec-perm-indices.h index 5e68f7f62ec4..f596aa48c9cd 100644 --- a/gcc/vec-perm-indices.h +++ b/gcc/vec-perm-indices.h @@ -55,7 +55,12 @@ public: vec_perm_indices (); vec_perm_indices (const vec_perm_builder &, unsigned int, poly_uint64); - void new_vector (const vec_perm_builder &, unsigned int, poly_uint64); + void new_vector (const vec_perm_builder &b, unsigned int ni, poly_uint64 ne) + { + new_vector (b, ni, false, false, ne); + } + void new_vector (const vec_perm_builder &, unsigned int, bool, bool, + poly_uint64); void new_expanded_vector (const vec_perm_indices &, unsigned int); bool new_shrunk_vector (const vec_perm_indices &, unsigned int); void rotate_inputs (int delta); @@ -70,6 +75,13 @@ public: /* Return the number of input vectors being permuted. */ unsigned int ninputs () const { return m_ninputs; } + /* Return whether the input N is known bitwise zero. */ + bool input_bitwise_zero_p (unsigned n) const + { + return (n == 0 ? m_input0_bitwise_zero_p + : (n == 1 ? m_input1_bitwise_zero_p : false)); + } + /* Return the number of elements in each input vector. */ poly_uint64 nelts_per_input () const { return m_nelts_per_input; } @@ -87,16 +99,21 @@ private: vec_perm_builder m_encoding; unsigned int m_ninputs; + bool m_input0_bitwise_zero_p; + bool m_input1_bitwise_zero_p; poly_uint64 m_nelts_per_input; }; bool tree_to_vec_perm_builder (vec_perm_builder *, tree); +bool tree_to_vec_perm_indices (vec_perm_indices *, tree, tree, tree); tree vec_perm_indices_to_tree (tree, const vec_perm_indices &); rtx vec_perm_indices_to_rtx (machine_mode, const vec_perm_indices &); inline vec_perm_indices::vec_perm_indices () : m_ninputs (0), + m_input0_bitwise_zero_p (false), + m_input1_bitwise_zero_p (false), m_nelts_per_input (0) { } @@ -110,7 +127,7 @@ vec_perm_indices::vec_perm_indices (const vec_perm_builder &elements, unsigned int ninputs, poly_uint64 nelts_per_input) { - new_vector (elements, ninputs, nelts_per_input); + new_vector (elements, ninputs, false, false, nelts_per_input); } /* Return the canonical value for permutation vector element ELT,
