Hi All, To make code review of the updates to add multiple exit supports to vectorizable_live_operation easier I've extracted the refactoring part to its own patch.
This patch is a straight extract of the function with no functional changes. Bootstrapped Regtested on aarch64-none-linux-gnu and no issues. Ok for master? Thanks, Tamar gcc/ChangeLog: * tree-vect-loop.cc (vectorizable_live_operation_1): New. (vectorizable_live_operation): Extract code to vectorizable_live_operation_1. --- inline copy of patch -- diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 8a50380de49bc12105be47ea1d8ee3cf1f2bdab4..df5e1d28fac2ce35e71decdec0d8e31fb75557f5 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -10481,6 +10481,95 @@ vectorizable_induction (loop_vec_info loop_vinfo, return true; } + +/* Function vectorizable_live_operation_1. + helper function for vectorizable_live_operation. */ +tree +vectorizable_live_operation_1 (loop_vec_info loop_vinfo, + stmt_vec_info stmt_info, edge exit_e, + tree vectype, int ncopies, slp_tree slp_node, + tree bitsize, tree bitstart, tree vec_lhs, + tree lhs_type, gimple_stmt_iterator *exit_gsi) +{ + basic_block exit_bb = exit_e->dest; + gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); + + tree vec_lhs_phi = copy_ssa_name (vec_lhs); + gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); + for (unsigned i = 0; i < gimple_phi_num_args (phi); i++) + SET_PHI_ARG_DEF (phi, i, vec_lhs); + + gimple_seq stmts = NULL; + tree new_tree; + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) + { + /* Emit: + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> + where VEC_LHS is the vectorized live-out result and MASK is + the loop mask for the final iteration. */ + gcc_assert (ncopies == 1 && !slp_node); + gimple_seq tem = NULL; + gimple_stmt_iterator gsi = gsi_last (tem); + tree len = vect_get_loop_len (loop_vinfo, &gsi, + &LOOP_VINFO_LENS (loop_vinfo), + 1, vectype, 0, 0); + /* BIAS - 1. */ + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + tree bias_minus_one + = int_const_binop (MINUS_EXPR, + build_int_cst (TREE_TYPE (len), biasval), + build_one_cst (TREE_TYPE (len))); + /* LAST_INDEX = LEN + (BIAS - 1). */ + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), + len, bias_minus_one); + /* This needs to implement extraction of the first index, but not sure + how the LEN stuff works. At the moment we shouldn't get here since + there's no LEN support for early breaks. But guard this so there's + no incorrect codegen. */ + gcc_assert (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); + + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ + tree scalar_res + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), + vec_lhs_phi, last_index); + /* Convert the extracted vector element to the scalar type. */ + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); + } + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) + { + /* Emit: + SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> + where VEC_LHS is the vectorized live-out result and MASK is + the loop mask for the final iteration. */ + gcc_assert (!slp_node); + tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); + gimple_seq tem = NULL; + gimple_stmt_iterator gsi = gsi_last (tem); + tree mask = vect_get_loop_mask (loop_vinfo, &gsi, + &LOOP_VINFO_MASKS (loop_vinfo), + 1, vectype, 0); + + gimple_seq_add_seq (&stmts, tem); + tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, + mask, vec_lhs_phi); + /* Convert the extracted vector element to the scalar type. */ + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); + } + else + { + tree bftype = TREE_TYPE (vectype); + if (VECTOR_BOOLEAN_TYPE_P (vectype)) + bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); + new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart); + new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), + &stmts, true, NULL_TREE); + } + *exit_gsi = gsi_after_labels (exit_bb); + if (stmts) + gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT); + return new_tree; +} + /* Function vectorizable_live_operation. STMT_INFO computes a value that is used outside the loop. Check if @@ -10690,79 +10779,13 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx, vec_lhs); - gimple_seq stmts = NULL; - tree new_tree; - if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) - { - /* Emit: - - SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> - - where VEC_LHS is the vectorized live-out result and MASK is - the loop mask for the final iteration. */ - gcc_assert (ncopies == 1 && !slp_node); - gimple_seq tem = NULL; - gimple_stmt_iterator gsi = gsi_last (tem); - tree len - = vect_get_loop_len (loop_vinfo, &gsi, - &LOOP_VINFO_LENS (loop_vinfo), - 1, vectype, 0, 0); - - /* BIAS - 1. */ - signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - tree bias_minus_one - = int_const_binop (MINUS_EXPR, - build_int_cst (TREE_TYPE (len), biasval), - build_one_cst (TREE_TYPE (len))); - - /* LAST_INDEX = LEN + (BIAS - 1). */ - tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), - len, bias_minus_one); - - /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ - tree scalar_res - = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), - vec_lhs_phi, last_index); - - /* Convert the extracted vector element to the scalar type. */ - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); - } - else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) - { - /* Emit: - - SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> - - where VEC_LHS is the vectorized live-out result and MASK is - the loop mask for the final iteration. */ - gcc_assert (ncopies == 1 && !slp_node); - tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); - gimple_seq tem = NULL; - gimple_stmt_iterator gsi = gsi_last (tem); - tree mask = vect_get_loop_mask (loop_vinfo, &gsi, - &LOOP_VINFO_MASKS (loop_vinfo), - 1, vectype, 0); - gimple_seq_add_seq (&stmts, tem); - tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, - mask, vec_lhs_phi); - - /* Convert the extracted vector element to the scalar type. */ - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); - } - else - { - tree bftype = TREE_TYPE (vectype); - if (VECTOR_BOOLEAN_TYPE_P (vectype)) - bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); - new_tree = build3 (BIT_FIELD_REF, bftype, - vec_lhs_phi, bitsize, bitstart); - new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), - &stmts, true, NULL_TREE); - } - - gimple_stmt_iterator exit_gsi = gsi_after_labels (exit_bb); - if (stmts) - gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT); + gimple_stmt_iterator exit_gsi; + tree new_tree + = vectorizable_live_operation_1 (loop_vinfo, stmt_info, + LOOP_VINFO_IV_EXIT (loop_vinfo), + vectype, ncopies, slp_node, bitsize, + bitstart, vec_lhs, lhs_type, + &exit_gsi); /* Remove existing phis that copy from lhs and create copies from new_tree. */ --
diff --git a/gcc/tree-vect-loop.cc b/gcc/tree-vect-loop.cc index 8a50380de49bc12105be47ea1d8ee3cf1f2bdab4..df5e1d28fac2ce35e71decdec0d8e31fb75557f5 100644 --- a/gcc/tree-vect-loop.cc +++ b/gcc/tree-vect-loop.cc @@ -10481,6 +10481,95 @@ vectorizable_induction (loop_vec_info loop_vinfo, return true; } + +/* Function vectorizable_live_operation_1. + helper function for vectorizable_live_operation. */ +tree +vectorizable_live_operation_1 (loop_vec_info loop_vinfo, + stmt_vec_info stmt_info, edge exit_e, + tree vectype, int ncopies, slp_tree slp_node, + tree bitsize, tree bitstart, tree vec_lhs, + tree lhs_type, gimple_stmt_iterator *exit_gsi) +{ + basic_block exit_bb = exit_e->dest; + gcc_assert (single_pred_p (exit_bb) || LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); + + tree vec_lhs_phi = copy_ssa_name (vec_lhs); + gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); + for (unsigned i = 0; i < gimple_phi_num_args (phi); i++) + SET_PHI_ARG_DEF (phi, i, vec_lhs); + + gimple_seq stmts = NULL; + tree new_tree; + if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) + { + /* Emit: + SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> + where VEC_LHS is the vectorized live-out result and MASK is + the loop mask for the final iteration. */ + gcc_assert (ncopies == 1 && !slp_node); + gimple_seq tem = NULL; + gimple_stmt_iterator gsi = gsi_last (tem); + tree len = vect_get_loop_len (loop_vinfo, &gsi, + &LOOP_VINFO_LENS (loop_vinfo), + 1, vectype, 0, 0); + /* BIAS - 1. */ + signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); + tree bias_minus_one + = int_const_binop (MINUS_EXPR, + build_int_cst (TREE_TYPE (len), biasval), + build_one_cst (TREE_TYPE (len))); + /* LAST_INDEX = LEN + (BIAS - 1). */ + tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), + len, bias_minus_one); + /* This needs to implement extraction of the first index, but not sure + how the LEN stuff works. At the moment we shouldn't get here since + there's no LEN support for early breaks. But guard this so there's + no incorrect codegen. */ + gcc_assert (!LOOP_VINFO_EARLY_BREAKS (loop_vinfo)); + + /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ + tree scalar_res + = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), + vec_lhs_phi, last_index); + /* Convert the extracted vector element to the scalar type. */ + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); + } + else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) + { + /* Emit: + SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> + where VEC_LHS is the vectorized live-out result and MASK is + the loop mask for the final iteration. */ + gcc_assert (!slp_node); + tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); + gimple_seq tem = NULL; + gimple_stmt_iterator gsi = gsi_last (tem); + tree mask = vect_get_loop_mask (loop_vinfo, &gsi, + &LOOP_VINFO_MASKS (loop_vinfo), + 1, vectype, 0); + + gimple_seq_add_seq (&stmts, tem); + tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, + mask, vec_lhs_phi); + /* Convert the extracted vector element to the scalar type. */ + new_tree = gimple_convert (&stmts, lhs_type, scalar_res); + } + else + { + tree bftype = TREE_TYPE (vectype); + if (VECTOR_BOOLEAN_TYPE_P (vectype)) + bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); + new_tree = build3 (BIT_FIELD_REF, bftype, vec_lhs_phi, bitsize, bitstart); + new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), + &stmts, true, NULL_TREE); + } + *exit_gsi = gsi_after_labels (exit_bb); + if (stmts) + gsi_insert_seq_before (exit_gsi, stmts, GSI_SAME_STMT); + return new_tree; +} + /* Function vectorizable_live_operation. STMT_INFO computes a value that is used outside the loop. Check if @@ -10690,79 +10779,13 @@ vectorizable_live_operation (vec_info *vinfo, stmt_vec_info stmt_info, gimple *phi = create_phi_node (vec_lhs_phi, exit_bb); SET_PHI_ARG_DEF (phi, LOOP_VINFO_IV_EXIT (loop_vinfo)->dest_idx, vec_lhs); - gimple_seq stmts = NULL; - tree new_tree; - if (LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo)) - { - /* Emit: - - SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1> - - where VEC_LHS is the vectorized live-out result and MASK is - the loop mask for the final iteration. */ - gcc_assert (ncopies == 1 && !slp_node); - gimple_seq tem = NULL; - gimple_stmt_iterator gsi = gsi_last (tem); - tree len - = vect_get_loop_len (loop_vinfo, &gsi, - &LOOP_VINFO_LENS (loop_vinfo), - 1, vectype, 0, 0); - - /* BIAS - 1. */ - signed char biasval = LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS (loop_vinfo); - tree bias_minus_one - = int_const_binop (MINUS_EXPR, - build_int_cst (TREE_TYPE (len), biasval), - build_one_cst (TREE_TYPE (len))); - - /* LAST_INDEX = LEN + (BIAS - 1). */ - tree last_index = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (len), - len, bias_minus_one); - - /* SCALAR_RES = VEC_EXTRACT <VEC_LHS, LEN + BIAS - 1>. */ - tree scalar_res - = gimple_build (&stmts, CFN_VEC_EXTRACT, TREE_TYPE (vectype), - vec_lhs_phi, last_index); - - /* Convert the extracted vector element to the scalar type. */ - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); - } - else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)) - { - /* Emit: - - SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK> - - where VEC_LHS is the vectorized live-out result and MASK is - the loop mask for the final iteration. */ - gcc_assert (ncopies == 1 && !slp_node); - tree scalar_type = TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info)); - gimple_seq tem = NULL; - gimple_stmt_iterator gsi = gsi_last (tem); - tree mask = vect_get_loop_mask (loop_vinfo, &gsi, - &LOOP_VINFO_MASKS (loop_vinfo), - 1, vectype, 0); - gimple_seq_add_seq (&stmts, tem); - tree scalar_res = gimple_build (&stmts, CFN_EXTRACT_LAST, scalar_type, - mask, vec_lhs_phi); - - /* Convert the extracted vector element to the scalar type. */ - new_tree = gimple_convert (&stmts, lhs_type, scalar_res); - } - else - { - tree bftype = TREE_TYPE (vectype); - if (VECTOR_BOOLEAN_TYPE_P (vectype)) - bftype = build_nonstandard_integer_type (tree_to_uhwi (bitsize), 1); - new_tree = build3 (BIT_FIELD_REF, bftype, - vec_lhs_phi, bitsize, bitstart); - new_tree = force_gimple_operand (fold_convert (lhs_type, new_tree), - &stmts, true, NULL_TREE); - } - - gimple_stmt_iterator exit_gsi = gsi_after_labels (exit_bb); - if (stmts) - gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT); + gimple_stmt_iterator exit_gsi; + tree new_tree + = vectorizable_live_operation_1 (loop_vinfo, stmt_info, + LOOP_VINFO_IV_EXIT (loop_vinfo), + vectype, ncopies, slp_node, bitsize, + bitstart, vec_lhs, lhs_type, + &exit_gsi); /* Remove existing phis that copy from lhs and create copies from new_tree. */