From: Kyrylo Tkachov <[email protected]> Add factor_out_conditional_load: it turns a diamond PHI <*P, *Q> whose two arguments are single-use, non-volatile scalar MEM_REF loads reading the same memory state into P' = PHI <P, Q> and a single load *P', reusing the PHI result so its range info is preserved. No speculative load is introduced (the load uses whichever pointer the taken edge selected).
When the two arms' TBAA alias-ptr types differ, the merged load conservatively falls back to ptr_type_node and drops the dependence clique/base, the way get_alias_type_for_stmts does, rather than rejecting the transform. Its eligibility test is split out into conditional_load_factorable_p so the sink pass (next patch) can gate on commonability without mutating; both are exported for it to reuse. Bootstrapped and tested on aarch64-none-linux-gnu. Signed-off-by: Kyrylo Tkachov <[email protected]> PR tree-optimization/125557 * tree-ssa-phiopt.h (conditional_load_factorable_p): Declare. (factor_out_conditional_load): Declare. * tree-ssa-phiopt.cc: Include "alias.h". (remove_factored_arm_defs): New. (conditional_load_factorable_p): New. (factor_out_conditional_load): New. --- gcc/tree-ssa-phiopt.cc | 132 +++++++++++++++++++++++++++++++++++++++++ gcc/tree-ssa-phiopt.h | 9 +++ 2 files changed, 141 insertions(+) diff --git a/gcc/tree-ssa-phiopt.cc b/gcc/tree-ssa-phiopt.cc index dc6c33a930d..b018590c378 100644 --- a/gcc/tree-ssa-phiopt.cc +++ b/gcc/tree-ssa-phiopt.cc @@ -49,6 +49,7 @@ along with GCC; see the file COPYING3. If not see #include "internal-fn.h" #include "gimple-range.h" #include "gimple-match.h" +#include "alias.h" #include "dbgcnt.h" #include "tree-ssa-propagate.h" #include "tree-ssa-dce.h" @@ -417,6 +418,137 @@ commutative: return -1; } +/* Remove the two arm-defining statements D0 and D1 (D1 may be NULL, e.g. when + one PHI arm is a constant with no defining statement) once their results have + been factored into the merge. Used by factor_out_conditional_load below. */ + +static void +remove_factored_arm_defs (gimple *d0, gimple *d1) +{ + gimple_stmt_iterator gsi = gsi_for_stmt (d0); + gsi_remove (&gsi, true); + release_defs (d0); + if (d1) + { + gsi = gsi_for_stmt (d1); + gsi_remove (&gsi, true); + release_defs (d1); + } +} + +/* Return true if factor_out_conditional_load (below) would factor the load PHI + at MERGE. E0/E1 are the arm->MERGE edges. This is factor_out_conditional_load's + eligibility test, split out so callers (the sink pass) can gate on commonability + without mutating: the two MUST stay in lockstep, otherwise a caller could common + a load the sink then cannot finish if-converting. */ + +bool +conditional_load_factorable_p (edge e0, edge e1, basic_block merge, gphi *phi) +{ + if (virtual_operand_p (gimple_phi_result (phi)) + || gimple_phi_num_args (phi) != 2 + || get_virtual_phi (merge)) + return false; + + tree arg0 = gimple_phi_arg_def (phi, e0->dest_idx); + tree arg1 = gimple_phi_arg_def (phi, e1->dest_idx); + if (TREE_CODE (arg0) != SSA_NAME || TREE_CODE (arg1) != SSA_NAME + || !has_single_use (arg0) || !has_single_use (arg1)) + return false; + + gimple *d0 = SSA_NAME_DEF_STMT (arg0); + gimple *d1 = SSA_NAME_DEF_STMT (arg1); + if (!is_gimple_assign (d0) || !gimple_assign_load_p (d0) + || !is_gimple_assign (d1) || !gimple_assign_load_p (d1) + || gimple_has_volatile_ops (d0) || gimple_has_volatile_ops (d1) + || gimple_vuse (d0) != gimple_vuse (d1)) + return false; + + tree ref0 = gimple_assign_rhs1 (d0); + tree ref1 = gimple_assign_rhs1 (d1); + /* Both must be plain *P loads (zero offset) of a compatible value type. The + TBAA alias-ptr type carried by MEM_REF operand 1 need not match; it is + merged the way get_alias_type_for_stmts does when the load is built. */ + if (TREE_CODE (ref0) != MEM_REF || TREE_CODE (ref1) != MEM_REF + || !integer_zerop (TREE_OPERAND (ref0, 1)) + || !integer_zerop (TREE_OPERAND (ref1, 1)) + || !types_compatible_p (TREE_TYPE (ref0), TREE_TYPE (ref1))) + return false; + + tree p0 = TREE_OPERAND (ref0, 0); + tree p1 = TREE_OPERAND (ref1, 0); + if (!types_compatible_p (TREE_TYPE (p0), TREE_TYPE (p1))) + return false; + + return true; +} + +/* If PHI at MERGE is a "load PHI" -- PHI <*P, *Q> whose two arguments are + single-use, non-volatile scalar MEM_REF loads reading the same memory state + (same VUSE) -- factor the load out: introduce P' = PHI <P, Q> and a single + load *P' replacing the PHI. No speculative load is introduced (the load uses + whichever pointer the taken edge selected). Together with operand factoring + this turns a conditional load into one reg-offset load. E0/E1 are the + arm->MERGE edges. Returns true if the load was factored out. */ + +bool +factor_out_conditional_load (edge e0, edge e1, basic_block merge, gphi *phi) +{ + if (!conditional_load_factorable_p (e0, e1, merge, phi)) + return false; + + /* Re-derive the loads and pointers validated by the predicate above. */ + gimple *d0 = SSA_NAME_DEF_STMT (gimple_phi_arg_def (phi, e0->dest_idx)); + gimple *d1 = SSA_NAME_DEF_STMT (gimple_phi_arg_def (phi, e1->dest_idx)); + tree ref0 = gimple_assign_rhs1 (d0); + tree ref1 = gimple_assign_rhs1 (d1); + tree p0 = TREE_OPERAND (ref0, 0); + tree p1 = TREE_OPERAND (ref1, 0); + + /* Build P' = PHI <P, Q> and the single load result = *P'. */ + tree res = gimple_phi_result (phi); + tree ptr = make_ssa_name (TREE_TYPE (p0)); + gphi *pphi = create_phi_node (ptr, merge); + add_phi_arg (pphi, p0, e0, gimple_phi_arg_location (phi, e0->dest_idx)); + add_phi_arg (pphi, p1, e1, gimple_phi_arg_location (phi, e1->dest_idx)); + + /* Merge the two arms' TBAA info as get_alias_type_for_stmts does: keep the + common alias-ptr type and dependence clique/base when the arms agree, + otherwise fall back to ptr_type_node (alias-everything) and drop the + clique/base, so the combined load conservatively conflicts with any store + either original arm could. */ + unsigned short clique = MR_DEPENDENCE_CLIQUE (ref0); + unsigned short base = MR_DEPENDENCE_BASE (ref0); + if (clique != MR_DEPENDENCE_CLIQUE (ref1) || base != MR_DEPENDENCE_BASE (ref1)) + clique = base = 0; + tree atype = TREE_TYPE (TREE_OPERAND (ref0, 1)); + if (!alias_ptr_types_compatible_p (atype, TREE_TYPE (TREE_OPERAND (ref1, 1)))) + { + atype = ptr_type_node; + clique = base = 0; + } + + /* Build the combined load RES = *PTR, reusing the PHI result so any range + info on it is preserved (as factor_out_conditional_operation does). */ + tree nref = build2 (MEM_REF, TREE_TYPE (ref0), ptr, build_int_cst (atype, 0)); + MR_DEPENDENCE_CLIQUE (nref) = clique; + MR_DEPENDENCE_BASE (nref) = base; + gassign *load = gimple_build_assign (res, nref); + gimple_set_vuse (load, gimple_vuse (d0)); + gimple_stmt_iterator gsi = gsi_after_labels (merge); + gsi_insert_before (&gsi, load, GSI_SAME_STMT); + + /* RES is now defined by the load; drop the original PHI. */ + gsi = gsi_for_stmt (phi); + gsi_remove (&gsi, true); + + /* The two arm loads are now dead. */ + remove_factored_arm_defs (d0, d1); + + statistics_counter_event (cfun, "factored load out of COND_EXPR", 1); + return true; +} + /* PR66726: Factor operations out of COND_EXPR. If the arguments of the PHI stmt are Unary operator, factor out the operation and perform the operation to the result of PHI stmt. COND_STMT is the controlling predicate. diff --git a/gcc/tree-ssa-phiopt.h b/gcc/tree-ssa-phiopt.h index bba3136965d..3bebcee6fb4 100644 --- a/gcc/tree-ssa-phiopt.h +++ b/gcc/tree-ssa-phiopt.h @@ -21,4 +21,13 @@ along with GCC; see the file COPYING3. If not see extern int find_different_opnum (const gimple_match_op &arg0_op, const gimple_match_op &arg1_op, tree *new_arg0, tree *new_arg1); + +/* True if factor_out_conditional_load would factor the load PHI at MERGE; lets + the sink pass gate on commonability without mutating. */ +extern bool conditional_load_factorable_p (edge e0, edge e1, basic_block merge, + gphi *phi); +/* Factor a load out of a "load PHI" PHI <*P, *Q> into P' = PHI <P, Q>; + result = *P'. */ +extern bool factor_out_conditional_load (edge e0, edge e1, basic_block merge, + gphi *phi); #endif -- 2.50.1 (Apple Git-155)
