The following is a start to how we handle gather/scatter, in
particular at this point we are re-doing stmt-based analysis
at many points but with SLP we're doing that only on the
representative for later analyses. The following does a first
step in trying to remember the dataref analysis result
(and with the patch verify that does not change in unexpected
ways). The 2nd step would elide the repeated calls, instead
access the available info directly.
Bootstrapped and tested on x86_64-unknown-linux-gnu.
Note this is for the CI to catch non-x86 (aka IFN) issues, also
due to low coverage of gather/scatter.
* tree-vectorizer.h (_stmt_vec_info::gs_info): New field.
* tree-vectorizer.cc (vec_info::new_stmt_vec_info): Initialize it.
(vec_info::free_stmt_vec_info): Free allocated info.
* tree-vect-data-refs.cc (vect_check_gather_scatter): Record
analysis result and verify it does not change in unexpected
ways.
---
gcc/tree-vect-data-refs.cc | 25 +++++++++++++++++++++++++
gcc/tree-vectorizer.cc | 3 +++
gcc/tree-vectorizer.h | 3 +++
3 files changed, 31 insertions(+)
diff --git a/gcc/tree-vect-data-refs.cc b/gcc/tree-vect-data-refs.cc
index c84cd29116e..20f4f39ab10 100644
--- a/gcc/tree-vect-data-refs.cc
+++ b/gcc/tree-vect-data-refs.cc
@@ -4873,6 +4873,31 @@ vect_check_gather_scatter (stmt_vec_info stmt_info,
loop_vec_info loop_vinfo,
info->scale = scale;
info->element_type = TREE_TYPE (vectype);
info->memory_type = memory_type;
+
+ if (!stmt_info->gs_info)
+ {
+ stmt_info->gs_info = XCNEW (gather_scatter_info);
+ memcpy (stmt_info->gs_info, info, sizeof (gather_scatter_info));
+ }
+ else
+ {
+ if (info->ifn != stmt_info->gs_info->ifn
+ || info->decl != stmt_info->gs_info->decl
+ /* ??? We are building an expression above. */
+ || !operand_equal_p (info->base, stmt_info->gs_info->base)
+ /* ??? When analyzing the loop for epilogue vectorization
+ we record the main loop offset def but at transform time
+ we re-do this on the loop copy. */
+ || (!LOOP_VINFO_EPILOGUE_P (loop_vinfo)
+ && info->offset != stmt_info->gs_info->offset)
+ || info->scale != stmt_info->gs_info->scale
+ || info->offset_dt != stmt_info->gs_info->offset_dt
+ || info->offset_vectype != stmt_info->gs_info->offset_vectype
+ || info->element_type != stmt_info->gs_info->element_type
+ || info->memory_type != stmt_info->gs_info->memory_type)
+ gcc_unreachable ();
+ memcpy (info, stmt_info->gs_info, sizeof (gather_scatter_info));
+ }
return true;
}
diff --git a/gcc/tree-vectorizer.cc b/gcc/tree-vectorizer.cc
index 2f77e46ba99..89fecd78088 100644
--- a/gcc/tree-vectorizer.cc
+++ b/gcc/tree-vectorizer.cc
@@ -739,6 +739,7 @@ vec_info::new_stmt_vec_info (gimple *stmt)
/* This is really "uninitialized" until vect_compute_data_ref_alignment. */
res->dr_aux.misalignment = DR_MISALIGNMENT_UNINITIALIZED;
+ res->gs_info = NULL;
return res;
}
@@ -792,6 +793,8 @@ vec_info::free_stmt_vec_info (stmt_vec_info stmt_info)
stmt_info->reduc_scalar_results.release ();
STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
STMT_VINFO_VEC_STMTS (stmt_info).release ();
+ if (stmt_info->gs_info)
+ free (stmt_info->gs_info);
free (stmt_info);
}
diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h
index 799d5fed7a9..a811e009835 100644
--- a/gcc/tree-vectorizer.h
+++ b/gcc/tree-vectorizer.h
@@ -1368,6 +1368,9 @@ public:
dr_vec_info dr_aux;
+ /* Gather-scatter info, this is present when STMT_VINFO_GATHER_SCATTER_P. */
+ struct gather_scatter_info *gs_info;
+
/* Information about the data-ref relative to this loop
nest (the loop that is being considered for vectorization). */
innermost_loop_behavior dr_wrt_vec_loop;
--
2.43.0
