gcc/ChangeLog:
* config/aarch64/aarch64-acle-builtins.cc
(gimple_folder::fold_contiguous_load): New function.
* config/aarch64/aarch64-acle-builtins.h
(gimple_folder::fold_contiguous_load): New function.
* config/aarch64/aarch64-sve-builtins-base.cc:
(svld1_impl::fold): Change to use fold_contiguous_load.
(svldxf1_extend_impl::fold): Change to use fold_contiguous_load.
* config/aarch64/aarch64-sve.md
(aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SVE_PARTIAL_I:mode>):
Add else operand.
Co-Authored-By: Alex Coplan <[email protected]>
---
gcc/config/aarch64/aarch64-acle-builtins.cc | 24 +++++++++++++++++++
gcc/config/aarch64/aarch64-acle-builtins.h | 1 +
.../aarch64/aarch64-sve-builtins-base.cc | 20 ++--------------
gcc/config/aarch64/aarch64-sve.md | 1 +
4 files changed, 28 insertions(+), 18 deletions(-)
diff --git a/gcc/config/aarch64/aarch64-acle-builtins.cc
b/gcc/config/aarch64/aarch64-acle-builtins.cc
index 98ba6c8d347..cf746171b4c 100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.cc
+++ b/gcc/config/aarch64/aarch64-acle-builtins.cc
@@ -2850,6 +2850,30 @@ gimple_folder::fold_contiguous_base (gimple_seq &stmts,
tree vectype)
return base;
}
+/* Common code to fold a contiguous load intrinsic call into a gimple
+ call to IFN_CODE. */
+gimple *
+gimple_folder::fold_contiguous_load (internal_fn ifn_code)
+{
+ if (vectors_per_tuple () != 1)
+ return nullptr;
+
+ tree vectype = vector_type (0);
+
+ /* Get the predicate and base pointer. */
+ gimple_seq stmts = NULL;
+ tree pred = convert_pred (stmts, vectype, 0);
+ tree base = fold_contiguous_base (stmts, vectype);
+ tree els = build_zero_cst (vectype);
+ gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+
+ tree cookie = load_store_cookie (TREE_TYPE (vectype));
+ gcall *new_call = gimple_build_call_internal (ifn_code, 4,
+ base, cookie, pred, els);
+ gimple_call_set_lhs (new_call, lhs);
+ return new_call;
+}
+
/* Return the alignment and TBAA argument to an internal load or store
function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses
memory elements of type TYPE. */
diff --git a/gcc/config/aarch64/aarch64-acle-builtins.h
b/gcc/config/aarch64/aarch64-acle-builtins.h
index 42150a2632b..4ce860450bb 100644
--- a/gcc/config/aarch64/aarch64-acle-builtins.h
+++ b/gcc/config/aarch64/aarch64-acle-builtins.h
@@ -650,6 +650,7 @@ public:
tree force_vector (gimple_seq &, tree, tree);
tree convert_pred (gimple_seq &, tree, unsigned int);
tree fold_contiguous_base (gimple_seq &, tree);
+ gimple *fold_contiguous_load (internal_fn);
tree load_store_cookie (tree);
gcall *redirect_call (const function_instance &);
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index 5d01d875a9d..800541a9262 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -1704,23 +1704,7 @@ public:
gimple *
fold (gimple_folder &f) const override
{
- if (f.vectors_per_tuple () != 1)
- return nullptr;
-
- tree vectype = f.vector_type (0);
-
- /* Get the predicate and base pointer. */
- gimple_seq stmts = NULL;
- tree pred = f.convert_pred (stmts, vectype, 0);
- tree base = f.fold_contiguous_base (stmts, vectype);
- tree els = build_zero_cst (vectype);
- gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT);
-
- tree cookie = f.load_store_cookie (TREE_TYPE (vectype));
- gcall *new_call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
- base, cookie, pred, els);
- gimple_call_set_lhs (new_call, f.lhs);
- return new_call;
+ return f.fold_contiguous_load (IFN_MASK_LOAD);
}
rtx
@@ -2108,7 +2092,7 @@ public:
insn_code icode = code_for_aarch64_ldf1 (m_unspec, extend_rtx_code (),
e.vector_mode (0),
e.memory_vector_mode ());
- return e.use_contiguous_load_insn (icode);
+ return e.use_contiguous_load_insn (icode, true);
}
/* The unspec associated with the load. */
diff --git a/gcc/config/aarch64/aarch64-sve.md
b/gcc/config/aarch64/aarch64-sve.md
index 3f59aa77437..878dadf6f61 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -1487,6 +1487,7 @@ (define_insn_and_rewrite
"@aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SV
(unspec:SVE_PARTIAL_I
[(match_operand:<SVE_PARTIAL_I:VPRED> 2 "register_operand" "Upl")
(match_operand:SVE_PARTIAL_I 1 "aarch64_sve_ld<fn>f1_operand"
"Ut<fn>")
+ (match_operand:SVE_PARTIAL_I 4 "aarch64_maskload_else_operand")
(reg:VNx16BI FFRT_REGNUM)]
SVE_LDFF1_LDNF1))]
UNSPEC_PRED_X))]
--
2.43.0
