Add internal functions and optabs for:
- First-faulting loads
- Non-faulting loads
- Set ffr state
- Read ffr state
This also adds an implicit idea of "global ff state" at the gimple
level, through VDEF's and VUSE's.
Remove ldff1 and ldnf1 from pfalse-load.c test, as the FFR state change
(VDEF) can no longer be removed.
gcc/ChangeLog:
* config/aarch64/aarch64-sve-builtins-base.cc (svld1_impl::fold):
Use fold_continuous_load.
(svldxf1_impl): Add LDFF/LDNF flag, and add folding to use
IFN_MASK_FIRSTFAULT_LOAD.
(svldxf1_extend_impl): Change fold to add else operand.
(svrdffr_impl): Add fold to IFN_READ_FAULT_STATE and update
expand.
(svsetffr_impl): Add fold to IFN_SET_FAULT_STATE and update
expand.
* config/aarch64/aarch64-sve-builtins.cc
(gimple_folder::fold_contiguous_load): New function.
* config/aarch64/aarch64-sve-builtins.h:
(gimple_folder::fold_contiguous_load): New function.
* config/aarch64/aarch64-sve.md (aarch64_wrffr): Remove.
(@set_first_fault_state<mode>): New instruction.
(aarch64_rdffr<mode>): Add mode.
(read_first_fault_state<mode>): New expand.
(aarch64_rdffr_z<mode>): Add mode.
(*aarch64_rdffr_z_ptest<mode>): Add mode.
(*aarch64_rdffr_ptest<mode>): Add mode.
(*aarch64_rdffr_z_cc<mode>): Add mode.
(*aarch64_rdffr_cc<mode>): Add mode.
(aarch64_ld<fn>f1<mode>): Add support for sparse
vectors via extending loads.
(mask_<nonfirst>fault_load<mode><vpred>): New expand.
(aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SVE_PARTIAL_I:mode>):
Add else operand.
* config/aarch64/iterators.md (nonfirst): New iterator.
* internal-fn.cc (read_first_fault_state_direct): New macro.
(set_first_fault_state_direct): New macro.
(expand_read_first_fault_state_optab_fn): New macro.
(expand_set_first_fault_state_optab_fn): New function.
(direct_read_first_fault_state_optab_supported_p): New macro.
(direct_set_first_fault_state_optab_supported_p): New macro.
(internal_load_fn_p): Add IFN_MASK_FIRSTFAULT_LOAD and
IFN_MASK_NONFAULT_LOAD.
(internal_fn_else_index): Ditto.
(internal_fn_mask_index): Ditto.
(internal_fn_alias_ptr_index): Ditto,
* internal-fn.def (SET_FAlLT_STATE): New internal function.
(MASK_NONFAULT_LOAD): Ditto.
(MASK_FIRSTFAULT_LOAD): Ditto
(READ_FAULT_STATE): Ditto
* optabs.def (mask_nonfault_load_optab): New optab
(mask_firstfault_load_optab): Ditto.
(read_first_fault_state_optab): Ditto.
(set_first_fault_state_optab): Ditto.
* tree-data-ref.cc (get_references_in_stmt): Add
IFN_MASK_FIRSTFAULT_LOAD and IFN_MASK_NONFAULT_LOAD.
* tree-ssa-alias.cc (ref_maybe_used_by_call_p_1): Ditto.
* tree-ssa-loop-ivopts.cc (get_alias_ptr_type_for_ptr_address):
Ditto.
* tree-vect-slp.cc (vect_get_operand_map): Ditto.
gcc/testsuite/ChangeLog:
* gcc.target/aarch64/sve/pfalse-load.c: Remove ldff1 and ldnf1
tests.
Co-Authored-By: Alex Coplan <[email protected]>
---
.../aarch64/aarch64-sve-builtins-base.cc | 78 +++----
gcc/config/aarch64/aarch64-sve-builtins.cc | 24 +++
gcc/config/aarch64/aarch64-sve-builtins.h | 1 +
gcc/config/aarch64/aarch64-sve.md | 201 +++++++++++++++---
gcc/config/aarch64/iterators.md | 2 +
gcc/internal-fn.cc | 24 +++
gcc/internal-fn.def | 18 ++
gcc/optabs.def | 5 +
.../gcc.target/aarch64/sve/pfalse-load.c | 6 +-
gcc/tree-data-ref.cc | 4 +
gcc/tree-ssa-alias.cc | 2 +
gcc/tree-ssa-loop-ivopts.cc | 2 +
gcc/tree-vect-slp.cc | 4 +
13 files changed, 307 insertions(+), 64 deletions(-)
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
index 1fa7473283d..9d0f8fde268 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
@@ -1682,23 +1682,7 @@ public:
gimple *
fold (gimple_folder &f) const override
{
- if (f.vectors_per_tuple () != 1)
- return nullptr;
-
- tree vectype = f.vector_type (0);
-
- /* Get the predicate and base pointer. */
- gimple_seq stmts = NULL;
- tree pred = f.convert_pred (stmts, vectype, 0);
- tree base = f.fold_contiguous_base (stmts, vectype);
- tree els = build_zero_cst (vectype);
- gsi_insert_seq_before (f.gsi, stmts, GSI_SAME_STMT);
-
- tree cookie = f.load_store_cookie (TREE_TYPE (vectype));
- gcall *new_call = gimple_build_call_internal (IFN_MASK_LOAD, 4,
- base, cookie, pred, els);
- gimple_call_set_lhs (new_call, f.lhs);
- return new_call;
+ return f.fold_contiguous_load (IFN_MASK_LOAD);
}
rtx
@@ -2040,7 +2024,10 @@ public:
class svldxf1_impl : public full_width_access
{
public:
- CONSTEXPR svldxf1_impl (int unspec) : m_unspec (unspec) {}
+ CONSTEXPR svldxf1_impl (int unspec) : m_first (unspec == UNSPEC_LDFF1)
+ {
+ gcc_checking_assert (m_first || unspec == UNSPEC_LDNF1);
+ }
unsigned int
call_properties (const function_instance &) const override
@@ -2048,19 +2035,23 @@ public:
return CP_READ_MEMORY | CP_READ_FFR | CP_WRITE_FFR;
}
- rtx
- expand (function_expander &e) const override
+ gimple *fold (gimple_folder &f) const override
{
- /* See the block comment in aarch64-sve.md for details about the
- FFR handling. */
- emit_insn (gen_aarch64_update_ffr_for_load ());
+ auto ifn_code = m_first ? IFN_MASK_FIRSTFAULT_LOAD :
IFN_MASK_NONFAULT_LOAD;
+ return f.fold_contiguous_load (ifn_code);
+ }
- machine_mode mode = e.vector_mode (0);
- return e.use_contiguous_load_insn (code_for_aarch64_ldf1 (m_unspec, mode));
+ rtx expand (function_expander &e) const override
+ {
+ auto optab
+ = m_first ? mask_firstfault_load_optab : mask_nonfault_load_optab;
+ auto icode = convert_optab_handler (optab,
+ e.vector_mode (0),
+ e.gp_mode (0));
+ return e.use_contiguous_load_insn (icode, true);
}
- /* The unspec associated with the load. */
- int m_unspec;
+ bool m_first;
};
/* Implements extending contiguous forms of svldff1 and svldnf1. */
@@ -2086,7 +2077,7 @@ public:
insn_code icode = code_for_aarch64_ldf1 (m_unspec, extend_rtx_code (),
e.vector_mode (0),
e.memory_vector_mode ());
- return e.use_contiguous_load_insn (icode);
+ return e.use_contiguous_load_insn (icode, true);
}
/* The unspec associated with the load. */
@@ -2842,15 +2833,24 @@ public:
return CP_READ_FFR;
}
- rtx
- expand (function_expander &e) const override
+ gimple *fold (gimple_folder &f) const override
+ {
+ if (f.pred == PRED_z)
+ return NULL;
+
+ gcall *new_call = gimple_build_call_internal (IFN_READ_FAULT_STATE, 0);
+ gimple_call_set_lhs (new_call, f.lhs);
+ return new_call;
+ }
+
+ rtx expand (function_expander &e) const override
{
/* See the block comment in aarch64-sve.md for details about the
FFR handling. */
emit_insn (gen_aarch64_copy_ffr_to_ffrt ());
rtx result = e.use_exact_insn (e.pred == PRED_z
- ? CODE_FOR_aarch64_rdffr_z
- : CODE_FOR_aarch64_rdffr);
+ ? CODE_FOR_aarch64_rdffr_zvnx16bi
+ : CODE_FOR_aarch64_rdffrvnx16bi);
emit_insn (gen_aarch64_update_ffrt ());
return result;
}
@@ -3035,11 +3035,17 @@ public:
return CP_WRITE_FFR;
}
- rtx
- expand (function_expander &e) const override
+ gimple *fold (gimple_folder &f ATTRIBUTE_UNUSED) const override
+ {
+ tree bool_type = acle_vector_types[0][VECTOR_TYPE_svbool_t];
+ return gimple_build_call_internal (IFN_SET_FAULT_STATE, 1,
+ build_all_ones_cst (bool_type));
+ }
+
+ rtx expand (function_expander &e) const override
{
e.args.quick_push (CONSTM1_RTX (VNx16BImode));
- return e.use_exact_insn (CODE_FOR_aarch64_wrffr);
+ return e.use_exact_insn (code_for_set_first_fault_state (VNx16BImode));
}
};
@@ -3465,7 +3471,7 @@ public:
rtx
expand (function_expander &e) const override
{
- return e.use_exact_insn (CODE_FOR_aarch64_wrffr);
+ return e.use_exact_insn (code_for_set_first_fault_state (VNx16BImode));
}
};
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc
b/gcc/config/aarch64/aarch64-sve-builtins.cc
index cd78713440f..6ffc2ef4273 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins.cc
@@ -3602,6 +3602,30 @@ gimple_folder::fold_contiguous_base (gimple_seq &stmts,
tree vectype)
return base;
}
+/* Common code to fold a contiguous load intrinsic call into a gimple
+ call to IFN_CODE. */
+gimple *
+gimple_folder::fold_contiguous_load (internal_fn ifn_code)
+{
+ if (vectors_per_tuple () != 1)
+ return nullptr;
+
+ tree vectype = vector_type (0);
+
+ /* Get the predicate and base pointer. */
+ gimple_seq stmts = NULL;
+ tree pred = convert_pred (stmts, vectype, 0);
+ tree base = fold_contiguous_base (stmts, vectype);
+ tree els = build_zero_cst (vectype);
+ gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
+
+ tree cookie = load_store_cookie (TREE_TYPE (vectype));
+ gcall *new_call = gimple_build_call_internal (ifn_code, 4,
+ base, cookie, pred, els);
+ gimple_call_set_lhs (new_call, lhs);
+ return new_call;
+}
+
/* Return the alignment and TBAA argument to an internal load or store
function like IFN_MASK_LOAD or IFN_MASK_STORE, given that it accesses
memory elements of type TYPE. */
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.h
b/gcc/config/aarch64/aarch64-sve-builtins.h
index f9e8c4cd729..95de95df824 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.h
+++ b/gcc/config/aarch64/aarch64-sve-builtins.h
@@ -650,6 +650,7 @@ public:
tree force_vector (gimple_seq &, tree, tree);
tree convert_pred (gimple_seq &, tree, unsigned int);
tree fold_contiguous_base (gimple_seq &, tree);
+ gimple *fold_contiguous_load (internal_fn);
tree load_store_cookie (tree);
gcall *redirect_call (const function_instance &);
diff --git a/gcc/config/aarch64/aarch64-sve.md
b/gcc/config/aarch64/aarch64-sve.md
index 4d67ad0dc87..2a1849f1ab2 100644
--- a/gcc/config/aarch64/aarch64-sve.md
+++ b/gcc/config/aarch64/aarch64-sve.md
@@ -1131,9 +1131,23 @@ (define_insn_and_split "mov<mode>"
;; [W1 in the block comment above about FFR handling]
;;
;; Write to the FFR and start a new FFRT scheduling region.
-(define_insn "aarch64_wrffr"
+(define_insn "@set_first_fault_state<mode>"
[(set (reg:VNx16BI FFR_REGNUM)
- (match_operand:VNx16BI 0 "aarch64_simd_reg_or_minus_one"))
+ (match_operand:VNx16BI_ONLY 0 "aarch64_simd_reg_or_minus_one"))
+ (set (reg:VNx16BI FFRT_REGNUM)
+ (unspec:VNx16BI [(match_dup 0)] UNSPEC_WRFFR))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {@ [ cons: 0 ]
+ [ Dm ] setffr
+ [ Upa ] wrffr\t%0.b
+ }
+ [(set_attr "sve_type" "sve_ffr")]
+)
+
+(define_insn "@set_first_fault_state<mode>"
+ [(set (reg:VNx16BI FFR_REGNUM)
+ (subreg:VNx16BI
+ (match_operand:PRED_HSD 0 "aarch64_simd_reg_or_minus_one") 0))
(set (reg:VNx16BI FFRT_REGNUM)
(unspec:VNx16BI [(match_dup 0)] UNSPEC_WRFFR))]
"TARGET_SVE && TARGET_NON_STREAMING"
@@ -1176,20 +1190,63 @@ (define_insn "aarch64_copy_ffr_to_ffrt"
;; [R2 in the block comment above about FFR handling]
;;
;; Read the FFR via the FFRT.
-(define_insn "aarch64_rdffr"
- [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
+(define_insn "aarch64_rdffr<mode>"
+ [(set (match_operand:VNx16BI_ONLY 0 "register_operand" "=Upa")
(reg:VNx16BI FFRT_REGNUM))]
"TARGET_SVE && TARGET_NON_STREAMING"
"rdffr\t%0.b"
[(set_attr "sve_type" "sve_ffr")]
)
+(define_insn "aarch64_rdffr<mode>"
+ [(set (match_operand:PRED_HSD 0 "register_operand" "=Upa")
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ "rdffr\t%0.b"
+ [(set_attr "sve_type" "sve_ffr")]
+)
+
+;; [R2 in the block comment above about FFR handling]
+;;
+(define_expand "read_first_fault_state<mode>"
+ [(set (reg:VNx16BI FFRT_REGNUM) ; copy_ffr_to_ffrt
+ (reg:VNx16BI FFR_REGNUM))
+ (set (match_operand:VNx16BI_ONLY 0 "register_operand" "=Upa") ; rdffr
+ (reg:VNx16BI FFRT_REGNUM))
+ (set (reg:VNx16BI FFRT_REGNUM) ; update ffrt
+ (unspec:VNx16BI [(reg:VNx16BI FFRT_REGNUM)] UNSPEC_UPDATE_FFRT))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+)
+
+(define_expand "read_first_fault_state<mode>"
+ [(set (reg:VNx16BI FFRT_REGNUM) ; copy_ffr_to_ffrt
+ (reg:VNx16BI FFR_REGNUM))
+ (set (match_operand:PRED_HSD 0 "register_operand" "=Upa") ; rdffr
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0))
+ (set (reg:VNx16BI FFRT_REGNUM) ; update ffrt
+ (unspec:VNx16BI [(reg:VNx16BI FFRT_REGNUM)] UNSPEC_UPDATE_FFRT))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+)
+
;; Likewise with zero predication.
-(define_insn "aarch64_rdffr_z"
- [(set (match_operand:VNx16BI 0 "register_operand")
- (and:VNx16BI
+(define_insn "aarch64_rdffr_z<mode>"
+ [(set (match_operand:VNx16BI_ONLY 0 "register_operand")
+ (and:VNx16BI_ONLY
(reg:VNx16BI FFRT_REGNUM)
- (match_operand:VNx16BI 1 "register_operand")))]
+ (match_operand:VNx16BI_ONLY 1 "register_operand")))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {@ [ cons: =0, 1 ; attrs: pred_clobber ]
+ [ &Upa , Upa ; yes ] rdffr\t%0.b, %1/z
+ [ ?Upa , 0Upa; yes ] ^
+ [ Upa , Upa ; no ] ^
+ }
+ [(set_attr "sve_type" "sve_ffr")]
+)
+(define_insn "aarch64_rdffr_z<mode>"
+ [(set (match_operand:PRED_HSD 0 "register_operand")
+ (and:PRED_HSD
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
+ (match_operand:PRED_HSD 1 "register_operand")))]
"TARGET_SVE && TARGET_NON_STREAMING"
{@ [ cons: =0, 1 ; attrs: pred_clobber ]
[ &Upa , Upa ; yes ] rdffr\t%0.b, %1/z
@@ -1200,17 +1257,36 @@ (define_insn "aarch64_rdffr_z"
)
;; Read the FFR to test for a fault, without using the predicate result.
-(define_insn "*aarch64_rdffr_z_ptest"
+(define_insn "*aarch64_rdffr_z_ptest<mode>"
[(set (reg:CC_NZC CC_REGNUM)
(unspec:CC_NZC
[(match_operand:VNx16BI 1 "register_operand")
(match_dup 1)
(match_operand:SI 2 "aarch64_sve_ptrue_flag")
- (and:VNx16BI
+ (and:VNx16BI_ONLY
(reg:VNx16BI FFRT_REGNUM)
(match_dup 1))]
UNSPEC_PTEST))
- (clobber (match_scratch:VNx16BI 0))]
+ (clobber (match_scratch:VNx16BI_ONLY 0))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {@ [ cons: =0, 1 ; attrs: pred_clobber ]
+ [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
+ [ ?Upa , 0Upa; yes ] ^
+ [ Upa , Upa ; no ] ^
+ }
+ [(set_attr "sve_type" "sve_ffr")]
+)
+(define_insn "*aarch64_rdffr_z_ptest<mode>"
+ [(set (reg:CC_NZC CC_REGNUM)
+ (unspec:CC_NZC
+ [(match_operand:VNx16BI 1 "register_operand")
+ (match_operand:PRED_HSD 3)
+ (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+ (and:PRED_HSD
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
+ (match_dup 3))]
+ UNSPEC_PTEST))
+ (clobber (match_scratch:PRED_HSD 0))]
"TARGET_SVE && TARGET_NON_STREAMING"
{@ [ cons: =0, 1 ; attrs: pred_clobber ]
[ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
@@ -1221,15 +1297,32 @@ (define_insn "*aarch64_rdffr_z_ptest"
)
;; Same for unpredicated RDFFR when tested with a known PTRUE.
-(define_insn "*aarch64_rdffr_ptest"
+(define_insn "*aarch64_rdffr_ptest<mode>"
[(set (reg:CC_NZC CC_REGNUM)
(unspec:CC_NZC
[(match_operand:VNx16BI 1 "register_operand")
- (match_dup 1)
+ (match_operand:VNx16BI_ONLY 2)
(const_int SVE_KNOWN_PTRUE)
(reg:VNx16BI FFRT_REGNUM)]
UNSPEC_PTEST))
- (clobber (match_scratch:VNx16BI 0))]
+ (clobber (match_scratch:VNx16BI_ONLY 0))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {@ [ cons: =0, 1 ; attrs: pred_clobber ]
+ [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
+ [ ?Upa , 0Upa; yes ] ^
+ [ Upa , Upa ; no ] ^
+ }
+ [(set_attr "sve_type" "sve_ffr")]
+)
+(define_insn "*aarch64_rdffr_ptest<mode>"
+ [(set (reg:CC_NZC CC_REGNUM)
+ (unspec:CC_NZC
+ [(match_operand:VNx16BI 1 "register_operand")
+ (match_operand:PRED_HSD 2)
+ (const_int SVE_KNOWN_PTRUE)
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)]
+ UNSPEC_PTEST))
+ (clobber (match_scratch:PRED_HSD 0))]
"TARGET_SVE && TARGET_NON_STREAMING"
{@ [ cons: =0, 1 ; attrs: pred_clobber ]
[ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
@@ -1240,18 +1333,18 @@ (define_insn "*aarch64_rdffr_ptest"
)
;; Read the FFR with zero predication and test the result.
-(define_insn "*aarch64_rdffr_z_cc"
+(define_insn "*aarch64_rdffr_z_cc<mode>"
[(set (reg:CC_NZC CC_REGNUM)
(unspec:CC_NZC
[(match_operand:VNx16BI 1 "register_operand")
(match_dup 1)
(match_operand:SI 2 "aarch64_sve_ptrue_flag")
- (and:VNx16BI
+ (and:VNx16BI_ONLY
(reg:VNx16BI FFRT_REGNUM)
(match_dup 1))]
UNSPEC_PTEST))
- (set (match_operand:VNx16BI 0 "register_operand")
- (and:VNx16BI
+ (set (match_operand:VNx16BI_ONLY 0 "register_operand")
+ (and:VNx16BI_ONLY
(reg:VNx16BI FFRT_REGNUM)
(match_dup 1)))]
"TARGET_SVE && TARGET_NON_STREAMING"
@@ -1262,9 +1355,31 @@ (define_insn "*aarch64_rdffr_z_cc"
}
[(set_attr "sve_type" "sve_ffr")]
)
+(define_insn "*aarch64_rdffr_z_cc<mode>"
+ [(set (reg:CC_NZC CC_REGNUM)
+ (unspec:CC_NZC
+ [(match_operand:VNx16BI 1 "register_operand")
+ (match_operand:PRED_HSD 3)
+ (match_operand:SI 2 "aarch64_sve_ptrue_flag")
+ (and:PRED_HSD
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
+ (match_dup 3))]
+ UNSPEC_PTEST))
+ (set (match_operand:PRED_HSD 0 "register_operand")
+ (and:PRED_HSD
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
+ (match_dup 3)))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {@ [ cons: =0, 1 ; attrs: pred_clobber ]
+ [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
+ [ ?Upa , 0Upa; yes ] ^
+ [ Upa , Upa ; no ] ^
+ }
+ [(set_attr "sve_type" "sve_ffr")]
+)
;; Same for unpredicated RDFFR when tested with a known PTRUE.
-(define_insn "*aarch64_rdffr_cc"
+(define_insn "*aarch64_rdffr_cc<mode>"
[(set (reg:CC_NZC CC_REGNUM)
(unspec:CC_NZC
[(match_operand:VNx16BI 1 "register_operand")
@@ -1272,7 +1387,7 @@ (define_insn "*aarch64_rdffr_cc"
(const_int SVE_KNOWN_PTRUE)
(reg:VNx16BI FFRT_REGNUM)]
UNSPEC_PTEST))
- (set (match_operand:VNx16BI 0 "register_operand")
+ (set (match_operand:VNx16BI_ONLY 0 "register_operand")
(reg:VNx16BI FFRT_REGNUM))]
"TARGET_SVE && TARGET_NON_STREAMING"
{@ [ cons: =0, 1 ; attrs: pred_clobber ]
@@ -1282,6 +1397,24 @@ (define_insn "*aarch64_rdffr_cc"
}
[(set_attr "sve_type" "sve_ffr")]
)
+(define_insn "*aarch64_rdffr_cc<mode>"
+ [(set (reg:CC_NZC CC_REGNUM)
+ (unspec:CC_NZC
+ [(match_operand:VNx16BI 1 "register_operand")
+ (match_operand:PRED_HSD 2)
+ (const_int SVE_KNOWN_PTRUE)
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)]
+ UNSPEC_PTEST))
+ (set (match_operand:PRED_HSD 0 "register_operand")
+ (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {@ [ cons: =0, 1 ; attrs: pred_clobber ]
+ [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
+ [ ?Upa , 0Upa; yes ] ^
+ [ Upa , Upa ; no ] ^
+ }
+ [(set_attr "sve_type" "sve_ffr")]
+)
;; [R3 in the block comment above about FFR handling]
;;
@@ -1448,18 +1581,37 @@ (define_insn_and_rewrite
"*aarch64_load_<ANY_EXTEND:optab>_mov<SVE_HSDI:mode><SV
;; -------------------------------------------------------------------------
;; Contiguous non-extending first-faulting or non-faulting loads.
+;; Also supports sparse vectors via extendign loads
(define_insn "@aarch64_ld<fn>f1<mode>"
- [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
- (unspec:SVE_FULL
+ [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
+ (unspec:SVE_ALL
[(match_operand:<VPRED> 2 "register_operand" "Upl")
- (match_operand:SVE_FULL 1 "aarch64_sve_ld<fn>f1_operand" "Ut<fn>")
+ (match_operand:SVE_ALL 1 "aarch64_sve_ld<fn>f1_operand" "Ut<fn>")
+ (match_operand:SVE_ALL 3 "aarch64_maskload_else_operand")
(reg:VNx16BI FFRT_REGNUM)]
SVE_LDFF1_LDNF1))]
"TARGET_SVE && TARGET_NON_STREAMING"
- "ld<fn>f1<Vesize>\t%0.<Vetype>, %2/z, %1"
+ "ld<fn>f1<Vesize>\t%0.<Vctype>, %2/z, %1"
[(set_attr "sve_type" "sve_load_1reg")]
)
+;; Expose {non,first}-faulting loads to the middle-end.
+(define_expand "mask_<nonfirst>fault_load<mode><vpred>"
+ [(set (match_operand:SVE_ALL 0 "register_operand")
+ (unspec:SVE_ALL
+ [(match_operand:<VPRED> 2 "register_operand")
+ (match_operand:SVE_ALL 1 "aarch64_sve_ld<fn>f1_operand")
+ (match_operand:SVE_ALL 3 "aarch64_maskload_else_operand")
+ (reg:VNx16BI FFRT_REGNUM)]
+ SVE_LDFF1_LDNF1))]
+ "TARGET_SVE && TARGET_NON_STREAMING"
+ {
+ /* Emit the L2 insn; L1 follows from the template above.
+ See the earlier block comment on FFR handling. */
+ emit_insn (gen_aarch64_update_ffr_for_load ());
+ }
+)
+
;; -------------------------------------------------------------------------
;; ---- First-faulting extending contiguous loads
;; -------------------------------------------------------------------------
@@ -1487,6 +1639,7 @@ (define_insn_and_rewrite
"@aarch64_ld<fn>f1_<ANY_EXTEND:optab><SVE_HSDI:mode><SV
(unspec:SVE_PARTIAL_I
[(match_operand:<SVE_PARTIAL_I:VPRED> 2 "register_operand" "Upl")
(match_operand:SVE_PARTIAL_I 1 "aarch64_sve_ld<fn>f1_operand"
"Ut<fn>")
+ (match_operand:SVE_PARTIAL_I 4 "aarch64_maskload_else_operand")
(reg:VNx16BI FFRT_REGNUM)]
SVE_LDFF1_LDNF1))]
UNSPEC_PRED_X))]
diff --git a/gcc/config/aarch64/iterators.md b/gcc/config/aarch64/iterators.md
index 2d1522a348a..10871f24221 100644
--- a/gcc/config/aarch64/iterators.md
+++ b/gcc/config/aarch64/iterators.md
@@ -4662,6 +4662,8 @@ (define_int_attr u [(UNSPEC_SQSHLU "u") (UNSPEC_SQSHL "")
(UNSPEC_UQSHL "")
(define_int_attr fn [(UNSPEC_LDFF1 "f") (UNSPEC_LDNF1 "n")])
+(define_int_attr nonfirst [(UNSPEC_LDNF1 "non") (UNSPEC_LDFF1 "first")])
+
(define_int_attr ab [(UNSPEC_CLASTA "a") (UNSPEC_CLASTB "b")
(UNSPEC_LASTA "a") (UNSPEC_LASTB "b")])
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index 0138c6f7ef0..a219d8fe59c 100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -196,6 +196,8 @@ init_internal_fns ()
#define crc_direct { 1, -1, true }
#define reduc_sbool_direct { 0, 0, true }
#define select_vl_direct { 2, 0, false }
+#define read_first_fault_state_direct { -1, -1, false }
+#define set_first_fault_state_direct { 0, 0, false }
const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
#define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
@@ -4315,6 +4317,18 @@ expand_reduc_sbool_optab_fn (internal_fn fn, gcall
*stmt, direct_optab optab)
#define expand_select_vl_optab_fn(FN, STMT, OPTAB) \
expand_convert_optab_fn (FN, STMT, OPTAB, 3)
+#define expand_read_first_fault_state_optab_fn(FN, STMT, OPTAB) \
+ expand_direct_optab_fn (FN, STMT, OPTAB, 0)
+
+static void
+expand_set_first_fault_state_optab_fn (internal_fn fn, gcall *stmt,
+ direct_optab optab)
+{
+ tree_pair types = direct_internal_fn_types (fn, stmt);
+ insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
+ expand_fn_using_insn (stmt, icode, 0, 1);
+}
+
/* Expanders for optabs that can use expand_convert_optab_fn. */
#define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
@@ -4432,6 +4446,8 @@ multi_vector_optab_supported_p (convert_optab optab,
tree_pair types,
#define direct_vec_extract_optab_supported_p convert_optab_supported_p
#define direct_reduc_sbool_optab_supported_p direct_optab_supported_p
#define direct_select_vl_optab_supported_p convert_optab_supported_p
+#define direct_read_first_fault_state_optab_supported_p
direct_optab_supported_p
+#define direct_set_first_fault_state_optab_supported_p direct_optab_supported_p
/* Return the optab used by internal function FN. */
@@ -5081,6 +5097,8 @@ internal_load_fn_p (internal_fn fn)
case IFN_MASK_LEN_GATHER_LOAD:
case IFN_LEN_LOAD:
case IFN_MASK_LEN_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
+ case IFN_MASK_NONFAULT_LOAD:
return true;
default:
@@ -5253,6 +5271,8 @@ internal_fn_else_index (internal_fn fn)
case IFN_MASK_LEN_LOAD:
case IFN_MASK_LOAD_LANES:
case IFN_MASK_LEN_LOAD_LANES:
+ case IFN_MASK_NONFAULT_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
return 3;
case IFN_COND_FMA:
@@ -5293,6 +5313,8 @@ internal_fn_mask_index (internal_fn fn)
case IFN_MASK_LEN_STORE_LANES:
case IFN_MASK_LEN_LOAD:
case IFN_MASK_LEN_STORE:
+ case IFN_MASK_NONFAULT_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
return 2;
case IFN_MASK_LEN_STRIDED_LOAD:
@@ -5361,6 +5383,8 @@ internal_fn_alias_ptr_index (internal_fn fn)
case IFN_SCATTER_STORE:
case IFN_MASK_SCATTER_STORE:
case IFN_MASK_LEN_SCATTER_STORE:
+ case IFN_MASK_NONFAULT_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
return 1;
default:
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index af9f92950c7..b03a50154ea 100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -649,6 +649,24 @@ DEF_INTERNAL_FN (BITINTTOFLOAT, ECF_PURE | ECF_LEAF, ". R
. ")
DEF_INTERNAL_OPTAB_FN (BIT_ANDN, ECF_CONST, andn, binary)
DEF_INTERNAL_OPTAB_FN (BIT_IORN, ECF_CONST, iorn, binary)
+/* First faulting load internal functions. */
+DEF_INTERNAL_OPTAB_FN (MASK_FIRSTFAULT_LOAD,
+ 0,
+ mask_firstfault_load,
+ mask_load)
+DEF_INTERNAL_OPTAB_FN (MASK_NONFAULT_LOAD,
+ ECF_NOTHROW,
+ mask_nonfault_load,
+ mask_load)
+DEF_INTERNAL_OPTAB_FN (SET_FAULT_STATE,
+ ECF_NOTHROW,
+ set_first_fault_state,
+ set_first_fault_state)
+DEF_INTERNAL_OPTAB_FN (READ_FAULT_STATE,
+ ECF_NOTHROW,
+ read_first_fault_state,
+ read_first_fault_state)
+
#undef DEF_INTERNAL_WIDENING_OPTAB_FN
#undef DEF_INTERNAL_SIGNED_COND_FN
#undef DEF_INTERNAL_COND_FN
diff --git a/gcc/optabs.def b/gcc/optabs.def
index 7ccea18543f..a444aec39f3 100644
--- a/gcc/optabs.def
+++ b/gcc/optabs.def
@@ -109,6 +109,8 @@ OPTAB_CD(mask_len_gather_load_optab,
"mask_len_gather_load$a$b")
OPTAB_CD(scatter_store_optab, "scatter_store$a$b")
OPTAB_CD(mask_scatter_store_optab, "mask_scatter_store$a$b")
OPTAB_CD(mask_len_scatter_store_optab, "mask_len_scatter_store$a$b")
+OPTAB_CD (mask_nonfault_load_optab, "mask_nonfault_load$a$b")
+OPTAB_CD (mask_firstfault_load_optab, "mask_firstfault_load$a$b")
OPTAB_CD(vec_extract_optab, "vec_extract$a$b")
OPTAB_CD(vec_init_optab, "vec_init$a$b")
OPTAB_CD (sdot_prod_optab, "sdot_prod$I$a$b")
@@ -238,6 +240,9 @@ OPTAB_D (push_optab, "push$a1")
OPTAB_D (reload_in_optab, "reload_in$a")
OPTAB_D (reload_out_optab, "reload_out$a")
+OPTAB_D (read_first_fault_state_optab, "read_first_fault_state$a")
+OPTAB_D (set_first_fault_state_optab, "set_first_fault_state$a")
+
OPTAB_DC(cbranch_optab, "cbranch$a4", COMPARE)
OPTAB_D (tbranch_eq_optab, "tbranch_eq$a3")
OPTAB_D (tbranch_ne_optab, "tbranch_ne$a3")
diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
index a32b636b278..71a068ec827 100644
--- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
+++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
@@ -24,9 +24,7 @@
T (F##_u32, uint32_t) \
T (F##_u64, uint64_t) \
-ALL_DATA (ldff1)
-ALL_DATA (ldnf1)
ALL_DATA (ldnt1)
-/* { dg-final { scan-assembler-times
{\t.cfi_startproc\n\tmovi?\t[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0\n\tret\n} 36
} } */
-/* { dg-final { scan-assembler-times {\t.cfi_startproc\n} 36 } } */
+/* { dg-final { scan-assembler-times
{\t.cfi_startproc\n\tmovi?\t[vdz]([0-9]+)\.?(?:[0-9]*[bhsd])?, #?0\n\tret\n} 12
} } */
+/* { dg-final { scan-assembler-times {\t.cfi_startproc\n} 12 } } */
diff --git a/gcc/tree-data-ref.cc b/gcc/tree-data-ref.cc
index 1114903784e..ec068524423 100644
--- a/gcc/tree-data-ref.cc
+++ b/gcc/tree-data-ref.cc
@@ -5881,6 +5881,8 @@ get_references_in_stmt (gimple *stmt, vec<data_ref_loc,
va_heap> *references)
break;
}
case IFN_MASK_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
+ case IFN_MASK_NONFAULT_LOAD:
case IFN_MASK_STORE:
break;
case IFN_MASK_CALL:
@@ -5938,6 +5940,8 @@ get_references_in_stmt (gimple *stmt, vec<data_ref_loc,
va_heap> *references)
switch (gimple_call_internal_fn (stmt))
{
case IFN_MASK_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
+ case IFN_MASK_NONFAULT_LOAD:
if (gimple_call_lhs (stmt) == NULL_TREE)
break;
ref.is_read = true;
diff --git a/gcc/tree-ssa-alias.cc b/gcc/tree-ssa-alias.cc
index 9f3dd2adac0..d279908173f 100644
--- a/gcc/tree-ssa-alias.cc
+++ b/gcc/tree-ssa-alias.cc
@@ -2862,6 +2862,8 @@ ref_maybe_used_by_call_p_1 (gcall *call, ao_ref *ref,
bool tbaa_p)
case IFN_MASK_LEN_STORE_LANES:
goto process_args;
case IFN_MASK_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
+ case IFN_MASK_NONFAULT_LOAD:
case IFN_LEN_LOAD:
case IFN_MASK_LEN_LOAD:
case IFN_MASK_LOAD_LANES:
diff --git a/gcc/tree-ssa-loop-ivopts.cc b/gcc/tree-ssa-loop-ivopts.cc
index bbd3cfaab24..42c5238edce 100644
--- a/gcc/tree-ssa-loop-ivopts.cc
+++ b/gcc/tree-ssa-loop-ivopts.cc
@@ -7565,6 +7565,8 @@ get_alias_ptr_type_for_ptr_address (iv_use *use)
switch (gimple_call_internal_fn (call))
{
case IFN_MASK_LOAD:
+ case IFN_MASK_FIRSTFAULT_LOAD:
+ case IFN_MASK_NONFAULT_LOAD:
case IFN_MASK_STORE:
case IFN_MASK_LOAD_LANES:
case IFN_MASK_STORE_LANES:
diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
index 2250f6f74a1..a5b79d7e608 100644
--- a/gcc/tree-vect-slp.cc
+++ b/gcc/tree-vect-slp.cc
@@ -581,6 +581,10 @@ vect_get_operand_map (const gimple *stmt, bool
gather_scatter_p,
case IFN_MASK_LOAD:
return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
+ case IFN_MASK_FIRSTFAULT_LOAD:
+ case IFN_MASK_NONFAULT_LOAD:
+ return arg2_arg3_map;
+
case IFN_GATHER_LOAD:
return arg2_map;
--
2.34.1
