> -----Original Message-----
> From: Alfie Richards <[email protected]>
> Sent: 08 August 2026 00:11
> To: [email protected]
> Cc: Alex Coplan <[email protected]>; Alice Carlotti
> <[email protected]>; [email protected];
> [email protected]; Tamar Christina <[email protected]>;
> [email protected]; [email protected]; [email protected]; Alfie
> Richards <[email protected]>
> Subject: [PATCH v2 04/13] ifn: optab: aarch64: Add optabs and internal
> functions for hssr operations.
>
> Also removes pfalse ldff1 tests as the calls can no longer be removed
> due to virtual operation which represents the HSSR state.
Ah, that's why the refactoring.
Please split off the AArch64 implementation bit from the middle-end machinery.
>
> gcc/ChangeLog:
>
> * config/aarch64/aarch64-sve-builtins-base.cc:
> (svldxf1_impl::fold): Update to use new optabs.
> (svldxf1_impl::expand): Update to use new optabs.
> (aarch64_wrffr::fold): Update to use new IFN's.
> (aarch64_wrffr::expand): Update to use new optabs.
> (svsetffr_impl::fold): Update to use new IFN's.
> (svsetffr_impl::expand): Update to use new optabs.
> (svwrffr_impl::fold): Update to use new optabs.
> (svwrffr_impl::expand): Update to use new IFN's.
> * config/aarch64/aarch64-sve.md
> (@set_hssr_state<mode>): Rename aarch64_wrffr and add new
> modes.
> (aarch64_rdffr<mode>): Update for new modes.
> (aarch64_ld<fn>f1<mode>): Add support for extending loads.
> (mask_ff_hssr_load<mode><vpred>): New define expand.
> * internal-fn.cc (read_hssr_state_direct): New macro.
> (set_hssr_state_direct): New macro.
> (detect_hssr_fault_direct): New macro.
> (expand_read_hssr_state_optab_fn): New macro.
> (expand_detect_hssr_fault_optab_fn): New macro.
> (expand_set_hssr_state_optab_fn): New function.
> (direct_read_hssr_state_optab_supported_p): New macro.
> (direct_set_hssr_state_optab_supported_p): New macro.
> (direct_detect_hssr_fault_optab_supported_p): New macro.
> (internal_load_fn_p): Add support for IFN_MASK_FF_HSSR_LOAD.
> (internal_fn_else_index): Add support for IFN_MASK_FF_HSSR_LOAD.
> (internal_fn_mask_index): Add support for
> IFN_MASK_FF_HSSR_LOAD.
> (internal_fn_alias_ptr_index): Add support for
> IFN_MASK_FF_HSSR_LOAD.
> * internal-fn.def (MASK_FF_HSSR_LOAD): New IFN.
> (SET_HSSR_STATE): New IFN.
> (READ_HSSR_STATE): New IFN.
> (DETECT_HSSR_FAULT): New IFN.
> * optabs.def (mask_ff_hssr_load_optab): New optab.
> (read_hssr_state_optab): New optab.
> (set_hssr_state_optab): New optab.
> (detect_hssr_fault_optab): New optab.
These are missing documentation, it's a bit hard to review this atm since I'm
not
sure what the arguments and constraints for these are.
IFNs need documenting in ifn.texi and the optabs in md.texi
> * tree-data-ref.cc (get_references_in_stmt):
> Add support for IFN_MASK_FF_HSSR_LOAD.
> * tree-ssa-alias.cc (ref_maybe_used_by_call_p_1):
> Add support for IFN_MASK_FF_HSSR_LOAD.
> * tree-ssa-loop-ivopts.cc (get_mem_type_for_internal_fn):
> Add support for IFN_MASK_FF_HSSR_LOAD.
> (get_alias_ptr_type_for_ptr_address):
> Add support for IFN_MASK_FF_HSSR_LOAD.
> * tree-vect-slp.cc (vect_get_operand_map):
> Add support for IFN_MASK_FF_HSSR_LOAD.
>
> gcc/testsuite/ChangeLog:
>
> * gcc.target/aarch64/sve/pfalse-load.c: Remove ldff1 tests.
Why is this? It should still be valid to fold to zero, at least in terms of
ACLE that's a regression.
Why doesn't using something like
gimple *
fold (gimple_folder &f) const override
{
if (!m_first)
return nullptr;
if (is_pfalse (gimple_call_arg (f.call, 0)))
return f.fold_call_to (build_zero_cst (TREE_TYPE (f.lhs)));
return f.fold_contiguous_load (IFN_MASK_FF_HSSR_LOAD);
}
work?
>
> Co-Authored-By: Alex Coplan <[email protected]>
>
> ---
> .../aarch64/aarch64-sve-builtins-base.cc | 74 ++++--
> gcc/config/aarch64/aarch64-sve.md | 221 ++++++++++++++++--
> gcc/internal-fn.cc | 25 ++
> gcc/internal-fn.def | 18 ++
> gcc/optabs.def | 8 +
> .../gcc.target/aarch64/sve/pfalse-load.c | 7 +-
> gcc/tree-data-ref.cc | 2 +
> gcc/tree-ssa-alias.cc | 1 +
> gcc/tree-ssa-loop-ivopts.cc | 2 +
> gcc/tree-vect-slp.cc | 3 +
> 10 files changed, 315 insertions(+), 46 deletions(-)
>
> diff --git a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
> b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
> index 800541a9262..febcfe163c6 100644
> --- a/gcc/config/aarch64/aarch64-sve-builtins-base.cc
> +++ b/gcc/config/aarch64/aarch64-sve-builtins-base.cc
> @@ -2046,7 +2046,10 @@ public:
> class svldxf1_impl : public full_width_access
> {
> public:
> - constexpr svldxf1_impl (unspec unspec) : m_unspec (unspec) {}
> + constexpr svldxf1_impl (unspec unspec) : m_first (unspec ==
> UNSPEC_LDFF1)
> + {
> + gcc_checking_assert (m_first || unspec == UNSPEC_LDNF1);
> + }
>
> unsigned int
> call_properties (const function_instance &) const override
> @@ -2054,19 +2057,33 @@ public:
> return CP_READ_MEMORY | CP_READ_FFR | CP_WRITE_FFR;
> }
>
> - rtx
> - expand (function_expander &e) const override
> + gimple *fold (gimple_folder &f) const override
> {
> - /* See the block comment in aarch64-sve.md for details about the
> - FFR handling. */
> - emit_insn (gen_aarch64_update_ffr_for_load ());
> + if (m_first)
> + return f.fold_contiguous_load (IFN_MASK_FF_HSSR_LOAD);
> + else
> + return nullptr;
> + }
>
> - machine_mode mode = e.vector_mode (0);
> - return e.use_contiguous_load_insn (code_for_aarch64_ldf1 (m_unspec,
> mode));
> + rtx expand (function_expander &e) const override
> + {
> + if (m_first)
> + {
> + auto icode = convert_optab_handler (mask_ff_hssr_load_optab,
> + e.vector_mode (0), e.gp_mode (0));
> + return e.use_contiguous_load_insn (icode, true);
> + }
> + else
> + {
> + emit_insn (gen_aarch64_update_ffr_for_load ());
> + machine_mode mode = e.vector_mode (0);
> + return e.use_contiguous_load_insn (code_for_aarch64_ldf1
> (UNSPEC_LDNF1,
> + mode),
> + true);
> + }
> }
>
> - /* The unspec associated with the load. */
> - unspec m_unspec;
> + bool m_first;
Can we call this something more descriptive.
Otherwise looks good but I'd like to see some docs before continuing to review
the RTL changes.
Thanks,
Tamar
> };
>
> /* Implements extending contiguous forms of svldff1 and svldnf1. */
> @@ -2848,15 +2865,24 @@ public:
> return CP_READ_FFR;
> }
>
> - rtx
> - expand (function_expander &e) const override
> + gimple *fold (gimple_folder &f) const override
> + {
> + if (f.pred == PRED_z)
> + return nullptr;
> +
> + gcall *new_call = gimple_build_call_internal (IFN_READ_HSSR_STATE, 0);
> + gimple_call_set_lhs (new_call, f.lhs);
> + return new_call;
> + }
> +
> + rtx expand (function_expander &e) const override
> {
> /* See the block comment in aarch64-sve.md for details about the
> FFR handling. */
> emit_insn (gen_aarch64_copy_ffr_to_ffrt ());
> rtx result = e.use_exact_insn (e.pred == PRED_z
> - ? CODE_FOR_aarch64_rdffr_z
> - : CODE_FOR_aarch64_rdffr);
> + ? CODE_FOR_aarch64_rdffr_zvnx16bi
> + : CODE_FOR_aarch64_rdffrvnx16bi);
> emit_insn (gen_aarch64_update_ffrt ());
> return result;
> }
> @@ -3041,11 +3067,17 @@ public:
> return CP_WRITE_FFR;
> }
>
> - rtx
> - expand (function_expander &e) const override
> + gimple *fold (gimple_folder &f ATTRIBUTE_UNUSED) const override
> + {
> + tree bool_type = acle_vector_types[0][VECTOR_TYPE_svbool_t];
> + return gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
> + build_all_ones_cst (bool_type));
> + }
> +
> + rtx expand (function_expander &e) const override
> {
> e.args.quick_push (CONSTM1_RTX (VNx16BImode));
> - return e.use_exact_insn (CODE_FOR_aarch64_wrffr);
> + return e.use_exact_insn (code_for_set_hssr_state (VNx16BImode));
> }
> };
>
> @@ -3468,10 +3500,16 @@ public:
> return CP_WRITE_FFR;
> }
>
> + gimple *fold (gimple_folder &f) const override
> + {
> + return gimple_build_call_internal (IFN_SET_HSSR_STATE, 1,
> + gimple_call_arg (f.call, 0));
> + }
> +
> rtx
> expand (function_expander &e) const override
> {
> - return e.use_exact_insn (CODE_FOR_aarch64_wrffr);
> + return e.use_exact_insn (code_for_set_hssr_state (VNx16BImode));
> }
> };
>
> diff --git a/gcc/config/aarch64/aarch64-sve.md
> b/gcc/config/aarch64/aarch64-sve.md
> index 878dadf6f61..eb56bc850b4 100644
> --- a/gcc/config/aarch64/aarch64-sve.md
> +++ b/gcc/config/aarch64/aarch64-sve.md
> @@ -1131,9 +1131,23 @@ (define_insn_and_split "mov<mode>"
> ;; [W1 in the block comment above about FFR handling]
> ;;
> ;; Write to the FFR and start a new FFRT scheduling region.
> -(define_insn "aarch64_wrffr"
> +(define_insn "@set_hssr_state<mode>"
> [(set (reg:VNx16BI FFR_REGNUM)
> - (match_operand:VNx16BI 0 "aarch64_simd_reg_or_minus_one"))
> + (match_operand:VNx16BI_ONLY 0
> "aarch64_simd_reg_or_minus_one"))
> + (set (reg:VNx16BI FFRT_REGNUM)
> + (unspec:VNx16BI [(match_dup 0)] UNSPEC_WRFFR))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {@ [ cons: 0 ]
> + [ Dm ] setffr
> + [ Upa ] wrffr\t%0.b
> + }
> + [(set_attr "sve_type" "sve_ffr")]
> +)
> +
> +(define_insn "@set_hssr_state<mode>"
> + [(set (reg:VNx16BI FFR_REGNUM)
> + (subreg:VNx16BI
> + (match_operand:PRED_HSD 0 "aarch64_simd_reg_or_minus_one")
> 0))
> (set (reg:VNx16BI FFRT_REGNUM)
> (unspec:VNx16BI [(match_dup 0)] UNSPEC_WRFFR))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> @@ -1176,20 +1190,63 @@ (define_insn "aarch64_copy_ffr_to_ffrt"
> ;; [R2 in the block comment above about FFR handling]
> ;;
> ;; Read the FFR via the FFRT.
> -(define_insn "aarch64_rdffr"
> - [(set (match_operand:VNx16BI 0 "register_operand" "=Upa")
> +(define_insn "aarch64_rdffr<mode>"
> + [(set (match_operand:VNx16BI_ONLY 0 "register_operand" "=Upa")
> (reg:VNx16BI FFRT_REGNUM))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> "rdffr\t%0.b"
> [(set_attr "sve_type" "sve_ffr")]
> )
>
> +(define_insn "aarch64_rdffr<mode>"
> + [(set (match_operand:PRED_HSD 0 "register_operand" "=Upa")
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + "rdffr\t%0.b"
> + [(set_attr "sve_type" "sve_ffr")]
> +)
> +
> +;; [R2 in the block comment above about FFR handling]
> +;;
> +(define_expand "read_hssr_state<mode>"
> + [(set (reg:VNx16BI FFRT_REGNUM) ; copy_ffr_to_ffrt
> + (reg:VNx16BI FFR_REGNUM))
> + (set (match_operand:VNx16BI_ONLY 0 "register_operand" "=Upa") ; rdffr
> + (reg:VNx16BI FFRT_REGNUM))
> + (set (reg:VNx16BI FFRT_REGNUM) ; update ffrt
> + (unspec:VNx16BI [(reg:VNx16BI FFRT_REGNUM)]
> UNSPEC_UPDATE_FFRT))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> +)
> +
> +(define_expand "read_hssr_state<mode>"
> + [(set (reg:VNx16BI FFRT_REGNUM) ; copy_ffr_to_ffrt
> + (reg:VNx16BI FFR_REGNUM))
> + (set (match_operand:PRED_HSD 0 "register_operand" "=Upa") ; rdffr
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0))
> + (set (reg:VNx16BI FFRT_REGNUM) ; update ffrt
> + (unspec:VNx16BI [(reg:VNx16BI FFRT_REGNUM)]
> UNSPEC_UPDATE_FFRT))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> +)
> +
> ;; Likewise with zero predication.
> -(define_insn "aarch64_rdffr_z"
> - [(set (match_operand:VNx16BI 0 "register_operand")
> - (and:VNx16BI
> +(define_insn "aarch64_rdffr_z<mode>"
> + [(set (match_operand:VNx16BI_ONLY 0 "register_operand")
> + (and:VNx16BI_ONLY
> (reg:VNx16BI FFRT_REGNUM)
> - (match_operand:VNx16BI 1 "register_operand")))]
> + (match_operand:VNx16BI_ONLY 1 "register_operand")))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> + [ &Upa , Upa ; yes ] rdffr\t%0.b, %1/z
> + [ ?Upa , 0Upa; yes ] ^
> + [ Upa , Upa ; no ] ^
> + }
> + [(set_attr "sve_type" "sve_ffr")]
> +)
> +(define_insn "aarch64_rdffr_z<mode>"
> + [(set (match_operand:PRED_HSD 0 "register_operand")
> + (and:PRED_HSD
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
> + (match_operand:PRED_HSD 1 "register_operand")))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> [ &Upa , Upa ; yes ] rdffr\t%0.b, %1/z
> @@ -1200,17 +1257,36 @@ (define_insn "aarch64_rdffr_z"
> )
>
> ;; Read the FFR to test for a fault, without using the predicate result.
> -(define_insn "*aarch64_rdffr_z_ptest"
> +(define_insn "*aarch64_rdffr_z_ptest<mode>"
> [(set (reg:CC_NZC CC_REGNUM)
> (unspec:CC_NZC
> [(match_operand:VNx16BI 1 "register_operand")
> (match_dup 1)
> (match_operand:SI 2 "aarch64_sve_ptrue_flag")
> - (and:VNx16BI
> + (and:VNx16BI_ONLY
> (reg:VNx16BI FFRT_REGNUM)
> (match_dup 1))]
> UNSPEC_PTEST))
> - (clobber (match_scratch:VNx16BI 0))]
> + (clobber (match_scratch:VNx16BI_ONLY 0))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> + [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
> + [ ?Upa , 0Upa; yes ] ^
> + [ Upa , Upa ; no ] ^
> + }
> + [(set_attr "sve_type" "sve_ffr")]
> +)
> +(define_insn "*aarch64_rdffr_z_ptest<mode>"
> + [(set (reg:CC_NZC CC_REGNUM)
> + (unspec:CC_NZC
> + [(match_operand:VNx16BI 1 "register_operand")
> + (match_operand:PRED_HSD 3)
> + (match_operand:SI 2 "aarch64_sve_ptrue_flag")
> + (and:PRED_HSD
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
> + (match_dup 3))]
> + UNSPEC_PTEST))
> + (clobber (match_scratch:PRED_HSD 0))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
> @@ -1221,15 +1297,32 @@ (define_insn "*aarch64_rdffr_z_ptest"
> )
>
> ;; Same for unpredicated RDFFR when tested with a known PTRUE.
> -(define_insn "*aarch64_rdffr_ptest"
> +(define_insn "*aarch64_rdffr_ptest<mode>"
> [(set (reg:CC_NZC CC_REGNUM)
> (unspec:CC_NZC
> [(match_operand:VNx16BI 1 "register_operand")
> - (match_dup 1)
> + (match_operand:VNx16BI_ONLY 2)
> (const_int SVE_KNOWN_PTRUE)
> (reg:VNx16BI FFRT_REGNUM)]
> UNSPEC_PTEST))
> - (clobber (match_scratch:VNx16BI 0))]
> + (clobber (match_scratch:VNx16BI_ONLY 0))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> + [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
> + [ ?Upa , 0Upa; yes ] ^
> + [ Upa , Upa ; no ] ^
> + }
> + [(set_attr "sve_type" "sve_ffr")]
> +)
> +(define_insn "*aarch64_rdffr_ptest<mode>"
> + [(set (reg:CC_NZC CC_REGNUM)
> + (unspec:CC_NZC
> + [(match_operand:VNx16BI 1 "register_operand")
> + (match_operand:PRED_HSD 2)
> + (const_int SVE_KNOWN_PTRUE)
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)]
> + UNSPEC_PTEST))
> + (clobber (match_scratch:PRED_HSD 0))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
> @@ -1240,18 +1333,18 @@ (define_insn "*aarch64_rdffr_ptest"
> )
>
> ;; Read the FFR with zero predication and test the result.
> -(define_insn "*aarch64_rdffr_z_cc"
> +(define_insn "*aarch64_rdffr_z_cc<mode>"
> [(set (reg:CC_NZC CC_REGNUM)
> (unspec:CC_NZC
> [(match_operand:VNx16BI 1 "register_operand")
> (match_dup 1)
> (match_operand:SI 2 "aarch64_sve_ptrue_flag")
> - (and:VNx16BI
> + (and:VNx16BI_ONLY
> (reg:VNx16BI FFRT_REGNUM)
> (match_dup 1))]
> UNSPEC_PTEST))
> - (set (match_operand:VNx16BI 0 "register_operand")
> - (and:VNx16BI
> + (set (match_operand:VNx16BI_ONLY 0 "register_operand")
> + (and:VNx16BI_ONLY
> (reg:VNx16BI FFRT_REGNUM)
> (match_dup 1)))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> @@ -1262,9 +1355,31 @@ (define_insn "*aarch64_rdffr_z_cc"
> }
> [(set_attr "sve_type" "sve_ffr")]
> )
> +(define_insn "*aarch64_rdffr_z_cc<mode>"
> + [(set (reg:CC_NZC CC_REGNUM)
> + (unspec:CC_NZC
> + [(match_operand:VNx16BI 1 "register_operand")
> + (match_operand:PRED_HSD 3)
> + (match_operand:SI 2 "aarch64_sve_ptrue_flag")
> + (and:PRED_HSD
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
> + (match_dup 3))]
> + UNSPEC_PTEST))
> + (set (match_operand:PRED_HSD 0 "register_operand")
> + (and:PRED_HSD
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)
> + (match_dup 3)))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> + [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
> + [ ?Upa , 0Upa; yes ] ^
> + [ Upa , Upa ; no ] ^
> + }
> + [(set_attr "sve_type" "sve_ffr")]
> +)
>
> ;; Same for unpredicated RDFFR when tested with a known PTRUE.
> -(define_insn "*aarch64_rdffr_cc"
> +(define_insn "*aarch64_rdffr_cc<mode>"
> [(set (reg:CC_NZC CC_REGNUM)
> (unspec:CC_NZC
> [(match_operand:VNx16BI 1 "register_operand")
> @@ -1272,7 +1387,7 @@ (define_insn "*aarch64_rdffr_cc"
> (const_int SVE_KNOWN_PTRUE)
> (reg:VNx16BI FFRT_REGNUM)]
> UNSPEC_PTEST))
> - (set (match_operand:VNx16BI 0 "register_operand")
> + (set (match_operand:VNx16BI_ONLY 0 "register_operand")
> (reg:VNx16BI FFRT_REGNUM))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> @@ -1282,6 +1397,24 @@ (define_insn "*aarch64_rdffr_cc"
> }
> [(set_attr "sve_type" "sve_ffr")]
> )
> +(define_insn "*aarch64_rdffr_cc<mode>"
> + [(set (reg:CC_NZC CC_REGNUM)
> + (unspec:CC_NZC
> + [(match_operand:VNx16BI 1 "register_operand")
> + (match_operand:PRED_HSD 2)
> + (const_int SVE_KNOWN_PTRUE)
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0)]
> + UNSPEC_PTEST))
> + (set (match_operand:PRED_HSD 0 "register_operand")
> + (subreg:PRED_HSD (reg:VNx16BI FFRT_REGNUM) 0))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {@ [ cons: =0, 1 ; attrs: pred_clobber ]
> + [ &Upa , Upa ; yes ] rdffrs\t%0.b, %1/z
> + [ ?Upa , 0Upa; yes ] ^
> + [ Upa , Upa ; no ] ^
> + }
> + [(set_attr "sve_type" "sve_ffr")]
> +)
>
> ;; [R3 in the block comment above about FFR handling]
> ;;
> @@ -1295,6 +1428,27 @@ (define_insn "aarch64_update_ffrt"
> [(set_attr "type" "no_insn")]
> )
>
> +/* Test if the last element of the predicate is not true. */
> +(define_expand "detect_hssr_fault<mode>"
> + [(match_operand:QI 0 "register_operand")
> + (match_operand:PRED_ALL 1 "register_operand")]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {
> + rtx ptrue = force_reg (VNx16BImode, aarch64_ptrue_all (<data_bytes>));
> + rtx cast_ptrue = gen_lowpart (<MODE>mode, ptrue);
> + rtx ptrue_flag = gen_int_mode (SVE_KNOWN_PTRUE, SImode);
> + emit_insn (gen_aarch64_ptestvnx16bi (ptrue, cast_ptrue, ptrue_flag,
> + operands[1]));
> +
> + rtx cc_reg = gen_rtx_REG (CC_NZCmode, CC_REGNUM);
> + rtx cmp = gen_rtx_fmt_ee (GEU, QImode, cc_reg, const0_rtx);
> +
> + emit_insn (gen_aarch64_cstoreqi (operands[0], cmp, cc_reg));
> +
> + DONE;
> + }
> +)
> +
> ;;
> ===================================================================
> ======
> ;; == Loads
> ;;
> ===================================================================
> ======
> @@ -1448,18 +1602,37 @@ (define_insn_and_rewrite
> "*aarch64_load_<ANY_EXTEND:optab>_mov<SVE_HSDI:mode><SV
> ;; -------------------------------------------------------------------------
>
> ;; Contiguous non-extending first-faulting or non-faulting loads.
> +;; Also supports sparse vectors via extending loads
> (define_insn "@aarch64_ld<fn>f1<mode>"
> - [(set (match_operand:SVE_FULL 0 "register_operand" "=w")
> - (unspec:SVE_FULL
> + [(set (match_operand:SVE_ALL 0 "register_operand" "=w")
> + (unspec:SVE_ALL
> [(match_operand:<VPRED> 2 "register_operand" "Upl")
> - (match_operand:SVE_FULL 1 "aarch64_sve_ld<fn>f1_operand"
> "Ut<fn>")
> + (match_operand:SVE_ALL 1 "aarch64_sve_ld<fn>f1_operand"
> "Ut<fn>")
> + (match_operand:SVE_ALL 3 "aarch64_maskload_else_operand")
> (reg:VNx16BI FFRT_REGNUM)]
> SVE_LDFF1_LDNF1))]
> "TARGET_SVE && TARGET_NON_STREAMING"
> - "ld<fn>f1<Vesize>\t%0.<Vetype>, %2/z, %1"
> + "ld<fn>f1<Vesize>\t%0.<Vctype>, %2/z, %1"
> [(set_attr "sve_type" "sve_load_1reg")]
> )
>
> +;; Expose {non,first}-faulting loads to the middle-end.
> +(define_expand "mask_ff_hssr_load<mode><vpred>"
> + [(set (match_operand:SVE_ALL 0 "register_operand")
> + (unspec:SVE_ALL
> + [(match_operand:<VPRED> 2 "register_operand")
> + (match_operand:SVE_ALL 1 "aarch64_sve_ldff1_operand")
> + (match_operand:SVE_ALL 3 "aarch64_maskload_else_operand")
> + (reg:VNx16BI FFRT_REGNUM)]
> + UNSPEC_LDFF1))]
> + "TARGET_SVE && TARGET_NON_STREAMING"
> + {
> + /* Emit the L2 insn; L1 follows from the template above.
> + See the earlier block comment on FFR handling. */
> + emit_insn (gen_aarch64_update_ffr_for_load ());
> + }
> +)
> +
> ;; -------------------------------------------------------------------------
> ;; ---- First-faulting extending contiguous loads
> ;; -------------------------------------------------------------------------
> diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
> index 0138c6f7ef0..5f5040cb0b9 100644
> --- a/gcc/internal-fn.cc
> +++ b/gcc/internal-fn.cc
> @@ -196,6 +196,9 @@ init_internal_fns ()
> #define crc_direct { 1, -1, true }
> #define reduc_sbool_direct { 0, 0, true }
> #define select_vl_direct { 2, 0, false }
> +#define read_hssr_state_direct { -1, -1, false }
> +#define set_hssr_state_direct { 0, 0, false }
> +#define detect_hssr_fault_direct { 0, 0, false }
>
> const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
> #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
> @@ -4315,6 +4318,21 @@ expand_reduc_sbool_optab_fn (internal_fn fn,
> gcall *stmt, direct_optab optab)
> #define expand_select_vl_optab_fn(FN, STMT, OPTAB) \
> expand_convert_optab_fn (FN, STMT, OPTAB, 3)
>
> +#define expand_read_hssr_state_optab_fn(FN, STMT, OPTAB) \
> + expand_direct_optab_fn (FN, STMT, OPTAB, 0)
> +
> +#define expand_detect_hssr_fault_optab_fn(FN, STMT, OPTAB) \
> + expand_direct_optab_fn (FN, STMT, OPTAB, 1)
> +
> +static void
> +expand_set_hssr_state_optab_fn (internal_fn fn, gcall *stmt,
> + direct_optab optab)
> +{
> + tree_pair types = direct_internal_fn_types (fn, stmt);
> + insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
> + expand_fn_using_insn (stmt, icode, 0, 1);
> +}
> +
> /* Expanders for optabs that can use expand_convert_optab_fn. */
>
> #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
> @@ -4432,6 +4450,9 @@ multi_vector_optab_supported_p (convert_optab
> optab, tree_pair types,
> #define direct_vec_extract_optab_supported_p convert_optab_supported_p
> #define direct_reduc_sbool_optab_supported_p direct_optab_supported_p
> #define direct_select_vl_optab_supported_p convert_optab_supported_p
> +#define direct_read_hssr_state_optab_supported_p
> direct_optab_supported_p
> +#define direct_set_hssr_state_optab_supported_p
> direct_optab_supported_p
> +#define direct_detect_hssr_fault_optab_supported_p
> direct_optab_supported_p
>
> /* Return the optab used by internal function FN. */
>
> @@ -5081,6 +5102,7 @@ internal_load_fn_p (internal_fn fn)
> case IFN_MASK_LEN_GATHER_LOAD:
> case IFN_LEN_LOAD:
> case IFN_MASK_LEN_LOAD:
> + case IFN_MASK_FF_HSSR_LOAD:
> return true;
>
> default:
> @@ -5253,6 +5275,7 @@ internal_fn_else_index (internal_fn fn)
> case IFN_MASK_LEN_LOAD:
> case IFN_MASK_LOAD_LANES:
> case IFN_MASK_LEN_LOAD_LANES:
> + case IFN_MASK_FF_HSSR_LOAD:
> return 3;
>
> case IFN_COND_FMA:
> @@ -5293,6 +5316,7 @@ internal_fn_mask_index (internal_fn fn)
> case IFN_MASK_LEN_STORE_LANES:
> case IFN_MASK_LEN_LOAD:
> case IFN_MASK_LEN_STORE:
> + case IFN_MASK_FF_HSSR_LOAD:
> return 2;
>
> case IFN_MASK_LEN_STRIDED_LOAD:
> @@ -5361,6 +5385,7 @@ internal_fn_alias_ptr_index (internal_fn fn)
> case IFN_SCATTER_STORE:
> case IFN_MASK_SCATTER_STORE:
> case IFN_MASK_LEN_SCATTER_STORE:
> + case IFN_MASK_FF_HSSR_LOAD:
> return 1;
>
> default:
> diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
> index 28ad5b7809d..68397c7bced 100644
> --- a/gcc/internal-fn.def
> +++ b/gcc/internal-fn.def
> @@ -651,6 +651,24 @@ DEF_INTERNAL_FN (BITINTTOFLOAT, ECF_PURE |
> ECF_LEAF, ". R . ")
> DEF_INTERNAL_OPTAB_FN (BIT_ANDN, ECF_CONST, andn, binary)
> DEF_INTERNAL_OPTAB_FN (BIT_IORN, ECF_CONST, iorn, binary)
>
> +/* Hardware Safe Speculative Reads internal functions. */
> +DEF_INTERNAL_OPTAB_FN (MASK_FF_HSSR_LOAD,
> + 0,
> + mask_ff_hssr_load,
> + mask_load)
> +DEF_INTERNAL_OPTAB_FN (SET_HSSR_STATE,
> + ECF_NOTHROW,
> + set_hssr_state,
> + set_hssr_state)
> +DEF_INTERNAL_OPTAB_FN (READ_HSSR_STATE,
> + ECF_NOTHROW,
> + read_hssr_state,
> + read_hssr_state)
> +DEF_INTERNAL_OPTAB_FN (DETECT_HSSR_FAULT,
> + ECF_CONST,
> + detect_hssr_fault,
> + detect_hssr_fault)
> +
> #undef DEF_INTERNAL_WIDENING_OPTAB_FN
> #undef DEF_INTERNAL_SIGNED_COND_FN
> #undef DEF_INTERNAL_COND_FN
> diff --git a/gcc/optabs.def b/gcc/optabs.def
> index da58147a513..9b5ca3434be 100644
> --- a/gcc/optabs.def
> +++ b/gcc/optabs.def
> @@ -118,6 +118,9 @@ OPTAB_CD (usdot_prod_optab,
> "usdot_prod$I$a$b")
> OPTAB_CD (while_ult_optab, "while_ult$a$b")
> OPTAB_CD (select_vl_optab, "select_vl$a$b")
>
> +/* Hardware Safe Speculative Reads optabs. */
> +OPTAB_CD (mask_ff_hssr_load_optab, "mask_ff_hssr_load$a$b")
> +
> OPTAB_NL(add_optab, "add$P$a3", PLUS, "add", '3',
> gen_int_fp_fixed_libfunc)
> OPTAB_NX(add_optab, "add$F$a3")
> OPTAB_NX(add_optab, "add$Q$a3")
> @@ -238,6 +241,11 @@ OPTAB_D (push_optab, "push$a1")
> OPTAB_D (reload_in_optab, "reload_in$a")
> OPTAB_D (reload_out_optab, "reload_out$a")
>
> +/* Hardware Safe Speculative Reads optabs. */
> +OPTAB_D (read_hssr_state_optab, "read_hssr_state$a")
> +OPTAB_D (set_hssr_state_optab, "set_hssr_state$a")
> +OPTAB_D (detect_hssr_fault_optab, "detect_hssr_fault$a")
> +
> OPTAB_DC(cbranch_optab, "cbranch$a4", COMPARE)
> OPTAB_D (tbranch_eq_optab, "tbranch_eq$a3")
> OPTAB_D (tbranch_ne_optab, "tbranch_ne$a3")
> diff --git a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
> b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
> index a32b636b278..faf433dfb61 100644
> --- a/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
> +++ b/gcc/testsuite/gcc.target/aarch64/sve/pfalse-load.c
> @@ -24,9 +24,8 @@
> T (F##_u32, uint32_t) \
> T (F##_u64, uint64_t) \
>
> -ALL_DATA (ldff1)
> -ALL_DATA (ldnf1)
> ALL_DATA (ldnt1)
> +ALL_DATA (ldnf1)
>
> -/* { dg-final { scan-assembler-times {\t.cfi_startproc\n\tmovi?\t[vdz]([0-
> 9]+)\.?(?:[0-9]*[bhsd])?, #?0\n\tret\n} 36 } } */
> -/* { dg-final { scan-assembler-times {\t.cfi_startproc\n} 36 } } */
> +/* { dg-final { scan-assembler-times {\t.cfi_startproc\n\tmovi?\t[vdz]([0-
> 9]+)\.?(?:[0-9]*[bhsd])?, #?0\n\tret\n} 24 } } */
> +/* { dg-final { scan-assembler-times {\t.cfi_startproc\n} 24 } } */
> diff --git a/gcc/tree-data-ref.cc b/gcc/tree-data-ref.cc
> index 8e9d7871348..e4f04a44e0c 100644
> --- a/gcc/tree-data-ref.cc
> +++ b/gcc/tree-data-ref.cc
> @@ -5893,6 +5893,7 @@ get_references_in_stmt (gimple *stmt,
> vec<data_ref_loc, va_heap> *references)
> break;
> }
> case IFN_MASK_LOAD:
> + case IFN_MASK_FF_HSSR_LOAD:
> case IFN_MASK_STORE:
> break;
> case IFN_MASK_CALL:
> @@ -5950,6 +5951,7 @@ get_references_in_stmt (gimple *stmt,
> vec<data_ref_loc, va_heap> *references)
> switch (gimple_call_internal_fn (stmt))
> {
> case IFN_MASK_LOAD:
> + case IFN_MASK_FF_HSSR_LOAD:
> if (gimple_call_lhs (stmt) == NULL_TREE)
> break;
> ref.is_read = true;
> diff --git a/gcc/tree-ssa-alias.cc b/gcc/tree-ssa-alias.cc
> index 1d10f6dbd97..be47194ef3b 100644
> --- a/gcc/tree-ssa-alias.cc
> +++ b/gcc/tree-ssa-alias.cc
> @@ -2888,6 +2888,7 @@ ref_maybe_used_by_call_p_1 (gcall *call, ao_ref
> *ref, bool tbaa_p)
> case IFN_MASK_LEN_STORE_LANES:
> goto process_args;
> case IFN_MASK_LOAD:
> + case IFN_MASK_FF_HSSR_LOAD:
> case IFN_LEN_LOAD:
> case IFN_MASK_LEN_LOAD:
> case IFN_MASK_LOAD_LANES:
> diff --git a/gcc/tree-ssa-loop-ivopts.cc b/gcc/tree-ssa-loop-ivopts.cc
> index 3c0ce1794c2..881055212d7 100644
> --- a/gcc/tree-ssa-loop-ivopts.cc
> +++ b/gcc/tree-ssa-loop-ivopts.cc
> @@ -2370,6 +2370,7 @@ get_mem_type_for_internal_fn (gcall *call, tree
> *op_p)
> switch (gimple_call_internal_fn (call))
> {
> case IFN_MASK_LOAD:
> + case IFN_MASK_FF_HSSR_LOAD:
> case IFN_MASK_LOAD_LANES:
> case IFN_MASK_LEN_LOAD_LANES:
> case IFN_LEN_LOAD:
> @@ -7567,6 +7568,7 @@ get_alias_ptr_type_for_ptr_address (iv_use *use)
> switch (gimple_call_internal_fn (call))
> {
> case IFN_MASK_LOAD:
> + case IFN_MASK_FF_HSSR_LOAD:
> case IFN_MASK_STORE:
> case IFN_MASK_LOAD_LANES:
> case IFN_MASK_STORE_LANES:
> diff --git a/gcc/tree-vect-slp.cc b/gcc/tree-vect-slp.cc
> index b4eb9c204cb..3d767a5822f 100644
> --- a/gcc/tree-vect-slp.cc
> +++ b/gcc/tree-vect-slp.cc
> @@ -581,6 +581,9 @@ vect_get_operand_map (const gimple *stmt, bool
> gather_scatter_p,
> case IFN_MASK_LOAD:
> return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
>
> + case IFN_MASK_FF_HSSR_LOAD:
> + return arg2_arg3_map;
> +
> case IFN_GATHER_LOAD:
> return arg2_map;
>
> --
> 2.43.0
