On 23/06/2026 12:18, Richard Sandiford wrote:
Alfie Richards <[email protected]> writes:
Hi All,
This is my first attempt at an implementation of FFR.
I think this needs some background and explanation
(probably should be a comment at some point)
== AArch64 first faulting read ==
AArch64 first faulting reads are safe to execute speculatively because the
hardware is able to return less than a full vector read.
The loads (both "first faulting" (FF) and "non-faulting" (NF)) set a predicate
register (first fault register) with a mask of loaded elements.
The FF variant will fault only if the first active element to be loaded
causes a fault. This guarantees forwards progress.
The NF variant will never cause a fault, at the expense of possibly
loading no elements.
Architecturally, these loads are guaranteed to never cause a fault unless
it is a FF load and there is a fault for the first element.
However, the hardware can also do partial loads whenever it wants.
For instance this could happen at page boundaries or cache faults.
This means an FFR vectorized loop needs to be able to recover from partial
loads back to vectoried code.
(this seems to not be the case for riscv's equivalent feature).
Also of note, the first fault register starts as all 1's, and each subsequent
load updates it if they fail to load any elements by diabling the bits of
elements not loaded and all bits afterwards.
For instance, it may start as
[1,1,1,1,1,1,1,1]
then a load is executed and could change that to
[1,1,1,1,1,1,0,0]
But then the next load, even if it loads all elements, will not re-enable
the last two bits.
Additionally, you never get a state like
[1,1,1,1,1,0,0,1]
where there are inactive elements followed by an active element.
== Vectorized code structure ==
The use case currently is for loops with mutually misaligned pointers, both
of which require safe speculative reads, where we cannot peel for alignment, or
would not want to incur the code size cost of versioning to check if we can
peel.
For aarch64 this is only for early break as there aren't any other situations
where we need safe speculative reads.
Then for a loop such as
int
foo_no_vect (uint16_t *const restrict src1,
uint16_t *const restrict src2,
unsigned int n)
{
for (int i = 0; i < n; i++)
{
uint16_t v1 = src1[i];
uint16_t v2 = src2[i];
if (v1 + v2 == 0)
return 1;
}
return 0;
}
We generate the following:
GIMPLE (after dce):
_78 = max_mask_75 & _77;
ffr_preservation_82 = .READ_FAULT_STATE ();
.SET_FAULT_STATE ({ -1, ... });
<bb 3> [local count: 1014686025]:
# vectp_src1.7_45 = PHI <vectp_src1.7_46(7), vectp_src1.8_41(13)>
# vectp_src2.11_60 = PHI <vectp_src2.11_61(7), vectp_src2.12_56(13)>
# ivtmp_72 = PHI <ivtmp_73(7), 0(13)>
# loop_mask_48 = PHI <next_mask_ffr_81(7), _78(13)>
vect_v1_14.9_49 = .MASK_FIRSTFAULT_LOAD (vectp_src1.7_45, 64B, loop_mask_48,
{ 0, ... });
vect__5.10_55 = (vector([4,4]) int) vect_v1_14.9_49;
vect_v2_16.13_63 = .MASK_FIRSTFAULT_LOAD (vectp_src2.11_60, 64B,
loop_mask_48, { 0, ... });
vect__6.14_65 = (vector([4,4]) int) vect_v2_16.13_63;
ffr_mask_83 = .READ_FAULT_STATE ();
if (ffr_mask_83 == { -1, ... })
goto <bb 14>; [99.95%]
else
goto <bb 15>; [0.05%]
<bb 15> [local count: 10146860]:
.SET_FAULT_STATE ({ -1, ... });
_84 = ~ffr_mask_83;
ffr_loop_mask_85 = loop_mask_48 & ffr_mask_83;
<bb 14> [local count: 1014686025]:
# ffr_loop_mask_68 = PHI <loop_mask_48(3), ffr_loop_mask_85(15)>
# ffr_num_iters_36 = PHI <POLY_INT_CST [4, 4](3), 0(15)>
# next_mask_ffr_80 = PHI <{ -1, ... }(3), _84(15)>
vect__7.15_66 = vect__5.10_55 + vect__6.14_65;
mask_patt_28.16_67 = vect__7.15_66 == { 0, ... };
vec_mask_and_69 = mask_patt_28.16_67 & ffr_loop_mask_68;
if (vec_mask_and_69 != { 0, ... })
goto <bb 9>; [5.50%]
else
goto <bb 4>; [94.50%]
<bb 9> [local count: 55807731]:
.SET_FAULT_STATE (ffr_preservation_82);
goto <bb 5>; [100.00%]
<bb 4> [local count: 958878295]:
_47 = ffr_num_iters_36 * 2;
vectp_src1.7_46 = vectp_src1.7_45 + _47;
vectp_src2.11_61 = vectp_src2.11_60 + _47;
ivtmp_73 = ivtmp_72 + ffr_num_iters_36;
next_mask_79 = .WHILE_ULT (ivtmp_73, _74, { 0, ... });
next_mask_ffr_81 = next_mask_79 & next_mask_ffr_80;
if (next_mask_ffr_81 != { 0, ... })
goto <bb 7>; [94.50%]
else
goto <bb 12>; [5.50%]
<bb 12> [local count: 52738306]:
.SET_FAULT_STATE (ffr_preservation_82);
goto <bb 5>; [100.00%]
<bb 7> [local count: 906139989]:
goto <bb 3>; [100.00%]
<bb 5> [local count: 114863531]:
# _10 = PHI <1(9), 0(12), 0(2)>
return _10;
Or final assembly:
.L5:
add x4, x4, x3
whilelo p7.s, x4, x2
add x0, x0, x3, lsl 1
and p7.b, p7/z, p14.b, p14.b
add x1, x1, x3, lsl 1
ptest p15, p7.b
b.none .L7
.L6:
ldff1h z31.s, p7/z, [x0]
ldff1h z30.s, p7/z, [x1]
cntw x3
rdffr p14.b
nots p13.b, p15/z, p14.b
b.any .L11
.L4:
add z31.s, z31.s, z30.s
cmpeq p7.s, p7/z, z31.s, #0
b.none .L5
mov w0, 1
ret
== Notes ==
- When there is a "partial read", instead of treating that as a partial
iteration
and advancing by the number of loaded elements, we intead advance by 0
iterations ard repeat the same iteration with the previously processed
elements masked out. This preserves alignment with the starting position
and avoids having to do anything awkward such as possibly rotating
invariant vectors.
- This prioritises the "good" case, by trying to keep
the "full read" path as tight as possible, and adding
a fixup branch to handle the case where there is a partial read.
Hmm, interesting. I hadn't seen that approach being proposed before.
I think the "classical" idea (is SVE old enough for things to be
classical?) was that we would have two copies of the non-speculative
part of the loop body: a full vector version for when no fault is
detected and an FFR-predicated version for when FFR has zero bits.
The FFR-predicated version would advance by less than a full vector,
which like you say would affect induction vectors. But as Robin says,
my understanding is that the .LEN code already supports that.
(We would still be advancing by a whole number of scalar iterations,
so invariant vectors shouldn't need to change.)
Hi Richard :)
Firstly, in hindsight the example code generation I provided is missing
sections and is overall unhelpful. I have attached two different
examples instead which hopefully will be more helpful.
Example 1 is a fairly simple loop. Example 2 shows an example with
multiple FFR reagions.
In regards to the points you raised, I think there are actually two
orthogonal discussions here.
The first (question A) is versioning the whole body of the loop (as you
call the "classical idea") vs a "fixup block" like I suggest. The fixup
block keeps one loop body, but with the addition of the branch to the
fixup block to calculate anything required to deal with a partial load
(such as AND of ffr and the loop mask).
In testing I actually found the overhead to be pretty similar for having
a fixup block verses duplicating the entire body of the loop. Then as
you say, using the fixup block allows multiple FFR regions (which this
patch series implements, see ex 2) and it requires less code overall, so
I strongly prefer this.
The second issue (question B) is what to do when you have a partial
load, either repeating the vectorised iteration with processed elements
masked out, or to advance IV's by the cntp (ffr) (number of successfully
loaded iterations.
I initially chose option 1 as I believed the invariants would be an
issue, but as you and Robin say, I am just mistaken in that. In this
case I think we should clearly go for option 2 and I can do that in the
next version of this.
I can reasonably easily change this and still use the fixup block (this
is my preferred option). This will also work better for other targets
who don't have a mask.
The supposed advantages of that were (IIRC):
(1) The full vector version would be the same as for a non-FFR loop.
That is, the overhead for the "good" case would be even lower
than above.
I think this is in reference to question A. As stated above, I measured
some example code gen and didn't see much difference between the two
(fixup vs 2 loop bodies).
From my examples you can hopefully see how we don't have a dependency
on the FFR value other than in the fixup block (other than for the nots
which I haven't yet fixed).
(2) It would reduce the number of vector iterations.
I think this is in regard to question B, and I agree, I think this is
preferable now I understand invariants aren't an issue.
(3) In practice, a load would only be suppressed starting at index X if
X corresponds to a natural boundary (at least a cache boundary).
Restarting the loop at that point might make future iterations
"more aligned", which might be faster in its own right, but might
also reduce the risk of future false suppressed faults.
Like you say below, it would be better to avoid first-faulting loads
if all pointers are well-aligned. If we do that, (3) would only come
into play for mutually misaligned pointers (or for -O2). But even for
mutually misaligned pointers, it could be better (or no worse) to be
aligned to one pointer than be misaligned to all pointers.
Agreed. I think preserving alignment is the wrong decision and we should
instead advance by cntp (ffr).
The advantage of your approach is that it would scale well for multiple
groups of FFR loads in the same loop. We don't support that yet AIUI,
but we could in future.
This patch series does support multiple FFR regions (my nomeclature).
Example 2 should show an example.
However, if I'm reading the above assembly correctly, it looks like:
next_mask_ffr_81 = next_mask_79 & next_mask_ffr_80;
ends up using the RDFFR result even for the "good" path. Is:
# next_mask_ffr_80 = PHI <{ -1, ... }(3), _84(15)>
getting optimised to _84 based on:
if (ffr_mask_83 == { -1, ... })
? That would be a legitimate optimisation as far as gimple is concerned,
but my understanding is that it would be harmful for performance.
I think this is actually my mistake and the example I gave is missing
bits and misleading. If you see example 1 and 2 you can hopefully see in
the "good" full load case, we do not depend on the value from FFR in the
loop body.
- We preserve the state of the FFR register over the vectoried loop.
This is to prevent code written with intrinsics breaking by clobbering
their value in the FFR register. However, as the FFR is not preserved by
the AAPCS, this is nearly always optimized out.
Do we also manage to optimise it away if the vectorised loop occurs between
two independent FFR-using intrinsic loops? If so, then yeah, I agree that
this is probably the right approach.
I just checked and they do indeed get removed correctly :)
- One downside of this approach is I dont think it will translate cleanly for
len based loop vectorization (riscv). However, as I understand it, the
riscv equivalent feature never does partial loads unless there is a genuine
fault, so we will not need to worry about recovering back to
vectorized code after a partial read (as it will either take the early break
or the fault). So no fixup should be needed and can use a subset of this.
== Remaining work to do ==
- Versioning
As FFR introduces overhead, it will always be slower than a mutually
aligned loop. So my ideal code generation for a vector with two pointers
requiring safe speculative reads is:
At O2, where we do not want to incur the code size cost of versioning,
instead just use FFR to vectorize the loop (if profitable).
At O3, version to create a mutually aligned non-FFR case and a (current code
gen)
if the pointers are mutually misaligned.
This "FFR versioning" is not yet implemented.
- Costing
I haven't done anything to cost FFR yet, we will want accurate costing of FFR
vs scalar and vs non-FFR vectorized to make the decisions on versioning and
what to use.
- Optimization of the generated code
The generated code has room for improvement, primarily using rdffrs would
be a performance gain, and reducing some of the moves within the hot
section of the loop.
Yeah. As mentioned privately, I think one option for the RDFFRS thing
would be to add an internal fn such as .FAULT_DETECTED and have ISEL
convert:
if (ffr_mask_83 == { -1, ... })
to:
temp = .FAULT_DETECTED (ffr_mask_83);
if (temp != 0)
when the optab is implemented. .FAULT_DETECTED could then take
advantage of the lack of 0->1 transitions.
Yes agreed, I plan to do this as you kindly suggested, but I haven't
gotten around to it yet as I wanted to get this out for feedback. Sorry!
Thanks,
Richard
Thanks for the feedback Richard
KR,
Alfie
;; Function foo (foo, funcdef_no=13, decl_uid=16260, cgraph_uid=14,
symbol_order=13)
Symbols to be put in SSA form
{ D.16272 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 10
Number of blocks to update: 9 ( 90%)
Symbols to be put in SSA form
{ D.16272 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 10
Number of blocks to update: 9 ( 90%)
Symbols to be put in SSA form
{ D.16272 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 12
Number of blocks to update: 11 ( 92%)
SSA replacement table
N_i -> { O_1 ... O_j } means that N_i replaces O_1, ..., O_j
.MEM_70 -> { .MEM_64 }
.MEM_71 -> { .MEM_64 }
Incremental SSA update started at block: 3
Number of blocks in CFG: 12
Number of blocks to update: 7 ( 58%)
Merging blocks 2 and 9
int foo (int * restrict a, int * restrict b)
{
vector([4,4]) <signed-boolean:4> mask_patt_13.11;
vector([4,4]) int vect__6.10;
int * vectp_b.9;
vector([4,4]) int * vectp_b.8;
vector([4,4]) int vect__4.7;
int * vectp_a.6;
vector([4,4]) int * vectp_a.5;
int * vectp_b.4;
int i;
long unsigned int _1;
long unsigned int _2;
int * _3;
int _4;
int * _5;
int _6;
int _8;
unsigned int ivtmp_15;
unsigned int ivtmp_16;
unsigned long ffr_num_iters_19;
unsigned long _20;
unsigned long _21;
unsigned long _22;
unsigned int _23;
sizetype _25;
sizetype _26;
sizetype _27;
sizetype _30;
vector([4,4]) <signed-boolean:4> loop_mask_31;
vector([4,4]) <signed-boolean:4> ffr_loop_mask_46;
vector([4,4]) <signed-boolean:4> vec_mask_and_47;
unsigned long ivtmp_49;
unsigned long ivtmp_50;
unsigned long _51;
vector([4,4]) <signed-boolean:4> max_mask_52;
vector([4,4]) <signed-boolean:4> _53;
vector([4,4]) <signed-boolean:4> _54;
vector([4,4]) <signed-boolean:4> _55;
vector([4,4]) <signed-boolean:4> next_mask_56;
vector([4,4]) <signed-boolean:4> next_mask_ffr_57;
vector([4,4]) <signed-boolean:4> next_mask_ffr_58;
vector([4,4]) <signed-boolean:4> ffr_preservation_59;
vector([4,4]) <signed-boolean:4> ffr_mask_60;
vector([4,4]) <signed-boolean:4> _61;
vector([4,4]) <signed-boolean:4> ffr_loop_mask_62;
<bb 2> [local count: 69202658]:
_20 = (unsigned long) b_11(D);
_21 = _20 & POLY_INT_CST [15, 16];
_22 = _21 >> 2;
_23 = (unsigned int) _22;
_25 = (sizetype) _23;
_26 = _25 * 4;
_27 = -_26;
vectp_a.6_24 = a_9(D) + _27;
vectp_b.9_36 = b_11(D) + _27;
_51 = _25 + 10000;
max_mask_52 = .WHILE_ULT (0, _51, { 0, ... });
_53 = .WHILE_ULT (0, _25, { 0, ... });
_54 = ~_53;
_55 = max_mask_52 & _54;
ffr_preservation_59 = .READ_FAULT_STATE ();
.SET_FAULT_STATE ({ -1, ... });
<bb 3> [local count: 1063004408]:
# i_14 = PHI <i_12(6), 0(2)>
# ivtmp_16 = PHI <ivtmp_15(6), 10000(2)>
# vectp_a.5_28 = PHI <vectp_a.5_29(6), vectp_a.6_24(2)>
# vectp_b.8_40 = PHI <vectp_b.8_41(6), vectp_b.9_36(2)>
# ivtmp_49 = PHI <ivtmp_50(6), 0(2)>
# loop_mask_31 = PHI <next_mask_ffr_58(6), _55(2)>
_1 = (long unsigned int) i_14;
_2 = _1 * 4;
_3 = a_9(D) + _2;
vect__4.7_32 = .MASK_FIRSTFAULT_LOAD (vectp_a.5_28, 128B, loop_mask_31, { 0,
... });
_4 = *_3;
_5 = b_11(D) + _2;
vect__6.10_43 = .MASK_FIRSTFAULT_LOAD (vectp_b.8_40, 128B, loop_mask_31, { 0,
... });
_6 = *_5;
ffr_mask_60 = .READ_FAULT_STATE ();
if (ffr_mask_60 == { -1, ... })
goto <bb 10>; [99.95%]
else
goto <bb 11>; [0.05%]
<bb 11> [local count: 10630044]:
.SET_FAULT_STATE ({ -1, ... });
_61 = ~ffr_mask_60;
ffr_loop_mask_62 = loop_mask_31 & ffr_mask_60;
<bb 10> [local count: 1063004408]:
# ffr_loop_mask_46 = PHI <loop_mask_31(3), ffr_loop_mask_62(11)>
# ffr_num_iters_19 = PHI <POLY_INT_CST [4, 4](3), 0(11)>
# next_mask_ffr_57 = PHI <{ -1, ... }(3), _61(11)>
mask_patt_13.11_45 = vect__4.7_32 == vect__6.10_43;
vec_mask_and_47 = mask_patt_13.11_45 & ffr_loop_mask_46;
if (vec_mask_and_47 != { 0, ... })
goto <bb 7>; [5.50%]
else
goto <bb 4>; [94.50%]
<bb 4> [local count: 1004539166]:
i_12 = i_14 + 1;
ivtmp_15 = ivtmp_16 - 1;
_30 = ffr_num_iters_19 * 4;
vectp_a.5_29 = vectp_a.5_28 + _30;
vectp_b.8_41 = vectp_b.8_40 + _30;
ivtmp_50 = ivtmp_49 + ffr_num_iters_19;
next_mask_56 = .WHILE_ULT (ivtmp_50, _51, { 0, ... });
next_mask_ffr_58 = next_mask_56 & next_mask_ffr_57;
if (next_mask_ffr_58 != { 0, ... })
goto <bb 6>; [98.93%]
else
goto <bb 8>; [1.07%]
<bb 8> [local count: 10737420]:
.SET_FAULT_STATE (ffr_preservation_59);
goto <bb 5>; [100.00%]
<bb 6> [local count: 993801746]:
goto <bb 3>; [100.00%]
<bb 7> [local count: 58465242]:
.SET_FAULT_STATE (ffr_preservation_59);
<bb 5> [local count: 69202658]:
# _8 = PHI <1(7), 0(8)>
return _8;
}
.arch armv8.2-a+crc+sve
.file "example1.c"
.text
.align 2
.p2align 5,,15
.global foo
.type foo, %function
foo:
.LFB13:
.cfi_startproc
cntb x4
sub x4, x4, #1
setffr
and x4, x1, x4
mov w3, 0
ptrue p15.s, all
and x2, x4, 17179869180
ubfx x4, x4, 2, 32
sub x0, x0, x2
whilelo p14.s, xzr, x4
sub x1, x1, x2
mov w2, 10000
add x4, x4, x2
whilelo p7.s, xzr, x4
not p7.b, p7/z, p14.b
b .L4
.p2align 2,,3
.L11:
add x3, x3, x2
whilelo p7.s, x3, x4
add x0, x0, x2, lsl 2
and p7.b, p7/z, p14.b, p14.b
add x1, x1, x2, lsl 2
ptest p15, p7.b
b.none .L9
.L4:
ldff1w z30.s, p7/z, [x0]
ldff1w z31.s, p7/z, [x1]
cntw x2
rdffr p14.b
nots p13.b, p15/z, p14.b
b.any .L10
.L2:
cmpeq p7.s, p7/z, z30.s, z31.s
b.none .L11
mov w0, 1
ret
.p2align 2,,3
.L9:
mov w0, 0
ret
.L10:
and p7.b, p7/z, p14.b, p14.b
mov w2, 0
setffr
mov p14.b, p13.b
b .L2
.cfi_endproc
.LFE13:
.size foo, .-foo
.ident "GCC: (GNU) 17.0.0 20260622 (experimental)"
.section .note.GNU-stack,"",@progbits
#include <math.h>
#include <stdint.h>
#include <stdlib.h>
#include <arm_sve.h>
int foo (int *restrict a, int *restrict b) {
for (int i = 0; i < 10000; i++)
{
if (a[i] == b[i]) {
return 1;
}
}
return 0;
}
;; Function bar (bar, funcdef_no=13, decl_uid=16269, cgraph_uid=14,
symbol_order=13)
Removing basic block 10
Symbols to be put in SSA form
{ D.16283 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 20
Number of blocks to update: 19 ( 95%)
Symbols to be put in SSA form
{ D.16283 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 20
Number of blocks to update: 19 ( 95%)
Symbols to be put in SSA form
{ D.16283 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 20
Number of blocks to update: 19 ( 95%)
Symbols to be put in SSA form
{ D.16283 }
Incremental SSA update started at block: 0
Number of blocks in CFG: 27
Number of blocks to update: 26 ( 96%)
SSA replacement table
N_i -> { O_1 ... O_j } means that N_i replaces O_1, ..., O_j
i_182 -> { i_30 }
i_183 -> { i_34 }
i_184 -> { i_34 }
_185 -> { _79 }
_186 -> { _79 }
.MEM_187 -> { .MEM_132 }
.MEM_188 -> { .MEM_173 }
.MEM_189 -> { .MEM_174 }
_190 -> { _79 }
Incremental SSA update started at block: 3
Number of blocks in CFG: 27
Number of blocks to update: 12 ( 44%)
Removing basic block 9
Removing basic block 16
Merging blocks 20 and 11
void bar (uint16_t * const restrict dst1, uint16_t * const restrict dst2,
uint16_t * const restrict src1, uint16_t * const restrict src1b, uint16_t *
const restrict src2, uint16_t * const restrict src3, svbool_t * p, svbool_t *
p2, unsigned int n)
{
vector([8,8]) <signed-boolean:2> mask_patt_15.29;
vector([8,8]) short unsigned int vect_v1b_19.28;
vector([8,8]) short unsigned int vect__4.27;
uint16_t * vectp_src1b.26;
vector([8,8]) short unsigned int * vectp_src1b.25;
vector([8,8]) <signed-boolean:2> mask_patt_39.24;
uint16_t * vectp_dst2.23;
vector([8,8]) short unsigned int * vectp_dst2.22;
vector([8,8]) short unsigned int vect__11.21;
vector([8,8]) short unsigned int vect_v3_27.20;
uint16_t * vectp_src3.19;
vector([8,8]) short unsigned int * vectp_src3.18;
uint16_t * vectp_dst1.17;
vector([8,8]) short unsigned int * vectp_dst1.16;
vector([8,8]) short unsigned int vect__8.15;
vector([8,8]) short unsigned int vect_v2_23.14;
uint16_t * vectp_src2.13;
vector([8,8]) short unsigned int * vectp_src2.12;
vector([8,8]) short unsigned int vect_v1_21.11;
uint16_t * vectp_src1.10;
vector([8,8]) short unsigned int * vectp_src1.9;
uint16_t * vectp_src2.8;
int tmp.7;
int tmp.6;
unsigned int niters.5;
uint16_t v3;
uint16_t v2;
uint16_t v1;
uint16_t v1b;
int i;
long unsigned int _1;
long unsigned int _2;
uint16_t * _3;
short unsigned int _4;
uint16_t * _5;
uint16_t * _6;
uint16_t * _7;
short unsigned int _8;
uint16_t * _9;
uint16_t * _10;
short unsigned int _11;
unsigned int i.0_12;
unsigned long ffr_num_iters_13;
long unsigned int _43;
long unsigned int _44;
uint16_t * _45;
short unsigned int _46;
uint16_t * _48;
uint16_t * _50;
uint16_t * _52;
short unsigned int _53;
unsigned int i.0_56;
uint16_t * _57;
uint16_t * _59;
short unsigned int _60;
unsigned int _68;
unsigned int _71;
unsigned long _72;
unsigned long _75;
unsigned long _76;
unsigned long _77;
unsigned int _78;
unsigned long _79;
unsigned long _80;
unsigned long _81;
unsigned long _82;
signed long _83;
signed long _84;
unsigned long _85;
sizetype _88;
sizetype _89;
sizetype _92;
vector([8,8]) <signed-boolean:2> loop_mask_93;
vector([8,8]) <signed-boolean:2> ffr_loop_mask_103;
vector([8,8]) <signed-boolean:2> ffr_loop_mask_114;
vector([8,8]) <signed-boolean:2> vec_mask_and_134;
vector([8,8]) short unsigned int _146;
vector([8,8]) <signed-boolean:2> _147;
vector([8,8]) short unsigned int _148;
vector([8,8]) <signed-boolean:2> vec_mask_and_151;
unsigned long _153;
unsigned long ivtmp_154;
unsigned long ivtmp_155;
unsigned long _156;
vector([8,8]) <signed-boolean:2> max_mask_157;
vector([8,8]) <signed-boolean:2> _158;
vector([8,8]) <signed-boolean:2> _159;
vector([8,8]) <signed-boolean:2> _160;
vector([8,8]) <signed-boolean:2> next_mask_161;
vector([8,8]) <signed-boolean:2> next_mask_ffr_162;
vector([8,8]) <signed-boolean:2> next_mask_ffr_163;
vector([8,8]) <signed-boolean:2> ffr_preservation_164;
vector([8,8]) <signed-boolean:2> ffr_mask_165;
vector([8,8]) <signed-boolean:2> _166;
vector([8,8]) <signed-boolean:2> ffr_loop_mask_167;
vector([8,8]) <signed-boolean:2> ffr_mask_168;
vector([8,8]) <signed-boolean:2> _169;
vector([8,8]) <signed-boolean:2> ffr_loop_mask_170;
unsigned long ffr_num_iters_171;
vector([8,8]) <signed-boolean:2> ffr_next_mask_172;
unsigned long _185;
unsigned long _186;
unsigned long _190;
<bb 2> [local count: 113634474]:
if (n_17(D) != 0)
goto <bb 7>; [96.34%]
else
goto <bb 6>; [3.66%]
<bb 7> [local count: 109475452]:
_68 = n_17(D) + 4294967295;
if (_68 < POLY_INT_CST [8, 8])
goto <bb 18>; [10.00%]
else
goto <bb 19>; [90.00%]
<bb 19> [local count: 98527907]:
_75 = (unsigned long) src2_22(D);
_76 = _75 & POLY_INT_CST [15, 16];
_77 = _76 >> 1;
_78 = (unsigned int) _77;
_81 = (unsigned long) _78;
_82 = -_81;
_88 = _81 * 2;
_89 = -_88;
vectp_src1.10_86 = src1_20(D) + _89;
vectp_src2.13_96 = src2_22(D) + _89;
vectp_dst1.17_107 = dst1_24(D) + _89;
vectp_src3.19_116 = src3_26(D) + _89;
vectp_dst2.23_125 = dst2_28(D) + _89;
vectp_src1b.26_135 = src1b_18(D) + _89;
_153 = (unsigned long) n_17(D);
_156 = _81 + _153;
max_mask_157 = .WHILE_ULT (0, _156, { 0, ... });
_158 = .WHILE_ULT (0, _81, { 0, ... });
_159 = ~_158;
_160 = max_mask_157 & _159;
ffr_preservation_164 = .READ_FAULT_STATE ();
.SET_FAULT_STATE ({ -1, ... });
<bb 3> [local count: 930998582]:
# i_34 = PHI <i_30(8), 0(19)>
# _79 = PHI <_80(8), _82(19)>
# vectp_src1.9_90 = PHI <vectp_src1.9_91(8), vectp_src1.10_86(19)>
# vectp_src2.12_100 = PHI <vectp_src2.12_101(8), vectp_src2.13_96(19)>
# vectp_dst1.16_111 = PHI <vectp_dst1.16_112(8), vectp_dst1.17_107(19)>
# vectp_src3.18_120 = PHI <vectp_src3.18_121(8), vectp_src3.19_116(19)>
# vectp_dst2.22_129 = PHI <vectp_dst2.22_130(8), vectp_dst2.23_125(19)>
# vectp_src1b.25_139 = PHI <vectp_src1b.25_140(8), vectp_src1b.26_135(19)>
# ivtmp_154 = PHI <ivtmp_155(8), 0(19)>
# loop_mask_93 = PHI <next_mask_ffr_163(8), _160(19)>
_1 = (long unsigned int) i_34;
_2 = _1 * 2;
_3 = src1b_18(D) + _2;
vect__4.27_142 = .MASK_FIRSTFAULT_LOAD (vectp_src1b.25_139, 128B,
loop_mask_93, { 0, ... });
_4 = *_3;
_5 = src1_20(D) + _2;
vect_v1_21.11_94 = .MASK_FIRSTFAULT_LOAD (vectp_src1.9_90, 128B,
loop_mask_93, { 0, ... });
v1_21 = *_5;
ffr_mask_165 = .READ_FAULT_STATE ();
if (ffr_mask_165 == { -1, ... })
goto <bb 23>; [99.95%]
else
goto <bb 24>; [0.05%]
<bb 24> [local count: 9309986]:
.SET_FAULT_STATE ({ -1, ... });
_166 = ~ffr_mask_165;
ffr_loop_mask_167 = loop_mask_93 & ffr_mask_165;
<bb 23> [local count: 930998582]:
# ffr_loop_mask_103 = PHI <loop_mask_93(3), ffr_loop_mask_167(24)>
# ffr_num_iters_13 = PHI <POLY_INT_CST [8, 8](3), 0(24)>
# next_mask_ffr_162 = PHI <{ -1, ... }(3), _166(24)>
_146 = VIEW_CONVERT_EXPR<vector([8,8]) short unsigned int>(vect__4.27_142);
_147 = VEC_COND_EXPR <ffr_loop_mask_103, { 1, ... }, { 0, ... }>;
_148 = test.simdclone.4 (_146, _147);
vect_v1b_19.28_149 = VIEW_CONVERT_EXPR<vector([8,8]) short unsigned
int>(_148);
mask_patt_15.29_150 = vect_v1b_19.28_149 == vect_v1_21.11_94;
vec_mask_and_151 = mask_patt_15.29_150 & ffr_loop_mask_103;
if (vec_mask_and_151 != { 0, ... })
goto <bb 21>; [3.66%]
else
goto <bb 4>; [96.34%]
<bb 21> [local count: 34074549]:
# i_183 = PHI <i_34(23)>
# _185 = PHI <_79(23)>
.SET_FAULT_STATE (ffr_preservation_164);
goto <bb 17>; [100.00%]
<bb 4> [local count: 896924033]:
_6 = src2_22(D) + _2;
vect_v2_23.14_104 = .MASK_FIRSTFAULT_LOAD (vectp_src2.12_100, 128B,
ffr_loop_mask_103, { 0, ... });
v2_23 = *_6;
ffr_mask_168 = .READ_FAULT_STATE ();
if (ffr_mask_168 == { -1, ... })
goto <bb 25>; [99.95%]
else
goto <bb 26>; [0.05%]
<bb 26> [local count: 8969240]:
.SET_FAULT_STATE ({ -1, ... });
_169 = ~ffr_mask_168;
ffr_loop_mask_170 = ffr_loop_mask_103 & ffr_mask_168;
<bb 25> [local count: 896924033]:
# ffr_loop_mask_114 = PHI <ffr_loop_mask_103(4), ffr_loop_mask_170(26)>
# ffr_num_iters_171 = PHI <ffr_num_iters_13(4), 0(26)>
# ffr_next_mask_172 = PHI <next_mask_ffr_162(4), _169(26)>
mask_patt_39.24_133 = vect_v2_23.14_104 == { 1, ... };
vect__8.15_106 = vect_v1_21.11_94 + vect_v2_23.14_104;
_7 = dst1_24(D) + _2;
_8 = v1_21 + v2_23;
vec_mask_and_134 = mask_patt_39.24_133 & ffr_loop_mask_114;
if (vec_mask_and_134 != { 0, ... })
goto <bb 22>; [3.66%]
else
goto <bb 5>; [96.34%]
<bb 22> [local count: 32827421]:
# i_184 = PHI <i_34(25)>
# _186 = PHI <_79(25)>
.SET_FAULT_STATE (ffr_preservation_164);
<bb 17> [local count: 66901970]:
# i_67 = PHI <i_184(22), i_183(21)>
# _190 = PHI <_186(22), _185(21)>
_83 = (signed long) _190;
_84 = MAX_EXPR <_83, 0>;
_85 = (unsigned long) _84;
_72 = _85;
tmp.6_73 = (int) _72;
goto <bb 18>; [100.00%]
<bb 5> [local count: 864096614]:
.MASK_STORE (vectp_dst1.16_111, 16B, ffr_loop_mask_114, vect__8.15_106);
_9 = src3_26(D) + _2;
vect_v3_27.20_123 = .MASK_LOAD (vectp_src3.18_120, 16B, loop_mask_93, { 0,
... });
vect__11.21_124 = vect__8.15_106 + vect_v3_27.20_123;
v3_27 = *_9;
_10 = dst2_28(D) + _2;
_11 = _8 + v3_27;
.MASK_STORE (vectp_dst2.22_129, 16B, ffr_loop_mask_114, vect__11.21_124);
i_30 = i_34 + 1;
i.0_12 = (unsigned int) i_30;
_80 = _79 + ffr_num_iters_13;
_92 = ffr_num_iters_13 * 2;
vectp_src1.9_91 = vectp_src1.9_90 + _92;
vectp_src2.12_101 = vectp_src2.12_100 + _92;
vectp_dst1.16_112 = vectp_dst1.16_111 + _92;
vectp_src3.18_121 = vectp_src3.18_120 + _92;
vectp_dst2.22_130 = vectp_dst2.22_129 + _92;
vectp_src1b.25_140 = vectp_src1b.25_139 + _92;
ivtmp_155 = ivtmp_154 + ffr_num_iters_13;
next_mask_161 = .WHILE_ULT (ivtmp_155, _156, { 0, ... });
next_mask_ffr_163 = next_mask_161 & next_mask_ffr_162;
if (next_mask_ffr_163 != { 0, ... })
goto <bb 8>; [96.34%]
else
goto <bb 20>; [3.66%]
<bb 8> [local count: 832470677]:
goto <bb 3>; [100.00%]
<bb 20> [local count: 31625937]:
# i_182 = PHI <i_30(5)>
.SET_FAULT_STATE (ffr_preservation_164);
_71 = n_17(D);
tmp.7_74 = (int) _71;
if (n_17(D) == _71)
goto <bb 6>; [12.50%]
else
goto <bb 18>; [87.50%]
<bb 18> [local count: 97159464]:
# i_69 = PHI <tmp.6_73(17), 0(7), tmp.7_74(20)>
<bb 12> [local count: 918068051]:
# i_41 = PHI <i_55(13), i_69(18)>
_43 = (long unsigned int) i_41;
_44 = _43 * 2;
_45 = src1b_18(D) + _44;
_46 = *_45;
v1b_47 = test (_46);
_48 = src1_20(D) + _44;
v1_49 = *_48;
if (v1b_47 == v1_49)
goto <bb 6>; [3.66%]
else
goto <bb 15>; [96.34%]
<bb 13> [local count: 820908588]:
goto <bb 12>; [100.00%]
<bb 14> [local count: 852095276]:
_50 = src3_26(D) + _44;
v3_51 = *_50;
_52 = dst2_28(D) + _44;
_53 = _60 + v3_51;
*_52 = _53;
i_55 = i_41 + 1;
i.0_56 = (unsigned int) i_55;
if (i.0_56 < n_17(D))
goto <bb 13>; [96.34%]
else
goto <bb 6>; [3.66%]
<bb 15> [local count: 884466760]:
_57 = src2_22(D) + _44;
v2_58 = *_57;
_59 = dst1_24(D) + _44;
_60 = v1_49 + v2_58;
*_59 = _60;
if (v2_58 == 1)
goto <bb 6>; [3.66%]
else
goto <bb 14>; [96.34%]
<bb 6> [local count: 113634474]:
return;
}
.arch armv8.2-a+crc+sve
.file "example2.c"
.text
.align 2
.p2align 5,,15
.global bar
.type bar, %function
bar:
.LFB13:
.cfi_startproc
stp x29, x30, [sp, -144]!
.cfi_def_cfa_offset 144
.cfi_offset 29, -144
.cfi_offset 30, -136
mov x29, sp
ldr w6, [sp, 144]
cbz w6, .L1
mov x7, x0
ldr w0, [sp, 144]
mov x8, x1
stp x19, x20, [sp, 16]
.cfi_offset 20, -120
.cfi_offset 19, -128
sub w1, w0, #1
stp x2, x3, [sp, 96]
mov x6, x2
cnth x0
stp x7, x4, [sp, 112]
stp x8, x5, [sp, 128]
cmp w1, w0
bcc .L13
stp x27, x28, [sp, 80]
.cfi_offset 28, -56
.cfi_offset 27, -64
cntb x28
setffr
sub x28, x28, #1
ptrue p15.h, all
and x28, x4, x28
ldr w0, [sp, 144]
and x2, x28, 8589934590
mov w20, 0
ubfx x28, x28, 1, 32
whilelo p14.h, xzr, x28
stp x21, x22, [sp, 32]
.cfi_offset 22, -104
.cfi_offset 21, -112
neg x19, x28
sub x22, x6, x2
add x28, x28, w0, uxtw
whilelo p6.h, xzr, x28
sub x21, x4, x2
not p6.b, p6/z, p14.b
stp x23, x24, [sp, 48]
.cfi_offset 24, -88
.cfi_offset 23, -96
sub x24, x7, x2
sub x23, x5, x2
stp x25, x26, [sp, 64]
.cfi_offset 26, -72
.cfi_offset 25, -80
sub x26, x8, x2
sub x25, x3, x2
b .L9
.p2align 2,,3
.L27:
ldff1h z31.h, p7/z, [x21]
ptrue p15.h, all
add z23.h, z23.h, z31.h
rdffr p13.b
nots p12.b, p15/z, p13.b
b.any .L24
.L7:
cmpeq p13.h, p7/z, z31.h, #1
b.any .L6
whilelo p13.h, x20, x28
st1h z23.h, p7, [x24]
ld1h z31.h, p6/z, [x23]
add z31.h, z31.h, z23.h
st1h z31.h, p7, [x26]
and p6.b, p13/z, p14.b, p14.b
add x19, x19, x27
add x22, x22, x27, lsl 1
ptest p15, p6.b
add x21, x21, x27, lsl 1
add x24, x24, x27, lsl 1
add x25, x25, x27, lsl 1
add x23, x23, x27, lsl 1
add x26, x26, x27, lsl 1
b.none .L25
.L9:
ldff1h z0.h, p6/z, [x25]
ldff1h z23.h, p6/z, [x22]
cnth x27
mov p7.b, p6.b
rdffr p14.b
nots p15.b, p15/z, p14.b
b.any .L26
.L4:
mov p0.b, p7.b
add x20, x20, x27
bl _ZGVsMxv_test
cmpeq p15.h, p7/z, z0.h, z23.h
b.none .L27
.L6:
ldp x21, x22, [sp, 32]
.cfi_restore 22
.cfi_restore 21
bic x19, x19, x19, asr #63
ldp x23, x24, [sp, 48]
.cfi_restore 24
.cfi_restore 23
ldp x25, x26, [sp, 64]
.cfi_restore 26
.cfi_restore 25
ldp x27, x28, [sp, 80]
.cfi_restore 28
.cfi_restore 27
.L3:
sxtw x19, w19
b .L11
.p2align 2,,3
.L20:
ldp x2, x0, [sp, 112]
ldrh w0, [x0, x19, lsl 1]
add w1, w1, w0
strh w1, [x2, x19, lsl 1]
cmp w0, 1
beq .L22
ldr x0, [sp, 136]
ldrh w0, [x0, x19, lsl 1]
add w1, w1, w0
ldr x0, [sp, 128]
strh w1, [x0, x19, lsl 1]
add x19, x19, 1
ldr w0, [sp, 144]
cmp w0, w19
bls .L22
.L11:
ldr x0, [sp, 104]
ldrh w0, [x0, x19, lsl 1]
bl test
ldr x1, [sp, 96]
ldrh w1, [x1, x19, lsl 1]
cmp w1, w0, uxth
bne .L20
.L22:
ldp x19, x20, [sp, 16]
.cfi_restore 20
.cfi_restore 19
.L1:
ldp x29, x30, [sp], 144
.cfi_restore 30
.cfi_restore 29
.cfi_def_cfa_offset 0
ret
.p2align 2,,3
.L25:
.cfi_def_cfa_offset 144
.cfi_offset 19, -128
.cfi_offset 20, -120
.cfi_offset 21, -112
.cfi_offset 22, -104
.cfi_offset 23, -96
.cfi_offset 24, -88
.cfi_offset 25, -80
.cfi_offset 26, -72
.cfi_offset 27, -64
.cfi_offset 28, -56
.cfi_offset 29, -144
.cfi_offset 30, -136
ldp x19, x20, [sp, 16]
.cfi_restore 20
.cfi_restore 19
ldp x21, x22, [sp, 32]
.cfi_restore 22
.cfi_restore 21
ldp x23, x24, [sp, 48]
.cfi_restore 24
.cfi_restore 23
ldp x25, x26, [sp, 64]
.cfi_restore 26
.cfi_restore 25
ldp x27, x28, [sp, 80]
.cfi_restore 28
.cfi_restore 27
ldp x29, x30, [sp], 144
.cfi_restore 30
.cfi_restore 29
.cfi_def_cfa_offset 0
ret
.p2align 2,,3
.L13:
.cfi_def_cfa_offset 144
.cfi_offset 19, -128
.cfi_offset 20, -120
.cfi_offset 29, -144
.cfi_offset 30, -136
mov w19, 0
b .L3
.L26:
.cfi_offset 21, -112
.cfi_offset 22, -104
.cfi_offset 23, -96
.cfi_offset 24, -88
.cfi_offset 25, -80
.cfi_offset 26, -72
.cfi_offset 27, -64
.cfi_offset 28, -56
and p7.b, p6/z, p14.b, p14.b
mov w27, 0
mov p14.b, p15.b
b .L4
.L24:
setffr
and p7.b, p7/z, p13.b, p13.b
b .L7
.cfi_endproc
.LFE13:
.size bar, .-bar
.variant_pcs _ZGVsMxv_test
.ident "GCC: (GNU) 17.0.0 20260622 (experimental)"
.section .note.GNU-stack,"",@progbits
#include <math.h>
#include <stdint.h>
#include <stdlib.h>
#include <arm_sve.h>
#define type uint16_t
type test [[gnu::simd, gnu::const]] (type);
void bar(
type * const restrict dst1,
type * const restrict dst2,
type * const restrict src1,
type * const restrict src1b,
type * const restrict src2,
type * const restrict src3,
svbool_t *p,
svbool_t *p2,
unsigned int n)
{
for (int i = 0; i < n; i++) {
type v1b = test(src1b[i]);
type v1 = src1[i];
if (v1 == v1b) {break;}
type v2 = src2[i];
dst1[i] = v1 + v2;
if (v2 == 1) {break;}
type v3 = src3[i];
dst2[i] = v1 + v2 + v3;
}
}
