Late RTL pass that inserts reg-free zero idioms (vpxor) in large loops
to break false dependencies on physical SSE registers. It walks
innermost loops in BFS order; when a register dying in one insn is
redefined far enough later, it emits a zeroing idiom at the kill point
as a hint to the hardware register renamer. The pass is gated on
TARGET_AVX && X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM (m_ZNVER, znver1-6)
and is tuned by the reg-free-zero-loop-threshold and
reg-free-zero-gap-threshold params.
gcc/ChangeLog:
* config/i386/i386-features.cc (class pass_insert_reg_free_zero_idiom):
New RTL pass.
(pass_insert_reg_free_zero_idiom::insert_zero_idiom_after): New method.
(pass_insert_reg_free_zero_idiom::execute): New method.
(make_pass_insert_reg_free_zero_idiom): New function.
* config/i386/i386-passes.def (INSERT_PASS_BEFORE): Register
pass_insert_reg_free_zero_idiom before pass_shorten_branches.
* config/i386/i386-protos.h (make_pass_insert_reg_free_zero_idiom):
Declare.
* config/i386/x86-tune.def (X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM):
New tune flag enabled for m_ZNVER.
* params.opt (-param=reg-free-zero-loop-threshold=): New param.
(-param=reg-free-zero-gap-threshold=): New param.
* doc/invoke.texi (x86 Options): Document the
insert_reg_free_zero_idiom tuning feature and the
reg-free-zero-loop-threshold and reg-free-zero-gap-threshold
parameters.
gcc/testsuite/ChangeLog:
* gcc.target/i386/reg-free-zero-1.c: New test.
* gcc.target/i386/reg-free-zero-2.c: New test.
* gcc.target/i386/reg-free-zero-3.c: New test.
* gcc.target/i386/reg-free-zero-4.c: New test.
* gcc.target/i386/reg-free-zero-5.c: New test.
* gcc.target/i386/reg-free-zero-6.c: New test.
* gcc.target/i386/reg-free-zero-7.c: New test.
* gcc.target/i386/reg-free-zero-8.c: New test.
Co-authored-by: Venkataramanan Kumar <[email protected]>
Signed-off-by: Sarvesh Chandra <[email protected]>
---
Bootstrapped and regression-tested on x86_64-pc-linux-gnu.
gcc/config/i386/i386-features.cc | 215 ++++++++++++++++++
gcc/config/i386/i386-passes.def | 1 +
gcc/config/i386/i386-protos.h | 1 +
gcc/config/i386/x86-tune.def | 6 +
gcc/doc/invoke.texi | 24 ++
gcc/params.opt | 8 +
.../gcc.target/i386/reg-free-zero-1.c | 31 +++
.../gcc.target/i386/reg-free-zero-2.c | 31 +++
.../gcc.target/i386/reg-free-zero-3.c | 36 +++
.../gcc.target/i386/reg-free-zero-4.c | 30 +++
.../gcc.target/i386/reg-free-zero-5.c | 12 +
.../gcc.target/i386/reg-free-zero-6.c | 72 ++++++
.../gcc.target/i386/reg-free-zero-7.c | 30 +++
.../gcc.target/i386/reg-free-zero-8.c | 30 +++
14 files changed, 527 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
index eb5f82e42cb..e713a23117b 100644
--- a/gcc/config/i386/i386-features.cc
+++ b/gcc/config/i386/i386-features.cc
@@ -3118,6 +3118,221 @@ make_pass_stv (gcc::context *ctxt)
return new pass_stv (ctxt);
}
+
+/* Pass to insert reg-free zero idioms (vpxor) in large loops to break
+ false dependencies on physical SSE/AVX registers. Walks innermost
+ loops in BFS order; when a register dying in one insn is redefined
+ far enough later, emits a zeroing idiom at the kill point as a hint
+ to the hardware renamer. Gated on TARGET_AVX and
+ X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM (m_ZNVER). */
+
+namespace {
+
+#define LOOP_INSN_THRESHOLD ((int) param_reg_free_zero_loop_threshold)
+#define DEAD_GAP_THRESHOLD ((int) param_reg_free_zero_gap_threshold)
+
+struct deferred_insertion
+{
+ rtx_insn *after;
+ unsigned int regno;
+};
+
+const pass_data pass_data_insert_reg_free_zero_idiom =
+{
+ RTL_PASS, /* type */
+ "insert_reg_free_zero_idiom", /* name */
+ OPTGROUP_NONE, /* optinfo_flags */
+ TV_MACH_DEP, /* tv_id */
+ 0, /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ TODO_df_finish, /* todo_flags_finish */
+};
+
+class pass_insert_reg_free_zero_idiom : public rtl_opt_pass
+{
+public:
+ pass_insert_reg_free_zero_idiom (gcc::context *ctxt)
+ : rtl_opt_pass (pass_data_insert_reg_free_zero_idiom, ctxt)
+ {}
+
+ bool gate (function *) final override
+ {
+ return TARGET_AVX
+ && ix86_tune_features[X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM];
+ }
+
+ unsigned int execute (function *) final override;
+
+private:
+ void insert_zero_idiom_after (rtx_insn *after_insn, unsigned int regno);
+};
+
+/* Emit a self-XOR of SSE register REGNO after AFTER_INSN. */
+void
+pass_insert_reg_free_zero_idiom::insert_zero_idiom_after (rtx_insn *after_insn,
+ unsigned int regno)
+{
+ if (JUMP_P (after_insn) || CALL_P (after_insn))
+ return;
+
+ machine_mode mode;
+ if (EXT_REX_SSE_REGNO_P (regno) && !TARGET_AVX512VL)
+ mode = V16SImode;
+ else
+ mode = V4SImode;
+
+ rtx reg = gen_rtx_REG (mode, regno);
+ rtx pat = gen_rtx_SET (reg, gen_rtx_XOR (mode, reg, reg));
+ rtx_insn *new_insn = emit_insn_after (pat, after_insn);
+
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "Inserted reg-free zero idiom for reg %d (insn %d) "
+ "after insn %d\n",
+ regno, INSN_UID (new_insn), INSN_UID (after_insn));
+ print_rtl_single (dump_file, new_insn);
+ }
+}
+
+unsigned int
+pass_insert_reg_free_zero_idiom::execute (function *fun)
+{
+ loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
+ df_note_add_problem ();
+ df_analyze ();
+
+ rtx_insn **dead_insn = XCNEWVEC (rtx_insn *, FIRST_PSEUDO_REGISTER);
+ int *dead_at = XCNEWVEC (int, FIRST_PSEUDO_REGISTER);
+ int *bb_entry_count = XCNEWVEC (int, last_basic_block_for_fn (fun));
+ auto_vec<basic_block> bbs;
+ auto_vec<deferred_insertion> insertions;
+
+ for (auto curr_loop : loops_list (fun, LI_ONLY_INNERMOST))
+ {
+ if (num_loop_insns (curr_loop) < LOOP_INSN_THRESHOLD)
+ continue;
+
+ memset (dead_insn, 0, FIRST_PSEUDO_REGISTER * sizeof (rtx_insn *));
+ memset (dead_at, 0, FIRST_PSEUDO_REGISTER * sizeof (int));
+ memset (bb_entry_count, 0,
+ last_basic_block_for_fn (fun) * sizeof (int));
+ bbs.truncate (0);
+ insertions.truncate (0);
+
+ auto_bitmap visited;
+ bbs.safe_push (curr_loop->header);
+ bitmap_set_bit (visited, curr_loop->header->index);
+ unsigned int bfs_head = 0;
+
+ while (bfs_head < bbs.length ())
+ {
+ basic_block bb = bbs[bfs_head++];
+ int insn_count = bb_entry_count[bb->index];
+
+ rtx_insn *insn;
+ FOR_BB_INSNS (bb, insn)
+ {
+ if (!INSN_P (insn))
+ continue;
+
+ df_ref use;
+ FOR_EACH_INSN_USE (use, insn)
+ {
+ unsigned int regno = DF_REF_REGNO (use);
+ if (!SSE_REGNO_P (regno))
+ continue;
+
+ if (find_regno_note (insn, REG_DEAD, regno))
+ {
+ dead_insn[regno] = insn;
+ dead_at[regno] = insn_count;
+ }
+ }
+
+ df_ref def;
+ FOR_EACH_INSN_DEF (def, insn)
+ {
+ unsigned int regno = DF_REF_REGNO (def);
+ if (!SSE_REGNO_P (regno))
+ continue;
+
+ if (dead_insn[regno])
+ {
+ int gap = insn_count - dead_at[regno];
+
+ if (gap > DEAD_GAP_THRESHOLD)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "BB %d: reg %d dead at insn %d, "
+ "redef at insn %d, gap=%d\n",
+ bb->index, regno,
+ INSN_UID (dead_insn[regno]),
+ INSN_UID (insn), gap);
+ deferred_insertion d
+ = { dead_insn[regno], regno };
+ insertions.safe_push (d);
+ }
+ dead_insn[regno] = NULL;
+ }
+ }
+
+ insn_count++;
+ }
+
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ if (!flow_bb_inside_loop_p (curr_loop, e->dest))
+ continue;
+
+ if (bitmap_set_bit (visited, e->dest->index))
+ {
+ bbs.safe_push (e->dest);
+ bb_entry_count[e->dest->index] = insn_count;
+ }
+ else if (insn_count > bb_entry_count[e->dest->index])
+ {
+ bb_entry_count[e->dest->index] = insn_count;
+ }
+ }
+ }
+
+ int loop_zero_count = 0;
+ for (unsigned j = 0; j < insertions.length (); j++)
+ {
+ insert_zero_idiom_after (insertions[j].after, insertions[j].regno);
+ loop_zero_count++;
+ }
+
+ if (dump_file && loop_zero_count > 0)
+ fprintf (dump_file,
+ "Loop %d: inserted %d reg-free zero idiom(s)\n",
+ curr_loop->num, loop_zero_count);
+ }
+
+ free (dead_insn);
+ free (dead_at);
+ free (bb_entry_count);
+ loop_optimizer_finalize ();
+ return 0;
+}
+
+#undef LOOP_INSN_THRESHOLD
+#undef DEAD_GAP_THRESHOLD
+
+} // anon namespace
+
+rtl_opt_pass *
+make_pass_insert_reg_free_zero_idiom (gcc::context *ctxt)
+{
+ return new pass_insert_reg_free_zero_idiom (ctxt);
+}
+
/* Inserting ENDBR and pseudo patchable-area instructions. */
static void
diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
index 4f461116cac..64e287c86d2 100644
--- a/gcc/config/i386/i386-passes.def
+++ b/gcc/config/i386/i386-passes.def
@@ -38,3 +38,4 @@ along with GCC; see the file COPYING3. If not see
INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
INSERT_PASS_AFTER (pass_rtl_ifcvt, 1, pass_apx_nf_convert);
+ INSERT_PASS_BEFORE (pass_shorten_branches, 1,
pass_insert_reg_free_zero_idiom);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index 2687de646b0..12df1ef5285 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -435,6 +435,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
+extern rtl_opt_pass *make_pass_insert_reg_free_zero_idiom (gcc::context *);
extern bool ix86_has_no_direct_extern_access;
extern bool ix86_rpad_gate ();
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index d4c27351ad7..d7787f6b984 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -812,3 +812,9 @@ DEF_TUNE (X86_TUNE_SLOW_STC, "slow_stc", m_PENT4)
/* X86_TUNE_USE_RCR: Controls use of rcr 1 instruction instead of shrd. */
DEF_TUNE (X86_TUNE_USE_RCR, "use_rcr", m_AMD_MULTIPLE | m_C86_4G)
+
+/* X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM: Insert register-free zero idiom
+ instructions to break false dependencies on vector registers for
+ Zen microarchitectures. */
+DEF_TUNE (X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM, "insert_reg_free_zero_idiom",
+ m_ZNVER)
diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index e1870c9c227..c528872c7e5 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -36684,6 +36684,30 @@ potentially result in compiler ICEs or runtime errors.
This option instructs GCC to turn off all tunable features. See also
@option{-mtune-ctrl=@var{feature-list}} and @option{-mdump-tune-features}.
+@opindex mtune-ctrl=insert_reg_free_zero_idiom
+@item -mtune-ctrl=insert_reg_free_zero_idiom
+When this tuning feature is enabled, GCC runs a late RTL pass that inserts
+register-free zero idioms (such as @code{vpxor}) into large innermost loops
+in order to break false dependencies on physical vector registers. A zeroing
+idiom is emitted at the point where a vector register last dies before being
+redefined much later in the loop, which acts as a hint to the hardware
+register renamer. This feature is enabled by default for AMD Zen
+(@samp{znver1} through @samp{znver6}) processors when @option{-mavx} is in
+effect. Its behavior is controlled by the
+@option{--param=reg-free-zero-loop-threshold} and
+@option{--param=reg-free-zero-gap-threshold} parameters described below.
+
+@item --param=reg-free-zero-loop-threshold=@var{n}
+When the @code{insert_reg_free_zero_idiom} tuning feature is enabled, only
+innermost loops containing at least @var{n} RTL instructions are considered
+for zero-idiom insertion. The default is 500.
+
+@item --param=reg-free-zero-gap-threshold=@var{n}
+When the @code{insert_reg_free_zero_idiom} tuning feature is enabled, a zero
+idiom is inserted only when the gap, measured in RTL instructions, between the
+last use of a vector register and its next redefinition is greater than
+@var{n}. The default is 80.
+
@opindex mcld
@opindex mno-cld
@item -mcld
diff --git a/gcc/params.opt b/gcc/params.opt
index 0c6a3e51457..6579d526343 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -1333,4 +1333,12 @@ When nonzero, the widening-multiply pass defers forming
an FMA whose result
feeds a loop-header PHI, leaving a separate multiply and add. Set to zero to
contract such loop-carried reductions into an FMA instead.
+-param=reg-free-zero-loop-threshold=
+Common Joined UInteger Var(param_reg_free_zero_loop_threshold) Init(500)
IntegerRange(0, 65536) Param Optimization
+Minimum number of RTL instructions in a loop for the
insert_reg_free_zero_idiom pass to consider it.
+
+-param=reg-free-zero-gap-threshold=
+Common Joined UInteger Var(param_reg_free_zero_gap_threshold) Init(80)
IntegerRange(0, 65536) Param Optimization
+Minimum instruction gap between last use and redefinition of an SSE register
for insert_reg_free_zero_idiom to insert a zero idiom.
+
; This comment is to ensure we retain the blank line above.
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
new file mode 100644
index 00000000000..ce27cea8c62
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 1
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 1 } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ v8sf vc = *(v8sf *)(c + i);
+ *(v8sf *)(a + i) = vb * vc;
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ v8sf vb2 = *(v8sf *)(b + i + 8);
+ *(v8sf *)(a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
new file mode 100644
index 00000000000..b20266c18e7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 1
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 1 } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ *(v8sf *)(a + i) = vb * vb;
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ v8sf va = *(v8sf *)(a + i + 8);
+ v8sf vc = *(v8sf *)(c + i + 8);
+ *(v8sf *)(a + i + 8) = va * vc + va;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
new file mode 100644
index 00000000000..aa64ae20b66
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 1
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 1 } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n, int cond)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ v8sf vc = *(v8sf *)(c + i);
+ *(v8sf *)(a + i) = vb * vc;
+
+ if (cond)
+ {
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+ }
+ else
+ s[i] = (float) (cond * 2);
+
+ v8sf vb2 = *(v8sf *)(b + i + 8);
+ *(v8sf *)(a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
new file mode 100644
index 00000000000..782ec9994c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=2
--param=reg-free-zero-gap-threshold=10000
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-not "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ v8sf vc = *(v8sf *)(c + i);
+ *(v8sf *)(a + i) = vb * vc;
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ v8sf vb2 = *(v8sf *)(b + i + 8);
+ *(v8sf *)(a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
new file mode 100644
index 00000000000..9e48950639e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10000
--param=reg-free-zero-gap-threshold=1 -fdump-rtl-insert_reg_free_zero_idiom" }
*/
+/* { dg-final { scan-rtl-dump-not "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, int n)
+{
+ for (int i = 0; i < n; i++)
+ a[i] = a[i] + 1.0f;
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
new file mode 100644
index 00000000000..471fd620008
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
@@ -0,0 +1,72 @@
+/* { dg-do run { target avx_runtime } } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom -save-temps" } */
+/* { dg-final { scan-rtl-dump "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler "vpxor\[ \\t\]+\[^\n\]*%xmm" } } */
+
+#include <stdlib.h>
+
+#define N 256
+
+/* Scalar, so no zero idiom here; the idiom goes in main's vector loop. */
+static void __attribute__((noipa, optimize ("no-tree-vectorize")))
+work (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i + 16 <= n; i += 15)
+ {
+ for (int k = 0; k < 8; k++)
+ a[i + k] = b[i + k] * c[i + k];
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ for (int k = 0; k < 8; k++)
+ a[i + 8 + k] = b[i + 8 + k] + c[i + k];
+ }
+}
+
+int
+main (void)
+{
+ static float a[N], b[N], c[N], s[N];
+ static float a_ref[N], s_ref[N];
+
+ for (int i = 0; i < N; i++)
+ {
+ b[i] = (float) (i + 1);
+ c[i] = (float) ((i * 3) + 2);
+ }
+
+ for (int i = 0; i + 16 <= N; i += 15)
+ {
+ for (int k = 0; k < 8; k++)
+ a_ref[i + k] = b[i + k] * c[i + k];
+
+ int t = (int) a_ref[i];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s_ref[i] = (float) t;
+
+ for (int k = 0; k < 8; k++)
+ a_ref[i + 8 + k] = b[i + 8 + k] + c[i + k];
+ }
+
+ work (a, b, c, s, N);
+
+ for (int i = 0; i < N; i++)
+ if (a[i] != a_ref[i] || s[i] != s_ref[i])
+ abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
new file mode 100644
index 00000000000..72df1310882
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
@@ -0,0 +1,30 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-require-effective-target avx512vl } */
+/* { dg-options "-O3 -mavx512vl -ffixed-xmm0 -ffixed-xmm1 -ffixed-xmm2
-ffixed-xmm3 -ffixed-xmm4 -ffixed-xmm5 -ffixed-xmm6 -ffixed-xmm7 -ffixed-xmm8
-ffixed-xmm9 -ffixed-xmm10 -ffixed-xmm11 -ffixed-xmm12 -ffixed-xmm13
-ffixed-xmm14 -ffixed-xmm15 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=1 --param=reg-free-zero-gap-threshold=1
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler "vpxord\[
\\t\]+%xmm(1\[6-9\]|2\[0-9\]|3\[01\])" } } */
+
+typedef long long v8di __attribute__((vector_size (64)));
+
+void
+test (long long *__restrict a, long long *__restrict b,
+ long long *__restrict c, long long *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8di vb = *(v8di *) (b + i);
+ v8di vc = *(v8di *) (c + i);
+ *(v8di *) (a + i) = vb * vc;
+
+ v8di g = *(v8di *) (a + i);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 3) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ *(v8di *) (s + i) = g;
+
+ v8di vb2 = *(v8di *) (b + i + 8);
+ *(v8di *) (a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
new file mode 100644
index 00000000000..efe39a2df1b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
@@ -0,0 +1,30 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-require-effective-target avx512f } */
+/* { dg-options "-O3 -mavx512f -mno-avx512vl -ffixed-xmm0 -ffixed-xmm1
-ffixed-xmm2 -ffixed-xmm3 -ffixed-xmm4 -ffixed-xmm5 -ffixed-xmm6 -ffixed-xmm7
-ffixed-xmm8 -ffixed-xmm9 -ffixed-xmm10 -ffixed-xmm11 -ffixed-xmm12
-ffixed-xmm13 -ffixed-xmm14 -ffixed-xmm15
-mtune-ctrl=insert_reg_free_zero_idiom --param=reg-free-zero-loop-threshold=1
--param=reg-free-zero-gap-threshold=1 -fdump-rtl-insert_reg_free_zero_idiom" }
*/
+/* { dg-final { scan-rtl-dump "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler "vpxord\[
\\t\]+%zmm(1\[6-9\]|2\[0-9\]|3\[01\])" } } */
+
+typedef long long v8di __attribute__((vector_size (64)));
+
+void
+test (long long *__restrict a, long long *__restrict b,
+ long long *__restrict c, long long *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8di vb = *(v8di *) (b + i);
+ v8di vc = *(v8di *) (c + i);
+ *(v8di *) (a + i) = vb * vc;
+
+ v8di g = *(v8di *) (a + i);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 3) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ *(v8di *) (s + i) = g;
+
+ v8di vb2 = *(v8di *) (b + i + 8);
+ *(v8di *) (a + i + 8) = vb2 + vc;
+ }
+}
--
2.34.1
