Late RTL pass that inserts reg-free zero idioms (vpxor) in large loops
to break false dependencies on physical SSE registers. It walks
innermost loops in BFS order; when a register dying in one insn is
redefined far enough later, it emits a zeroing idiom at the kill point
as a hint to the hardware register renamer. The pass is gated on
TARGET_AVX && X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM (m_ZNVER, znver1-6)
and is tuned by the reg-free-zero-loop-threshold and
reg-free-zero-gap-threshold params.
gcc/ChangeLog:
* config/i386/i386-features.cc (class pass_insert_reg_free_zero_idiom):
New RTL pass.
(pass_insert_reg_free_zero_idiom::insert_zero_idiom_after): New method.
(pass_insert_reg_free_zero_idiom::execute): New method.
(make_pass_insert_reg_free_zero_idiom): New function.
* config/i386/i386-passes.def (INSERT_PASS_BEFORE): Register
pass_insert_reg_free_zero_idiom before pass_shorten_branches.
* config/i386/i386-protos.h (make_pass_insert_reg_free_zero_idiom):
Declare.
* config/i386/x86-tune.def (X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM):
New tune flag enabled for m_ZNVER.
* params.opt (-param=reg-free-zero-loop-threshold=): New param.
(-param=reg-free-zero-gap-threshold=): New param.
gcc/testsuite/ChangeLog:
* gcc.target/i386/reg-free-zero-1.c: New test.
* gcc.target/i386/reg-free-zero-2.c: New test.
* gcc.target/i386/reg-free-zero-3.c: New test.
* gcc.target/i386/reg-free-zero-4.c: New test.
* gcc.target/i386/reg-free-zero-5.c: New test.
* gcc.target/i386/reg-free-zero-6.c: New test.
* gcc.target/i386/reg-free-zero-7.c: New test.
* gcc.target/i386/reg-free-zero-8.c: New test.
Co-authored-by: Venkataramanan Kumar <[email protected]>
Signed-off-by: Sarvesh Chandra <[email protected]>
---
Bootstrapped and regression-tested on x86_64-pc-linux-gnu.
gcc/config/i386/i386-features.cc | 215 ++++++++++++++++++
gcc/config/i386/i386-passes.def | 1 +
gcc/config/i386/i386-protos.h | 1 +
gcc/config/i386/x86-tune.def | 6 +
gcc/params.opt | 8 +
.../gcc.target/i386/reg-free-zero-1.c | 31 +++
.../gcc.target/i386/reg-free-zero-2.c | 31 +++
.../gcc.target/i386/reg-free-zero-3.c | 36 +++
.../gcc.target/i386/reg-free-zero-4.c | 30 +++
.../gcc.target/i386/reg-free-zero-5.c | 12 +
.../gcc.target/i386/reg-free-zero-6.c | 71 ++++++
.../gcc.target/i386/reg-free-zero-7.c | 29 +++
.../gcc.target/i386/reg-free-zero-8.c | 29 +++
13 files changed, 500 insertions(+)
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
create mode 100644 gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
diff --git a/gcc/config/i386/i386-features.cc b/gcc/config/i386/i386-features.cc
index e0fc13efe86..0a3f6857e63 100644
--- a/gcc/config/i386/i386-features.cc
+++ b/gcc/config/i386/i386-features.cc
@@ -3117,6 +3117,221 @@ make_pass_stv (gcc::context *ctxt)
return new pass_stv (ctxt);
}
+
+/* Pass to insert reg-free zero idioms (vpxor) in large loops to break
+ false dependencies on physical SSE/AVX registers. Walks innermost
+ loops in BFS order; when a register dying in one insn is redefined
+ far enough later, emits a zeroing idiom at the kill point as a hint
+ to the hardware renamer. Gated on TARGET_AVX and
+ X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM (m_ZNVER). */
+
+namespace {
+
+#define LOOP_INSN_THRESHOLD ((int) param_reg_free_zero_loop_threshold)
+#define DEAD_GAP_THRESHOLD ((int) param_reg_free_zero_gap_threshold)
+
+struct deferred_insertion
+{
+ rtx_insn *after;
+ unsigned int regno;
+};
+
+const pass_data pass_data_insert_reg_free_zero_idiom =
+{
+ RTL_PASS, /* type */
+ "insert_reg_free_zero_idiom", /* name */
+ OPTGROUP_NONE, /* optinfo_flags */
+ TV_MACH_DEP, /* tv_id */
+ 0, /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ TODO_df_finish, /* todo_flags_finish */
+};
+
+class pass_insert_reg_free_zero_idiom : public rtl_opt_pass
+{
+public:
+ pass_insert_reg_free_zero_idiom (gcc::context *ctxt)
+ : rtl_opt_pass (pass_data_insert_reg_free_zero_idiom, ctxt)
+ {}
+
+ bool gate (function *) final override
+ {
+ return TARGET_AVX
+ && ix86_tune_features[X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM];
+ }
+
+ unsigned int execute (function *) final override;
+
+private:
+ void insert_zero_idiom_after (rtx_insn *after_insn, unsigned int regno);
+};
+
+/* Emit a self-XOR of SSE register REGNO after AFTER_INSN. */
+void
+pass_insert_reg_free_zero_idiom::insert_zero_idiom_after (rtx_insn *after_insn,
+ unsigned int regno)
+{
+ if (JUMP_P (after_insn) || CALL_P (after_insn))
+ return;
+
+ machine_mode mode;
+ if (EXT_REX_SSE_REGNO_P (regno) && !TARGET_AVX512VL)
+ mode = V16SImode;
+ else
+ mode = V4SImode;
+
+ rtx reg = gen_rtx_REG (mode, regno);
+ rtx pat = gen_rtx_SET (reg, gen_rtx_XOR (mode, reg, reg));
+ rtx_insn *new_insn = emit_insn_after (pat, after_insn);
+
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "Inserted reg-free zero idiom for reg %d (insn %d) "
+ "after insn %d\n",
+ regno, INSN_UID (new_insn), INSN_UID (after_insn));
+ print_rtl_single (dump_file, new_insn);
+ }
+}
+
+unsigned int
+pass_insert_reg_free_zero_idiom::execute (function *fun)
+{
+ loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
+ df_note_add_problem ();
+ df_analyze ();
+
+ rtx_insn **dead_insn = XCNEWVEC (rtx_insn *, FIRST_PSEUDO_REGISTER);
+ int *dead_at = XCNEWVEC (int, FIRST_PSEUDO_REGISTER);
+ int *bb_entry_count = XCNEWVEC (int, last_basic_block_for_fn (fun));
+ auto_vec<basic_block> bbs;
+ auto_vec<deferred_insertion> insertions;
+
+ for (auto curr_loop : loops_list (fun, LI_ONLY_INNERMOST))
+ {
+ if (num_loop_insns (curr_loop) < LOOP_INSN_THRESHOLD)
+ continue;
+
+ memset (dead_insn, 0, FIRST_PSEUDO_REGISTER * sizeof (rtx_insn *));
+ memset (dead_at, 0, FIRST_PSEUDO_REGISTER * sizeof (int));
+ memset (bb_entry_count, 0,
+ last_basic_block_for_fn (fun) * sizeof (int));
+ bbs.truncate (0);
+ insertions.truncate (0);
+
+ auto_bitmap visited;
+ bbs.safe_push (curr_loop->header);
+ bitmap_set_bit (visited, curr_loop->header->index);
+ unsigned int bfs_head = 0;
+
+ while (bfs_head < bbs.length ())
+ {
+ basic_block bb = bbs[bfs_head++];
+ int insn_count = bb_entry_count[bb->index];
+
+ rtx_insn *insn;
+ FOR_BB_INSNS (bb, insn)
+ {
+ if (!INSN_P (insn))
+ continue;
+
+ df_ref use;
+ FOR_EACH_INSN_USE (use, insn)
+ {
+ unsigned int regno = DF_REF_REGNO (use);
+ if (!SSE_REGNO_P (regno))
+ continue;
+
+ if (find_regno_note (insn, REG_DEAD, regno))
+ {
+ dead_insn[regno] = insn;
+ dead_at[regno] = insn_count;
+ }
+ }
+
+ df_ref def;
+ FOR_EACH_INSN_DEF (def, insn)
+ {
+ unsigned int regno = DF_REF_REGNO (def);
+ if (!SSE_REGNO_P (regno))
+ continue;
+
+ if (dead_insn[regno])
+ {
+ int gap = insn_count - dead_at[regno];
+
+ if (gap > DEAD_GAP_THRESHOLD)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "BB %d: reg %d dead at insn %d, "
+ "redef at insn %d, gap=%d\n",
+ bb->index, regno,
+ INSN_UID (dead_insn[regno]),
+ INSN_UID (insn), gap);
+ deferred_insertion d
+ = { dead_insn[regno], regno };
+ insertions.safe_push (d);
+ }
+ dead_insn[regno] = NULL;
+ }
+ }
+
+ insn_count++;
+ }
+
+ edge e;
+ edge_iterator ei;
+ FOR_EACH_EDGE (e, ei, bb->succs)
+ {
+ if (!flow_bb_inside_loop_p (curr_loop, e->dest))
+ continue;
+
+ if (bitmap_set_bit (visited, e->dest->index))
+ {
+ bbs.safe_push (e->dest);
+ bb_entry_count[e->dest->index] = insn_count;
+ }
+ else if (insn_count > bb_entry_count[e->dest->index])
+ {
+ bb_entry_count[e->dest->index] = insn_count;
+ }
+ }
+ }
+
+ int loop_zero_count = 0;
+ for (unsigned j = 0; j < insertions.length (); j++)
+ {
+ insert_zero_idiom_after (insertions[j].after, insertions[j].regno);
+ loop_zero_count++;
+ }
+
+ if (dump_file && loop_zero_count > 0)
+ fprintf (dump_file,
+ "Loop %d: inserted %d reg-free zero idiom(s)\n",
+ curr_loop->num, loop_zero_count);
+ }
+
+ free (dead_insn);
+ free (dead_at);
+ free (bb_entry_count);
+ loop_optimizer_finalize ();
+ return 0;
+}
+
+#undef LOOP_INSN_THRESHOLD
+#undef DEAD_GAP_THRESHOLD
+
+} // anon namespace
+
+rtl_opt_pass *
+make_pass_insert_reg_free_zero_idiom (gcc::context *ctxt)
+{
+ return new pass_insert_reg_free_zero_idiom (ctxt);
+}
+
/* Inserting ENDBR and pseudo patchable-area instructions. */
static void
diff --git a/gcc/config/i386/i386-passes.def b/gcc/config/i386/i386-passes.def
index 4f461116cac..64e287c86d2 100644
--- a/gcc/config/i386/i386-passes.def
+++ b/gcc/config/i386/i386-passes.def
@@ -38,3 +38,4 @@ along with GCC; see the file COPYING3. If not see
INSERT_PASS_AFTER (pass_late_combine, 1, pass_x86_cse);
INSERT_PASS_AFTER (pass_late_combine, 1, pass_remove_partial_avx_dependency);
INSERT_PASS_AFTER (pass_rtl_ifcvt, 1, pass_apx_nf_convert);
+ INSERT_PASS_BEFORE (pass_shorten_branches, 1,
pass_insert_reg_free_zero_idiom);
diff --git a/gcc/config/i386/i386-protos.h b/gcc/config/i386/i386-protos.h
index d11fa2df35d..5627c47ac80 100644
--- a/gcc/config/i386/i386-protos.h
+++ b/gcc/config/i386/i386-protos.h
@@ -438,6 +438,7 @@ extern rtl_opt_pass *make_pass_remove_partial_avx_dependency
extern rtl_opt_pass *make_pass_x86_cse (gcc::context *);
extern rtl_opt_pass *make_pass_apx_nf_convert (gcc::context *);
extern rtl_opt_pass *make_pass_align_tight_loops (gcc::context *);
+extern rtl_opt_pass *make_pass_insert_reg_free_zero_idiom (gcc::context *);
extern bool ix86_has_no_direct_extern_access;
extern bool ix86_rpad_gate ();
diff --git a/gcc/config/i386/x86-tune.def b/gcc/config/i386/x86-tune.def
index d4c27351ad7..d7787f6b984 100644
--- a/gcc/config/i386/x86-tune.def
+++ b/gcc/config/i386/x86-tune.def
@@ -812,3 +812,9 @@ DEF_TUNE (X86_TUNE_SLOW_STC, "slow_stc", m_PENT4)
/* X86_TUNE_USE_RCR: Controls use of rcr 1 instruction instead of shrd. */
DEF_TUNE (X86_TUNE_USE_RCR, "use_rcr", m_AMD_MULTIPLE | m_C86_4G)
+
+/* X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM: Insert register-free zero idiom
+ instructions to break false dependencies on vector registers for
+ Zen microarchitectures. */
+DEF_TUNE (X86_TUNE_INSERT_REG_FREE_ZERO_IDIOM, "insert_reg_free_zero_idiom",
+ m_ZNVER)
diff --git a/gcc/params.opt b/gcc/params.opt
index 90f9943c8cb..bc9b688d059 100644
--- a/gcc/params.opt
+++ b/gcc/params.opt
@@ -1323,4 +1323,12 @@ Maximum number of outgoing edges in a switch before VRP
does not process it.
Common Joined UInteger Var(param_vrp_vector_threshold) Init(250) Optimization
Param
Maximum number of basic blocks for VRP to use a basic cache vector.
+-param=reg-free-zero-loop-threshold=
+Common Joined UInteger Var(param_reg_free_zero_loop_threshold) Init(500)
IntegerRange(0, 65536) Param Optimization
+Minimum number of RTL instructions in a loop for the
insert_reg_free_zero_idiom pass to consider it.
+
+-param=reg-free-zero-gap-threshold=
+Common Joined UInteger Var(param_reg_free_zero_gap_threshold) Init(80)
IntegerRange(0, 65536) Param Optimization
+Minimum instruction gap between last use and redefinition of an SSE register
for insert_reg_free_zero_idiom to insert a zero idiom.
+
; This comment is to ensure we retain the blank line above.
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
new file mode 100644
index 00000000000..ce27cea8c62
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-1.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 1
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 1 } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ v8sf vc = *(v8sf *)(c + i);
+ *(v8sf *)(a + i) = vb * vc;
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ v8sf vb2 = *(v8sf *)(b + i + 8);
+ *(v8sf *)(a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
new file mode 100644
index 00000000000..b20266c18e7
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-2.c
@@ -0,0 +1,31 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 1
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 1 } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ *(v8sf *)(a + i) = vb * vb;
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ v8sf va = *(v8sf *)(a + i + 8);
+ v8sf vc = *(v8sf *)(c + i + 8);
+ *(v8sf *)(a + i + 8) = va * vc + va;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
new file mode 100644
index 00000000000..aa64ae20b66
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-3.c
@@ -0,0 +1,36 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 1
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 1 } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n, int cond)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ v8sf vc = *(v8sf *)(c + i);
+ *(v8sf *)(a + i) = vb * vc;
+
+ if (cond)
+ {
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+ }
+ else
+ s[i] = (float) (cond * 2);
+
+ v8sf vb2 = *(v8sf *)(b + i + 8);
+ *(v8sf *)(a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
new file mode 100644
index 00000000000..782ec9994c8
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-4.c
@@ -0,0 +1,30 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=2
--param=reg-free-zero-gap-threshold=10000
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump-not "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8sf vb = *(v8sf *)(b + i);
+ v8sf vc = *(v8sf *)(c + i);
+ *(v8sf *)(a + i) = vb * vc;
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ v8sf vb2 = *(v8sf *)(b + i + 8);
+ *(v8sf *)(a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
new file mode 100644
index 00000000000..9e48950639e
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-5.c
@@ -0,0 +1,12 @@
+/* { dg-do compile } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10000
--param=reg-free-zero-gap-threshold=1 -fdump-rtl-insert_reg_free_zero_idiom" }
*/
+/* { dg-final { scan-rtl-dump-not "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+
+typedef float v8sf __attribute__((vector_size (32)));
+
+void __attribute__((noipa))
+test (float *__restrict a, int n)
+{
+ for (int i = 0; i < n; i++)
+ a[i] = a[i] + 1.0f;
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
new file mode 100644
index 00000000000..57bb9303f28
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-6.c
@@ -0,0 +1,71 @@
+/* { dg-do run { target avx_runtime } } */
+/* { dg-options "-O3 -mavx2 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=10 --param=reg-free-zero-gap-threshold=5
-fdump-rtl-insert_reg_free_zero_idiom -save-temps" } */
+/* { dg-final { scan-rtl-dump-times "Inserted reg-free zero idiom" 2
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler-times "vpxor\[ \\t\]+\[^\n\]*%xmm" 2 } } */
+
+#include <stdlib.h>
+
+#define N 256
+
+static void __attribute__((noipa))
+work (float *__restrict a, float *__restrict b, float *__restrict c,
+ float *__restrict s, int n)
+{
+ for (int i = 0; i + 16 <= n; i += 15)
+ {
+ for (int k = 0; k < 8; k++)
+ a[i + k] = b[i + k] * c[i + k];
+
+ volatile float *p = a + i;
+ int t = (int) p[0];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s[i] = (float) t;
+
+ for (int k = 0; k < 8; k++)
+ a[i + 8 + k] = b[i + 8 + k] + c[i + k];
+ }
+}
+
+int
+main (void)
+{
+ static float a[N], b[N], c[N], s[N];
+ static float a_ref[N], s_ref[N];
+
+ for (int i = 0; i < N; i++)
+ {
+ b[i] = (float) (i + 1);
+ c[i] = (float) ((i * 3) + 2);
+ }
+
+ for (int i = 0; i + 16 <= N; i += 15)
+ {
+ for (int k = 0; k < 8; k++)
+ a_ref[i + k] = b[i + k] * c[i + k];
+
+ int t = (int) a_ref[i];
+ t = (t >> 1) ^ (t * 5);
+ t = (t >> 2) ^ (t * 9);
+ t = (t >> 1) ^ (t * 3);
+ t = (t >> 3) ^ (t * 7);
+ t = (t >> 2) ^ (t * 11);
+ t = (t >> 1) ^ (t * 13);
+ s_ref[i] = (float) t;
+
+ for (int k = 0; k < 8; k++)
+ a_ref[i + 8 + k] = b[i + 8 + k] + c[i + k];
+ }
+
+ work (a, b, c, s, N);
+
+ for (int i = 0; i < N; i++)
+ if (a[i] != a_ref[i] || s[i] != s_ref[i])
+ abort ();
+
+ return 0;
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
new file mode 100644
index 00000000000..529a8bcbc3b
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-7.c
@@ -0,0 +1,29 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -mavx512vl -ffixed-xmm0 -ffixed-xmm1 -ffixed-xmm2
-ffixed-xmm3 -ffixed-xmm4 -ffixed-xmm5 -ffixed-xmm6 -ffixed-xmm7 -ffixed-xmm8
-ffixed-xmm9 -ffixed-xmm10 -ffixed-xmm11 -ffixed-xmm12 -ffixed-xmm13
-ffixed-xmm14 -ffixed-xmm15 -mtune-ctrl=insert_reg_free_zero_idiom
--param=reg-free-zero-loop-threshold=1 --param=reg-free-zero-gap-threshold=1
-fdump-rtl-insert_reg_free_zero_idiom" } */
+/* { dg-final { scan-rtl-dump "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler "vpxord\[
\\t\]+%xmm(1\[6-9\]|2\[0-9\]|3\[01\])" } } */
+
+typedef long long v8di __attribute__((vector_size (64)));
+
+void
+test (long long *__restrict a, long long *__restrict b,
+ long long *__restrict c, long long *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8di vb = *(v8di *) (b + i);
+ v8di vc = *(v8di *) (c + i);
+ *(v8di *) (a + i) = vb * vc;
+
+ v8di g = *(v8di *) (a + i);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 3) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ *(v8di *) (s + i) = g;
+
+ v8di vb2 = *(v8di *) (b + i + 8);
+ *(v8di *) (a + i + 8) = vb2 + vc;
+ }
+}
diff --git a/gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
b/gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
new file mode 100644
index 00000000000..f568f2b4793
--- /dev/null
+++ b/gcc/testsuite/gcc.target/i386/reg-free-zero-8.c
@@ -0,0 +1,29 @@
+/* { dg-do compile { target { ! ia32 } } } */
+/* { dg-options "-O3 -mavx512f -mno-avx512vl -ffixed-xmm0 -ffixed-xmm1
-ffixed-xmm2 -ffixed-xmm3 -ffixed-xmm4 -ffixed-xmm5 -ffixed-xmm6 -ffixed-xmm7
-ffixed-xmm8 -ffixed-xmm9 -ffixed-xmm10 -ffixed-xmm11 -ffixed-xmm12
-ffixed-xmm13 -ffixed-xmm14 -ffixed-xmm15
-mtune-ctrl=insert_reg_free_zero_idiom --param=reg-free-zero-loop-threshold=1
--param=reg-free-zero-gap-threshold=1 -fdump-rtl-insert_reg_free_zero_idiom" }
*/
+/* { dg-final { scan-rtl-dump "Inserted reg-free zero idiom"
"insert_reg_free_zero_idiom" } } */
+/* { dg-final { scan-assembler "vpxord\[
\\t\]+%zmm(1\[6-9\]|2\[0-9\]|3\[01\])" } } */
+
+typedef long long v8di __attribute__((vector_size (64)));
+
+void
+test (long long *__restrict a, long long *__restrict b,
+ long long *__restrict c, long long *__restrict s, int n)
+{
+ for (int i = 0; i < n; i += 15)
+ {
+ v8di vb = *(v8di *) (b + i);
+ v8di vc = *(v8di *) (c + i);
+ *(v8di *) (a + i) = vb * vc;
+
+ v8di g = *(v8di *) (a + i);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ g = (g >> 1) ^ (g + g);
+ g = (g >> 3) ^ (g + g);
+ g = (g >> 2) ^ (g + g);
+ *(v8di *) (s + i) = g;
+
+ v8di vb2 = *(v8di *) (b + i + 8);
+ *(v8di *) (a + i + 8) = vb2 + vc;
+ }
+}
--
2.34.1
