Fold integer VEC_UNPACK_LO_EXPR and VEC_UNPACK_HI_EXPR for suitable
variable-length VECTOR_CSTs.
The fixed-length case already folds. This extends the same idea to VLA
constants when the input has twice as many elements as the output and the
selected half can be represented as a valid VECTOR_CST. This helps fold
known predicate constants before RTL expansion, avoiding unnecessary
predicate unpack instructions.
The selftests cover the foldable low/high cases and cases that should not
fold.
This show up often with known iteration loops and folding these in gimple
allows us to fold away unrolled masks which end up being empty.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
OK for master?
Thanks,
Tamar
gcc/ChangeLog:
* fold-const.cc (const_unop): Fold VEC_UNPACK_LO_EXPR and
VEC_UNPACK_HI_EXPR for suitable variable-length VECTOR_CSTs.
(selftest::test_vec_unpack_folding): New function.
(selftest::fold_const_cc_tests): Call it.
---
diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
index
4c4b535d44041ba9fd035ac3841d6862ab5700d1..289c5d5580b4793001ddce55a4a0a0f498db9e53
100644
--- a/gcc/fold-const.cc
+++ b/gcc/fold-const.cc
@@ -1956,7 +1956,53 @@ const_unop (enum tree_code code, tree type, tree arg0)
return NULL_TREE;
if (!VECTOR_CST_NELTS (arg0).is_constant (&in_nelts))
- return NULL_TREE;
+ {
+ if (code != VEC_UNPACK_LO_EXPR && code != VEC_UNPACK_HI_EXPR)
+ return NULL_TREE;
+ if (!known_eq (VECTOR_CST_NELTS (arg0),
+ TYPE_VECTOR_SUBPARTS (type) * 2)
+ || VECTOR_CST_STEPPED_P (arg0))
+ return NULL_TREE;
+
+ unsigned int min_out_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (type));
+ tree fill = fold_convert_const (NOP_EXPR, TREE_TYPE (type),
+ vector_cst_elt (arg0,
+ min_out_nelts));
+ if (fill == NULL_TREE || !CONSTANT_CLASS_P (fill))
+ return NULL_TREE;
+
+ /* Check that the selected high half is a single repeated value.
+ Since the input is not stepped, checking one full pattern is
+ enough to prove the scalable tail. */
+ for (i = 1; i < VECTOR_CST_NPATTERNS (arg0); ++i)
+ {
+ tree elt = fold_convert_const (NOP_EXPR, TREE_TYPE (type),
+ vector_cst_elt
+ (arg0,
+ min_out_nelts + i));
+ if (elt == NULL_TREE
+ || !CONSTANT_CLASS_P (elt)
+ || !operand_equal_p (elt, fill, 0))
+ return NULL_TREE;
+ }
+
+ if ((!BYTES_BIG_ENDIAN) ^ (code == VEC_UNPACK_LO_EXPR))
+ return build_vector_from_val (type, fill);
+
+ tree_vector_builder elts (type, min_out_nelts, 2);
+ for (i = 0; i < min_out_nelts; ++i)
+ {
+ tree elt = fold_convert_const (NOP_EXPR, TREE_TYPE (type),
+ vector_cst_elt (arg0, i));
+ if (elt == NULL_TREE || !CONSTANT_CLASS_P (elt))
+ return NULL_TREE;
+ elts.quick_push (elt);
+ }
+ for (i = 0; i < min_out_nelts; ++i)
+ elts.quick_push (fill);
+ return elts.build ();
+ }
out_nelts = in_nelts / 2;
gcc_assert (known_eq (out_nelts, TYPE_VECTOR_SUBPARTS (type)));
@@ -17753,6 +17799,96 @@ test_vec_duplicate_folding ()
ASSERT_TRUE (operand_equal_p (dup5_expr, dup5_cst, 0));
}
+/* Verify folding of VEC_UNPACK_{LO,HI}_EXPRs. */
+
+static void
+test_vec_unpack_folding ()
+{
+ machine_mode vmode;
+ FOR_EACH_MODE_IN_CLASS (vmode, MODE_VECTOR_INT)
+ {
+ poly_uint64 in_nelts = GET_MODE_NUNITS (vmode);
+ if (!test_fold_vec_perm_cst::is_simple_vla_size (in_nelts)
+ || in_nelts.coeffs[0] < 2
+ || !targetm.vector_mode_supported_p (vmode))
+ continue;
+
+ unsigned int in_bits = GET_MODE_UNIT_BITSIZE (vmode);
+ if (in_bits >= HOST_BITS_PER_WIDE_INT)
+ continue;
+
+ unsigned int min_out_nelts = in_nelts.coeffs[0] / 2;
+ tree in_inner_type = lang_hooks.types.type_for_mode
+ (GET_MODE_INNER (vmode), 1);
+ tree out_inner_type = build_nonstandard_integer_type (in_bits * 2, 1);
+ tree in_type = build_vector_type_for_mode (in_inner_type, vmode);
+ tree out_type = build_vector_type (out_inner_type,
+ exact_div (in_nelts, 2));
+
+ tree_vector_builder builder (in_type, in_nelts.coeffs[0], 2);
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ builder.quick_push (build_int_cst (in_inner_type, i + 1));
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ builder.quick_push (build_int_cst (in_inner_type,
+ i < min_out_nelts ? 100 + i : 77));
+ tree arg = builder.build ();
+
+ tree lo = const_unop (VEC_UNPACK_LO_EXPR, out_type, arg);
+ ASSERT_TRUE (lo != NULL_TREE);
+ tree hi = const_unop (VEC_UNPACK_HI_EXPR, out_type, arg);
+ ASSERT_TRUE (hi != NULL_TREE);
+
+ tree prefix = (!BYTES_BIG_ENDIAN ? lo : hi);
+ tree dup = (!BYTES_BIG_ENDIAN ? hi : lo);
+
+ ASSERT_EQ (VECTOR_CST, TREE_CODE (prefix));
+ ASSERT_EQ (VECTOR_CST, TREE_CODE (dup));
+ ASSERT_KNOWN_EQ (TYPE_VECTOR_SUBPARTS (out_type),
+ VECTOR_CST_NELTS (prefix));
+ ASSERT_KNOWN_EQ (TYPE_VECTOR_SUBPARTS (out_type),
+ VECTOR_CST_NELTS (dup));
+ ASSERT_EQ (min_out_nelts, VECTOR_CST_NPATTERNS (prefix));
+ ASSERT_EQ (2, VECTOR_CST_NELTS_PER_PATTERN (prefix));
+ ASSERT_EQ (1, VECTOR_CST_NPATTERNS (dup));
+ ASSERT_EQ (1, VECTOR_CST_NELTS_PER_PATTERN (dup));
+
+ for (unsigned int i = 0; i < min_out_nelts; ++i)
+ {
+ tree elt = build_int_cst (out_inner_type, i + 1);
+ ASSERT_TRUE (operand_equal_p (VECTOR_CST_ELT (prefix, i),
+ elt, 0));
+ tree fill = build_int_cst (out_inner_type, 77);
+ ASSERT_TRUE (operand_equal_p (VECTOR_CST_ELT
+ (prefix, min_out_nelts + i),
+ fill, 0));
+ ASSERT_TRUE (operand_equal_p (VECTOR_CST_ELT (dup, i),
+ fill, 0));
+ }
+
+ tree_vector_builder varied (in_type, in_nelts.coeffs[0], 2);
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ varied.quick_push (build_int_cst (in_inner_type, i + 1));
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ varied.quick_push (build_int_cst (in_inner_type, 100 + i));
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_LO_EXPR, out_type,
+ varied.build ()));
+
+ tree_vector_builder stepped (in_type, 1, 3);
+ stepped.quick_push (build_int_cst (in_inner_type, 1));
+ stepped.quick_push (build_int_cst (in_inner_type, 2));
+ stepped.quick_push (build_int_cst (in_inner_type, 3));
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_LO_EXPR, out_type,
+ stepped.build ()));
+
+ tree wrong_out_type = build_vector_type (out_inner_type, in_nelts);
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_LO_EXPR, wrong_out_type,
+ arg));
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_FLOAT_LO_EXPR, out_type,
+ arg));
+ return;
+ }
+}
+
/* Run all of the selftests within this file. */
void
@@ -17761,6 +17897,7 @@ fold_const_cc_tests ()
test_arithmetic_folding ();
test_vector_folding ();
test_vec_duplicate_folding ();
+ test_vec_unpack_folding ();
test_fold_vec_perm_cst::test ();
test_operand_equality::test ();
}
--
diff --git a/gcc/fold-const.cc b/gcc/fold-const.cc
index 4c4b535d44041ba9fd035ac3841d6862ab5700d1..289c5d5580b4793001ddce55a4a0a0f498db9e53 100644
--- a/gcc/fold-const.cc
+++ b/gcc/fold-const.cc
@@ -1956,7 +1956,53 @@ const_unop (enum tree_code code, tree type, tree arg0)
return NULL_TREE;
if (!VECTOR_CST_NELTS (arg0).is_constant (&in_nelts))
- return NULL_TREE;
+ {
+ if (code != VEC_UNPACK_LO_EXPR && code != VEC_UNPACK_HI_EXPR)
+ return NULL_TREE;
+ if (!known_eq (VECTOR_CST_NELTS (arg0),
+ TYPE_VECTOR_SUBPARTS (type) * 2)
+ || VECTOR_CST_STEPPED_P (arg0))
+ return NULL_TREE;
+
+ unsigned int min_out_nelts
+ = constant_lower_bound (TYPE_VECTOR_SUBPARTS (type));
+ tree fill = fold_convert_const (NOP_EXPR, TREE_TYPE (type),
+ vector_cst_elt (arg0,
+ min_out_nelts));
+ if (fill == NULL_TREE || !CONSTANT_CLASS_P (fill))
+ return NULL_TREE;
+
+ /* Check that the selected high half is a single repeated value.
+ Since the input is not stepped, checking one full pattern is
+ enough to prove the scalable tail. */
+ for (i = 1; i < VECTOR_CST_NPATTERNS (arg0); ++i)
+ {
+ tree elt = fold_convert_const (NOP_EXPR, TREE_TYPE (type),
+ vector_cst_elt
+ (arg0,
+ min_out_nelts + i));
+ if (elt == NULL_TREE
+ || !CONSTANT_CLASS_P (elt)
+ || !operand_equal_p (elt, fill, 0))
+ return NULL_TREE;
+ }
+
+ if ((!BYTES_BIG_ENDIAN) ^ (code == VEC_UNPACK_LO_EXPR))
+ return build_vector_from_val (type, fill);
+
+ tree_vector_builder elts (type, min_out_nelts, 2);
+ for (i = 0; i < min_out_nelts; ++i)
+ {
+ tree elt = fold_convert_const (NOP_EXPR, TREE_TYPE (type),
+ vector_cst_elt (arg0, i));
+ if (elt == NULL_TREE || !CONSTANT_CLASS_P (elt))
+ return NULL_TREE;
+ elts.quick_push (elt);
+ }
+ for (i = 0; i < min_out_nelts; ++i)
+ elts.quick_push (fill);
+ return elts.build ();
+ }
out_nelts = in_nelts / 2;
gcc_assert (known_eq (out_nelts, TYPE_VECTOR_SUBPARTS (type)));
@@ -17753,6 +17799,96 @@ test_vec_duplicate_folding ()
ASSERT_TRUE (operand_equal_p (dup5_expr, dup5_cst, 0));
}
+/* Verify folding of VEC_UNPACK_{LO,HI}_EXPRs. */
+
+static void
+test_vec_unpack_folding ()
+{
+ machine_mode vmode;
+ FOR_EACH_MODE_IN_CLASS (vmode, MODE_VECTOR_INT)
+ {
+ poly_uint64 in_nelts = GET_MODE_NUNITS (vmode);
+ if (!test_fold_vec_perm_cst::is_simple_vla_size (in_nelts)
+ || in_nelts.coeffs[0] < 2
+ || !targetm.vector_mode_supported_p (vmode))
+ continue;
+
+ unsigned int in_bits = GET_MODE_UNIT_BITSIZE (vmode);
+ if (in_bits >= HOST_BITS_PER_WIDE_INT)
+ continue;
+
+ unsigned int min_out_nelts = in_nelts.coeffs[0] / 2;
+ tree in_inner_type = lang_hooks.types.type_for_mode
+ (GET_MODE_INNER (vmode), 1);
+ tree out_inner_type = build_nonstandard_integer_type (in_bits * 2, 1);
+ tree in_type = build_vector_type_for_mode (in_inner_type, vmode);
+ tree out_type = build_vector_type (out_inner_type,
+ exact_div (in_nelts, 2));
+
+ tree_vector_builder builder (in_type, in_nelts.coeffs[0], 2);
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ builder.quick_push (build_int_cst (in_inner_type, i + 1));
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ builder.quick_push (build_int_cst (in_inner_type,
+ i < min_out_nelts ? 100 + i : 77));
+ tree arg = builder.build ();
+
+ tree lo = const_unop (VEC_UNPACK_LO_EXPR, out_type, arg);
+ ASSERT_TRUE (lo != NULL_TREE);
+ tree hi = const_unop (VEC_UNPACK_HI_EXPR, out_type, arg);
+ ASSERT_TRUE (hi != NULL_TREE);
+
+ tree prefix = (!BYTES_BIG_ENDIAN ? lo : hi);
+ tree dup = (!BYTES_BIG_ENDIAN ? hi : lo);
+
+ ASSERT_EQ (VECTOR_CST, TREE_CODE (prefix));
+ ASSERT_EQ (VECTOR_CST, TREE_CODE (dup));
+ ASSERT_KNOWN_EQ (TYPE_VECTOR_SUBPARTS (out_type),
+ VECTOR_CST_NELTS (prefix));
+ ASSERT_KNOWN_EQ (TYPE_VECTOR_SUBPARTS (out_type),
+ VECTOR_CST_NELTS (dup));
+ ASSERT_EQ (min_out_nelts, VECTOR_CST_NPATTERNS (prefix));
+ ASSERT_EQ (2, VECTOR_CST_NELTS_PER_PATTERN (prefix));
+ ASSERT_EQ (1, VECTOR_CST_NPATTERNS (dup));
+ ASSERT_EQ (1, VECTOR_CST_NELTS_PER_PATTERN (dup));
+
+ for (unsigned int i = 0; i < min_out_nelts; ++i)
+ {
+ tree elt = build_int_cst (out_inner_type, i + 1);
+ ASSERT_TRUE (operand_equal_p (VECTOR_CST_ELT (prefix, i),
+ elt, 0));
+ tree fill = build_int_cst (out_inner_type, 77);
+ ASSERT_TRUE (operand_equal_p (VECTOR_CST_ELT
+ (prefix, min_out_nelts + i),
+ fill, 0));
+ ASSERT_TRUE (operand_equal_p (VECTOR_CST_ELT (dup, i),
+ fill, 0));
+ }
+
+ tree_vector_builder varied (in_type, in_nelts.coeffs[0], 2);
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ varied.quick_push (build_int_cst (in_inner_type, i + 1));
+ for (unsigned int i = 0; i < in_nelts.coeffs[0]; ++i)
+ varied.quick_push (build_int_cst (in_inner_type, 100 + i));
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_LO_EXPR, out_type,
+ varied.build ()));
+
+ tree_vector_builder stepped (in_type, 1, 3);
+ stepped.quick_push (build_int_cst (in_inner_type, 1));
+ stepped.quick_push (build_int_cst (in_inner_type, 2));
+ stepped.quick_push (build_int_cst (in_inner_type, 3));
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_LO_EXPR, out_type,
+ stepped.build ()));
+
+ tree wrong_out_type = build_vector_type (out_inner_type, in_nelts);
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_LO_EXPR, wrong_out_type,
+ arg));
+ ASSERT_EQ (NULL_TREE, const_unop (VEC_UNPACK_FLOAT_LO_EXPR, out_type,
+ arg));
+ return;
+ }
+}
+
/* Run all of the selftests within this file. */
void
@@ -17761,6 +17897,7 @@ fold_const_cc_tests ()
test_arithmetic_folding ();
test_vector_folding ();
test_vec_duplicate_folding ();
+ test_vec_unpack_folding ();
test_fold_vec_perm_cst::test ();
test_operand_equality::test ();
}
