To revectorize a target builtin we need to ask the target whether it has an
implementation for the requested argument types, result type and mask. We
cannot make that query by changing the public builtin declaration.
For example:
#include <arm_neon.h>
uint8x16_t aes_step (uint8x16_t a, uint8x16_t b)
{
return vaeseq_u8 (a, b);
}
We use IFN_MASK_CALL node to carry the proposed types while the
function address identifies the original operation:
requested_result = .MASK_CALL (&vaeseq_u8,
requested_a, requested_b, mask);
The middle operands and result have the requested types. The final operand
specifies which elements need valid results. We then extend gimple_fold_call
to recognize a BUILT_IN_MD function address and dispatches the request through
targetm.gimple_fold_builtin.
The target replaces a supported request with its implementation. If the result
is still MASK_CALL, the caller knows the query failed. These requests are
temporary analysis input and must be resolved for the vectorizer to accept it.
Non-masked calls are simpler. We just make a dummy expression with the arguments
and results carrying the correct types and call gimple_fold_call again so the
target can validate. If the fold doesn't return NULL we proceed with the
implementation.
Note that there may be a discrepancy in types here. Normally the target
builtins
use target intrinsics types which can differ from the compiler's internal
anonymous types. The vectorizer will not try to compensate here and its is
the target's responsibility.
We document this contract in the hook and have targets reject internal calls
until their folders support it. This avoids reading an ordinary callee
declaration from an internal call. The existing MASK_CALL use for if-converted
SIMD-clone calls is unchanged.
This patch adds dispatch only, so aes_step's codegen is unchanged. The next
patch adds the AArch64 resolver.
Bootstrapped Regtested on aarch64-none-linux-gnu,
arm-none-linux-gnueabihf, x86_64-pc-linux-gnu
-m32, -m64 and no issues.
Any feedback?
Thanks,
Tamar
gcc/ChangeLog:
PR tree-optimization/119187
* config/arm/arm.cc (arm_gimple_fold_builtin): Reject internal calls.
* config/i386/i386.cc (ix86_gimple_fold_builtin): Likewise.
* config/riscv/riscv-builtins.cc (riscv_gimple_fold_builtin): Likewise.
* config/rs6000/rs6000-builtin.cc (rs6000_gimple_fold_builtin):
Likewise.
* config/s390/s390.cc (s390_gimple_fold_builtin): Likewise.
* doc/tm.texi: Document the masked target builtin folding protocol.
* gimple-fold.cc (gimple_fold_call): Dispatch masked target builtin
folding requests.
* internal-fn.cc (expand_MASK_CALL): Document private builtin-folding
requests.
* internal-fn.def (MASK_CALL): Document the target builtin folding
protocol.
* target.def: Document the masked target builtin folding protocol.
* config/aarch64/aarch64.cc (aarch64_gimple_fold_builtin): Reject
internal calls.
---
--
diff --git a/gcc/config/aarch64/aarch64.cc b/gcc/config/aarch64/aarch64.cc
index
263c0afd4abcdbb09006fce8f97d60e67788f841..566d65903b915b70db85ae2e5ae9d9466d4f0cdb
100644
--- a/gcc/config/aarch64/aarch64.cc
+++ b/gcc/config/aarch64/aarch64.cc
@@ -17159,6 +17159,8 @@ static bool
aarch64_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
+ if (gimple_call_internal_p (stmt))
+ return false;
tree fndecl = gimple_call_fndecl (stmt);
unsigned int code = DECL_MD_FUNCTION_CODE (fndecl);
unsigned int subcode = code >> AARCH64_BUILTIN_SHIFT;
diff --git a/gcc/config/arm/arm.cc b/gcc/config/arm/arm.cc
index
2a8254c366ed08609843c7023af2dd08d9ad78b9..bbd68951ee79a1debcbc76bd52bdbd4fef4253da
100644
--- a/gcc/config/arm/arm.cc
+++ b/gcc/config/arm/arm.cc
@@ -2839,6 +2839,9 @@ static bool
arm_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
+ if (gimple_call_internal_p (stmt))
+ return false;
+
tree fndecl = gimple_call_fndecl (stmt);
unsigned int code = DECL_MD_FUNCTION_CODE (fndecl);
unsigned int subcode = code >> ARM_BUILTIN_SHIFT;
diff --git a/gcc/config/i386/i386.cc b/gcc/config/i386/i386.cc
index
ad34cfeda02c3a23e497cc70c6390db3d6dcb78d..ac114565484142bd9423462252957fbafef099e9
100644
--- a/gcc/config/i386/i386.cc
+++ b/gcc/config/i386/i386.cc
@@ -19861,6 +19861,9 @@ ix86_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
gimple *stmt = gsi_stmt (*gsi), *g;
gimple_seq stmts = NULL;
+ if (gimple_call_internal_p (stmt))
+ return false;
+
tree fndecl = gimple_call_fndecl (stmt);
gcc_checking_assert (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_MD));
int n_args = gimple_call_num_args (stmt);
diff --git a/gcc/config/riscv/riscv-builtins.cc
b/gcc/config/riscv/riscv-builtins.cc
index
2084cc24232367ddf1b55796d6d8b2edf38952bd..06bfcdaaae9c34234654a020ac7bf5c172b319ae
100644
--- a/gcc/config/riscv/riscv-builtins.cc
+++ b/gcc/config/riscv/riscv-builtins.cc
@@ -385,6 +385,9 @@ bool
riscv_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
gcall *stmt = as_a<gcall *> (gsi_stmt (*gsi));
+ if (gimple_call_internal_p (stmt))
+ return false;
+
tree fndecl = gimple_call_fndecl (stmt);
unsigned int code = DECL_MD_FUNCTION_CODE (fndecl);
unsigned int subcode = code >> RISCV_BUILTIN_SHIFT;
diff --git a/gcc/config/rs6000/rs6000-builtin.cc
b/gcc/config/rs6000/rs6000-builtin.cc
index
33c82b949528097c3d030361ece8379240ad72c9..fde49c9b216d077691e613e4482e1dcf097f3fee
100644
--- a/gcc/config/rs6000/rs6000-builtin.cc
+++ b/gcc/config/rs6000/rs6000-builtin.cc
@@ -1421,6 +1421,9 @@ bool
rs6000_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
gimple *stmt = gsi_stmt (*gsi);
+ if (gimple_call_internal_p (stmt))
+ return false;
+
tree fndecl = gimple_call_fndecl (stmt);
gcc_checking_assert (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD);
enum rs6000_gen_builtins fn_code
diff --git a/gcc/config/s390/s390.cc b/gcc/config/s390/s390.cc
index
5b615aaef00edea8f973d988957856096fc3f884..519f10f00bfde1de6d1b2984fe5c1bb98b1d2679
100644
--- a/gcc/config/s390/s390.cc
+++ b/gcc/config/s390/s390.cc
@@ -847,6 +847,9 @@ static bool
s390_gimple_fold_builtin (gimple_stmt_iterator *gsi)
{
gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
+ if (gimple_call_internal_p (stmt))
+ return false;
+
tree fndecl = gimple_call_fndecl (stmt);
unsigned int fcode = DECL_MD_FUNCTION_CODE (fndecl);
gimple *new_stmt;
diff --git a/gcc/doc/tm.texi b/gcc/doc/tm.texi
index
65cf4c640e7b4f1e5baf5d1e28065c9567f48121..e82375956c0916002665023f09882ce17ddc2cf5
100644
--- a/gcc/doc/tm.texi
+++ b/gcc/doc/tm.texi
@@ -12491,7 +12491,12 @@ containing a simplified expression for the call's
result. If
@deftypefn {Target Hook} bool TARGET_GIMPLE_FOLD_BUILTIN (gimple_stmt_iterator
*@var{gsi})
Fold a call to a machine specific built-in function that was set up
by @samp{TARGET_INIT_BUILTINS}. @var{gsi} points to the gimple
-statement holding the function call. Returns true if any change
+statement holding the function call. It can also point to an
+@code{IFN_MASK_CALL} whose first argument is the address of such a
+built-in function. The vectorizer uses this form to request an
+implementation with the given operand and result types and execution
+mask. Leave the call unchanged if no such implementation is available.
+Returns true if any change
was made to the GIMPLE stream.
@end deftypefn
diff --git a/gcc/gimple-fold.cc b/gcc/gimple-fold.cc
index
9bc88da352b053c0696c57d1de111afd016212c0..bc142c5be6249cd411d7448fad00f3497b4096c6
100644
--- a/gcc/gimple-fold.cc
+++ b/gcc/gimple-fold.cc
@@ -6287,6 +6287,15 @@ gimple_fold_call (gimple_stmt_iterator *gsi, bool
inplace)
tree overflow = NULL_TREE;
switch (gimple_call_internal_fn (stmt))
{
+ case IFN_MASK_CALL:
+ /* Target builtin folding also resolves vector calls with new
+ operand types. Keep an unsuccessful request as a MASK_CALL,
+ so that the caller can distinguish it from a supported call. */
+ if (tree fn = gimple_call_addr_fndecl (gimple_call_arg (stmt, 0)))
+ if (fndecl_built_in_p (fn, BUILT_IN_MD))
+ changed |= targetm.gimple_fold_builtin (gsi);
+
+ break;
case IFN_ASSUME:
/* Remove .ASSUME calls during the last fold since it is no
longer needed. */
diff --git a/gcc/internal-fn.cc b/gcc/internal-fn.cc
index
d277a07eaf7ffba6bb907f7874b651833e57dc1f..fe7fcd2d942dfac77e40d7e0d9b3bafc180ef7ca
100644
--- a/gcc/internal-fn.cc
+++ b/gcc/internal-fn.cc
@@ -5824,7 +5824,8 @@ expand_ASSUME (internal_fn, gcall *)
void
expand_MASK_CALL (internal_fn, gcall *)
{
- /* This IFN should only exist between ifcvt and vect passes. */
+ /* Live calls should only exist between ifcvt and vect passes. The
+ vectorizer's private builtin-folding requests must also be resolved. */
gcc_unreachable ();
}
diff --git a/gcc/internal-fn.def b/gcc/internal-fn.def
index
7626836ba6484c0f05ef9af8b3916744e0f48758..18c00722e6d886f0fe82ecb80e82caffee5606db
100644
--- a/gcc/internal-fn.def
+++ b/gcc/internal-fn.def
@@ -645,7 +645,11 @@ DEF_INTERNAL_FN (SPACESHIP, ECF_CONST | ECF_LEAF |
ECF_NOTHROW, NULL)
DEF_INTERNAL_FN (ASSUME, ECF_PURE | ECF_LEAF | ECF_NOTHROW
| ECF_LOOPING_CONST_OR_PURE, NULL)
-/* For if-conversion of inbranch SIMD clones. */
+/* For if-conversion of inbranch SIMD clones, and for folding target vector
+ builtins. The first argument is the original function address and the
+ last is the execution mask. Target folding can resolve new vector types
+ for the intervening arguments and the result. Such requests must be
+ folded successfully before being inserted into the function. */
DEF_INTERNAL_FN (MASK_CALL, ECF_NOVOPS, NULL)
/* _BitInt support. */
diff --git a/gcc/target.def b/gcc/target.def
index
877bd166b239a555be7d7c2094be790c3de35bde..5ada852c9b822ac74a983fe5cb5b350b8dd3dc6f
100644
--- a/gcc/target.def
+++ b/gcc/target.def
@@ -2603,7 +2603,12 @@ DEFHOOK
(gimple_fold_builtin,
"Fold a call to a machine specific built-in function that was set up\n\
by @samp{TARGET_INIT_BUILTINS}. @var{gsi} points to the gimple\n\
-statement holding the function call. Returns true if any change\n\
+statement holding the function call. It can also point to an\n\
+@code{IFN_MASK_CALL} whose first argument is the address of such a\n\
+built-in function. The vectorizer uses this form to request an\n\
+implementation with the given operand and result types and execution\n\
+mask. Leave the call unchanged if no such implementation is available.\n\
+Returns true if any change\n\
was made to the GIMPLE stream.",
bool, (gimple_stmt_iterator *gsi),
hook_bool_gsiptr_false)
