https://github.com/huntergr-arm updated
https://github.com/llvm/llvm-project/pull/73941
>From a3f46f46483b2d83a5b38c197caebf7f68af8d56 Mon Sep 17 00:00:00 2001
From: Graham Hunter <graham.hun...@arm.com>
Date: Wed, 11 Oct 2023 17:06:09 +0100
Subject: [PATCH] [LV] Add support for linear arguments for vector function
variants
If we have vectorized variants of a function which take linear
parameters, we should be able to vectorize assuming the strides
match.
---
.../Transforms/Vectorize/LoopVectorize.cpp | 24 ++++++++++
.../AArch64/vector-call-linear-args.ll | 44 ++++++++++++-------
2 files changed, 53 insertions(+), 15 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 09a6e01226ab68c..4b6eac56597c232 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -7035,6 +7035,30 @@ void
LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) {
ParamsOk = false;
break;
}
+ case VFParamKind::OMP_Linear: {
+ Value *ScalarParam = CI->getArgOperand(Param.ParamPos);
+ // Find the stride for the scalar parameter in this loop and see if
+ // it matches the stride for the variant.
+ // TODO: do we need to figure out the cost of an extract to get the
+ // first lane? Or do we hope that it will be folded away?
+ ScalarEvolution *SE = PSE.getSE();
+ const auto *SAR =
+ dyn_cast<SCEVAddRecExpr>(SE->getSCEV(ScalarParam));
+
+ if (!SAR || SAR->getLoop() != TheLoop) {
+ ParamsOk = false;
+ break;
+ }
+
+ const SCEVConstant *Step =
+ dyn_cast<SCEVConstant>(SAR->getStepRecurrence(*SE));
+
+ if (!Step ||
+ Step->getAPInt().getSExtValue() != Param.LinearStepOrPos)
+ ParamsOk = false;
+
+ break;
+ }
case VFParamKind::GlobalPredicate:
UsesMask = true;
break;
diff --git
a/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll
b/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll
index ba9d57e1e4a16fd..ee7f243d5b3734c 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll
@@ -1,4 +1,4 @@
-; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
UTC_ARGS: --filter "call" --version 2
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
UTC_ARGS: --filter "call.*(foo|bar|baz|quux)" --version 2
; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S |
FileCheck %s --check-prefixes=NEON
; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1
-S | FileCheck %s --check-prefixes=SVE_OR_NEON
; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1
-S -prefer-predicate-over-epilogue=predicate-dont-vectorize | FileCheck %s
--check-prefixes=SVE_TF
@@ -10,15 +10,18 @@ target triple = "aarch64-unknown-linux-gnu"
define void @test_linear(ptr noalias %a, ptr readnone %b, i64 %n) {
; NEON-LABEL: define void @test_linear
; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) {
+; NEON: [[TMP3:%.*]] = call <2 x i64> @neon_foo_linear(ptr [[TMP2:%.*]])
; NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR0:[0-9]+]]
;
; SVE_OR_NEON-LABEL: define void @test_linear
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64
[[N:%.*]]) #[[ATTR0:[0-9]+]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]])
#[[ATTR1:[0-9]+]]
+; SVE_OR_NEON: [[TMP13:%.*]] = call <vscale x 2 x i64>
@sve_foo_linear_nomask(ptr [[TMP12:%.*]])
+; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]])
#[[ATTR2:[0-9]+]]
;
; SVE_TF-LABEL: define void @test_linear
; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]])
#[[ATTR0:[0-9]+]] {
-; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]]
+; SVE_TF: [[TMP19:%.*]] = call <vscale x 2 x i64> @sve_foo_linear(ptr
[[TMP18:%.*]], <vscale x 2 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]]
;
entry:
br label %for.body
@@ -40,15 +43,17 @@ for.cond.cleanup:
define void @test_linear_with_vector(ptr noalias %a, ptr readnone %b, ptr
readonly %c, i64 %n) {
; NEON-LABEL: define void @test_linear_with_vector
; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly
[[C:%.*]], i64 [[N:%.*]]) {
+; NEON: [[TMP5:%.*]] = call <4 x i32> @neon_baz_vector_and_linear(<4 x i32>
[[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]])
; NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]])
#[[ATTR1:[0-9]+]]
;
; SVE_OR_NEON-LABEL: define void @test_linear_with_vector
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr
readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr
[[GEPB:%.*]]) #[[ATTR2:[0-9]+]]
+; SVE_OR_NEON: [[TMP15:%.*]] = call <vscale x 4 x i32>
@sve_baz_vector_and_linear(<vscale x 4 x i32> [[WIDE_LOAD:%.*]], ptr
[[TMP14:%.*]])
+; SVE_OR_NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr
[[GEPB:%.*]]) #[[ATTR3:[0-9]+]]
;
; SVE_TF-LABEL: define void @test_linear_with_vector
; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly
[[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_TF: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]])
#[[ATTR2:[0-9]+]]
+; SVE_TF: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]])
#[[ATTR4:[0-9]+]]
;
entry:
br label %for.body
@@ -76,11 +81,11 @@ define void @test_linear_bad_stride(ptr noalias %a, ptr
readnone %b, i64 %n) {
;
; SVE_OR_NEON-LABEL: define void @test_linear_bad_stride
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64
[[N:%.*]]) #[[ATTR0]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]])
#[[ATTR3:[0-9]+]]
+; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]])
#[[ATTR4:[0-9]+]]
;
; SVE_TF-LABEL: define void @test_linear_bad_stride
; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]])
#[[ATTR0]] {
-; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]]
+; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR5:[0-9]+]]
;
entry:
br label %for.body
@@ -102,15 +107,17 @@ for.cond.cleanup:
define void @test_linear_wide_stride(ptr noalias %a, ptr readnone %b, i64 %n) {
; NEON-LABEL: define void @test_linear_wide_stride
; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) {
+; NEON: [[TMP4:%.*]] = call <2 x i64> @neon_foo_linear_2x(ptr [[TMP3:%.*]])
; NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR2]]
;
; SVE_OR_NEON-LABEL: define void @test_linear_wide_stride
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64
[[N:%.*]]) #[[ATTR0]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3]]
+; SVE_OR_NEON: [[TMP14:%.*]] = call <vscale x 2 x i64>
@sve_foo_linear_nomask_2x(ptr [[TMP13:%.*]])
+; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR4]]
;
; SVE_TF-LABEL: define void @test_linear_wide_stride
; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]])
#[[ATTR0]] {
-; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3]]
+; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR5]]
;
entry:
br label %for.body
@@ -133,15 +140,18 @@ for.cond.cleanup:
define void @test_linear_mixed_types(ptr noalias %a, ptr readnone %b, ptr
readonly %c, i64 %n) {
; NEON-LABEL: define void @test_linear_mixed_types
; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly
[[C:%.*]], i64 [[N:%.*]]) {
+; NEON: [[TMP5:%.*]] = call <4 x i32> @neon_quux_linear(ptr [[TMP3:%.*]],
ptr [[TMP4:%.*]])
; NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]])
#[[ATTR3:[0-9]+]]
;
; SVE_OR_NEON-LABEL: define void @test_linear_mixed_types
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr
readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr
[[GEPB:%.*]]) #[[ATTR4:[0-9]+]]
+; SVE_OR_NEON: [[TMP15:%.*]] = call <vscale x 4 x i32>
@sve_quux_linear_mask(ptr [[TMP13:%.*]], ptr [[TMP14:%.*]], <vscale x 4 x i1>
shufflevector (<vscale x 4 x i1> insertelement (<vscale x 4 x i1> poison, i1
true, i64 0), <vscale x 4 x i1> poison, <vscale x 4 x i32> zeroinitializer))
+; SVE_OR_NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr
[[GEPB:%.*]]) #[[ATTR5:[0-9]+]]
;
; SVE_TF-LABEL: define void @test_linear_mixed_types
; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly
[[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_TF: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]])
#[[ATTR4:[0-9]+]]
+; SVE_TF: [[TMP21:%.*]] = call <vscale x 4 x i32> @sve_quux_linear_mask(ptr
[[TMP19:%.*]], ptr [[TMP20:%.*]], <vscale x 4 x i1> [[ACTIVE_LANE_MASK:%.*]])
+; SVE_TF: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]])
#[[ATTR6:[0-9]+]]
;
entry:
br label %for.body
@@ -164,15 +174,17 @@ for.cond.cleanup:
define void @test_linear_non_ptr(ptr noalias %a, i64 %n) {
; NEON-LABEL: define void @test_linear_non_ptr
; NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
+; NEON: [[TMP3:%.*]] = call <4 x i32> @neon_bar_linear(i32 [[TMP2:%.*]])
; NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR4:[0-9]+]]
;
; SVE_OR_NEON-LABEL: define void @test_linear_non_ptr
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]])
#[[ATTR5:[0-9]+]]
+; SVE_OR_NEON: [[TMP13:%.*]] = call <vscale x 4 x i32> @sve_bar_linear(i32
[[TMP12:%.*]])
+; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]])
#[[ATTR6:[0-9]+]]
;
; SVE_TF-LABEL: define void @test_linear_non_ptr
; SVE_TF-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]])
#[[ATTR5:[0-9]+]]
+; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]])
#[[ATTR7:[0-9]+]]
;
entry:
br label %for.body
@@ -195,15 +207,17 @@ for.cond.cleanup:
define void @test_linear_non_ptr_neg_stride(ptr noalias %a, i64 %n) {
; NEON-LABEL: define void @test_linear_non_ptr_neg_stride
; NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) {
+; NEON: [[TMP3:%.*]] = call <4 x i32> @neon_bar_neg_linear(i32 [[TMP2:%.*]])
; NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]])
#[[ATTR5:[0-9]+]]
;
; SVE_OR_NEON-LABEL: define void @test_linear_non_ptr_neg_stride
; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]])
#[[ATTR6:[0-9]+]]
+; SVE_OR_NEON: [[TMP13:%.*]] = call <vscale x 4 x i32>
@sve_bar_neg_linear(i32 [[TMP12:%.*]])
+; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]])
#[[ATTR7:[0-9]+]]
;
; SVE_TF-LABEL: define void @test_linear_non_ptr_neg_stride
; SVE_TF-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
-; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]])
#[[ATTR6:[0-9]+]]
+; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]])
#[[ATTR8:[0-9]+]]
;
entry:
br label %for.body
_______________________________________________
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
