Hi rjmccall, rsmith, doug.gregor, ABataev, hfinkel, fraggamuffin,
Please review CodeGen of the 'linear' clause for the 'omp simd' directive.
The linear variable is privatized (similar to 'private') and its value on
current iteration is calculated, similar to the loop counter variables.
http://reviews.llvm.org/D8375
Files:
include/clang/AST/DataRecursiveASTVisitor.h
include/clang/AST/OpenMPClause.h
include/clang/AST/RecursiveASTVisitor.h
include/clang/AST/StmtOpenMP.h
lib/AST/Stmt.cpp
lib/AST/StmtProfile.cpp
lib/CodeGen/CGStmtOpenMP.cpp
lib/Sema/SemaOpenMP.cpp
lib/Serialization/ASTReaderStmt.cpp
lib/Serialization/ASTWriterStmt.cpp
test/OpenMP/simd_codegen.cpp
tools/libclang/CIndex.cpp
EMAIL PREFERENCES
http://reviews.llvm.org/settings/panel/emailpreferences/
Index: test/OpenMP/simd_codegen.cpp
===================================================================
--- test/OpenMP/simd_codegen.cpp
+++ test/OpenMP/simd_codegen.cpp
@@ -7,6 +7,9 @@
#ifndef HEADER
#define HEADER
+long long get_val() { return 0; }
+double *g_ptr;
+
// CHECK-LABEL: define {{.*void}} @{{.*}}simple{{.*}}(float* {{.+}}, float* {{.+}}, float* {{.+}}, float* {{.+}})
void simple(float *a, float *b, float *c, float *d) {
#pragma omp simd
@@ -33,7 +36,13 @@
}
// CHECK: [[SIMPLE_LOOP1_END]]
- #pragma omp simd
+ long long k = get_val();
+
+ #pragma omp simd linear(k : 3)
+// CHECK: [[K0:%.+]] = call {{.*}}i64 @{{.*}}get_val
+// CHECK-NEXT: store i64 [[K0]], i64* [[K_VAR:%[^,]+]]
+// CHECK: [[K0LOAD:%.+]] = load i64, i64* [[K_VAR]]
+// CHECK-NEXT: store i64 [[K0LOAD]], i64* [[LIN0:%[^,]+]]
// CHECK: store i32 0, i32* [[OMP_IV2:%[^,]+]]
// CHECK: [[IV2:%.+]] = load i32, i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID:[0-9]+]]
@@ -47,17 +56,45 @@
// CHECK-NEXT: [[IV2_1:%.+]] = mul nsw i32 [[IV2_0]], 1
// CHECK-NEXT: [[LC_I_1:%.+]] = sub nsw i32 10, [[IV2_1]]
// CHECK-NEXT: store i32 [[LC_I_1]], i32* {{.+}}, !llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]]
- a[i]++;
+//
+// CHECK-NEXT: [[LIN0_1:%.+]] = load i64, i64* [[LIN0]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]]
+// CHECK-NEXT: [[IV2_2:%.+]] = load i32, i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]]
+// CHECK-NEXT: [[LIN_MUL1:%.+]] = mul nsw i32 [[IV2_2]], 3
+// CHECK-NEXT: [[LIN_EXT1:%.+]] = sext i32 [[LIN_MUL1]] to i64
+// CHECK-NEXT: [[LIN_ADD1:%.+]] = add nsw i64 [[LIN0_1]], [[LIN_EXT1]]
+// Update of the privatized version of linear variable!
+// CHECK-NEXT: store i64 [[LIN_ADD1]], i64* [[K_PRIVATIZED:%[^,]+]]
+ a[k]++;
+ k = k + 3;
// CHECK: [[IV2_2:%.+]] = load i32, i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]]
// CHECK-NEXT: [[ADD2_2:%.+]] = add nsw i32 [[IV2_2]], 1
// CHECK-NEXT: store i32 [[ADD2_2]], i32* [[OMP_IV2]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP2_ID]]
// br label {{.+}}, !llvm.loop ![[SIMPLE_LOOP2_ID]]
}
// CHECK: [[SIMPLE_LOOP2_END]]
+//
+// Update linear vars after loop, as the loop was operating on a private version.
+// CHECK: [[LIN0_2:%.+]] = load i64, i64* [[LIN0]]
+// CHECK-NEXT: [[LIN_ADD2:%.+]] = add nsw i64 [[LIN0_2]], 27
+// CHECK-NEXT: store i64 [[LIN_ADD2]], i64* [[K_VAR]]
+//
+
+ int lin = 12;
+ #pragma omp simd linear(lin : get_val()), linear(g_ptr)
+
+// Init linear private var.
+// CHECK: store i32 12, i32* [[LIN_VAR:%[^,]+]]
+// CHECK: [[LIN_LOAD:%.+]] = load i32, i32* [[LIN_VAR]]
+// CHECK-NEXT: store i32 [[LIN_LOAD]], i32* [[LIN_START:%[^,]+]]
+// CHECK: [[GLIN_LOAD:%.+]] = load double*, double** [[GLIN_VAR:@[^,]+]]
+// CHECK-NEXT: store double* [[GLIN_LOAD]], double** [[GLIN_START:%[^,]+]]
- #pragma omp simd
// CHECK: store i64 0, i64* [[OMP_IV3:%[^,]+]]
+// Remember linear step.
+// CHECK: [[CALL_VAL:%.+]] = invoke
+// CHECK: store i64 [[CALL_VAL]], i64* [[LIN_STEP:%[^,]+]]
+
// CHECK: [[IV3:%.+]] = load i64, i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID:[0-9]+]]
// CHECK-NEXT: [[CMP3:%.+]] = icmp ult i64 [[IV3]], 4
// CHECK-NEXT: br i1 [[CMP3]], label %[[SIMPLE_LOOP3_BODY:.+]], label %[[SIMPLE_LOOP3_END:[^,]+]]
@@ -68,12 +105,34 @@
// CHECK-NEXT: [[LC_IT_1:%.+]] = mul i64 [[IV3_0]], 400
// CHECK-NEXT: [[LC_IT_2:%.+]] = sub i64 2000, [[LC_IT_1]]
// CHECK-NEXT: store i64 [[LC_IT_2]], i64* {{.+}}, !llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
- a[it]++;
+//
+// Linear start and step are used to calculate current value of the linear variable.
+// CHECK: [[LINSTART:.+]] = load i32, i32* [[LIN_START]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+// CHECK: [[LINSTEP:.+]] = load i64, i64* [[LIN_STEP]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+// CHECK-NOT: store i32 {{.+}}, i32* [[LIN_VAR]],{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+// CHECK: [[GLINSTART:.+]] = load double*, double** [[GLIN_START]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+// CHECK-NEXT: [[IV3_1:%.+]] = load i64, i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+// CHECK-NEXT: [[MUL:%.+]] = mul i64 [[IV3_1]], 1
+// CHECK-NEXT: [[GEP:%.+]] = getelementptr{{.*}}[[GLINSTART]]{{.*}}[[MUL]]
+// CHECK-NEXT: store double* [[GEP]], double** [[G_PTR_CUR:%[^,]+]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+ *g_ptr++ = 0.0;
+// CHECK: [[GEP_VAL:%.+]] = load double{{.*}}[[G_PTR_CUR]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+// CHECK: store double{{.*}}[[GEP_VAL]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
+ a[it + lin]++;
+// CHECK: [[FLT_INC:%.+]] = fadd float
+// CHECK-NEXT: store float [[FLT_INC]],{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
// CHECK: [[IV3_2:%.+]] = load i64, i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
// CHECK-NEXT: [[ADD3_2:%.+]] = add i64 [[IV3_2]], 1
// CHECK-NEXT: store i64 [[ADD3_2]], i64* [[OMP_IV3]]{{.*}}!llvm.mem.parallel_loop_access ![[SIMPLE_LOOP3_ID]]
}
// CHECK: [[SIMPLE_LOOP3_END]]
+//
+// Linear start and step are used to calculate final value of the linear variables.
+// CHECK: [[LINSTART:.+]] = load i32, i32* [[LIN_START]]
+// CHECK: [[LINSTEP:.+]] = load i64, i64* [[LIN_STEP]]
+// CHECK: store i32 {{.+}}, i32* [[LIN_VAR]],
+// CHECK: [[GLINSTART:.+]] = load double*, double** [[GLIN_START]]
+// CHECK: store double* {{.*}}[[GLIN_VAR]]
#pragma omp simd
// CHECK: store i32 0, i32* [[OMP_IV4:%[^,]+]]
Index: include/clang/AST/StmtOpenMP.h
===================================================================
--- include/clang/AST/StmtOpenMP.h
+++ include/clang/AST/StmtOpenMP.h
@@ -95,6 +95,7 @@
/// This iterator visits only those declarations that meet some run-time
/// criteria.
template <class FilterPredicate> class filtered_clause_iterator {
+ protected:
ArrayRef<OMPClause *>::const_iterator Current;
ArrayRef<OMPClause *>::const_iterator End;
FilterPredicate Pred;
@@ -126,6 +127,27 @@
bool operator!() { return Current == End; }
operator bool() { return Current != End; }
+ bool empty() const { return Current == End; }
+ };
+
+ /// \brief A filter to iterate over 'linear' clauses using a C++ range
+ /// for loop.
+ struct linear_filter : public filtered_clause_iterator<
+ std::function<bool(const OMPClause *)> > {
+ linear_filter(ArrayRef<OMPClause *> Arr)
+ : filtered_clause_iterator(Arr, [](const OMPClause *C)->bool {
+ return C->getClauseKind() == OMPC_linear;
+ }) {}
+ const OMPLinearClause *operator*() const {
+ return cast<OMPLinearClause>(*Current);
+ }
+ const OMPLinearClause *operator->() const {
+ return cast<OMPLinearClause>(*Current);
+ }
+ friend linear_filter begin(const linear_filter &range) { return range; }
+ friend linear_filter end(const linear_filter &range) {
+ return linear_filter(ArrayRef<OMPClause *>(range.End, range.End));
+ }
};
/// \brief Gets a single clause of the specified kind \a K associated with the
@@ -410,6 +432,8 @@
Expr *IterationVarRef;
/// \brief Loop last iteration number.
Expr *LastIteration;
+ /// \brief Loop number of iterations.
+ Expr *NumIterations;
/// \brief Calculation of last iteration.
Expr *CalcLastIteration;
/// \brief Loop pre-condition.
@@ -447,8 +471,9 @@
/// worksharing ones).
bool builtAll() {
return IterationVarRef != nullptr && LastIteration != nullptr &&
- PreCond != nullptr && Cond != nullptr &&
- SeparatedCond != nullptr && Init != nullptr && Inc != nullptr;
+ NumIterations != nullptr && PreCond != nullptr &&
+ Cond != nullptr && SeparatedCond != nullptr && Init != nullptr &&
+ Inc != nullptr;
}
/// \brief Initialize all the fields to null.
Index: include/clang/AST/DataRecursiveASTVisitor.h
===================================================================
--- include/clang/AST/DataRecursiveASTVisitor.h
+++ include/clang/AST/DataRecursiveASTVisitor.h
@@ -2531,7 +2531,17 @@
template <typename Derived>
bool RecursiveASTVisitor<Derived>::VisitOMPLinearClause(OMPLinearClause *C) {
TRY_TO(TraverseStmt(C->getStep()));
+ TRY_TO(TraverseStmt(C->getCalcStep()));
TRY_TO(VisitOMPClauseList(C));
+ for (auto *E : C->inits()) {
+ TRY_TO(TraverseStmt(E));
+ }
+ for (auto *E : C->updates()) {
+ TRY_TO(TraverseStmt(E));
+ }
+ for (auto *E : C->finals()) {
+ TRY_TO(TraverseStmt(E));
+ }
return true;
}
Index: include/clang/AST/RecursiveASTVisitor.h
===================================================================
--- include/clang/AST/RecursiveASTVisitor.h
+++ include/clang/AST/RecursiveASTVisitor.h
@@ -2561,7 +2561,17 @@
template <typename Derived>
bool RecursiveASTVisitor<Derived>::VisitOMPLinearClause(OMPLinearClause *C) {
TRY_TO(TraverseStmt(C->getStep()));
+ TRY_TO(TraverseStmt(C->getCalcStep()));
TRY_TO(VisitOMPClauseList(C));
+ for (auto *E : C->inits()) {
+ TRY_TO(TraverseStmt(E));
+ }
+ for (auto *E : C->updates()) {
+ TRY_TO(TraverseStmt(E));
+ }
+ for (auto *E : C->finals()) {
+ TRY_TO(TraverseStmt(E));
+ }
return true;
}
Index: include/clang/AST/OpenMPClause.h
===================================================================
--- include/clang/AST/OpenMPClause.h
+++ include/clang/AST/OpenMPClause.h
@@ -1356,7 +1356,10 @@
SourceLocation ColonLoc;
/// \brief Sets the linear step for clause.
- void setStep(Expr *Step) { *varlist_end() = Step; }
+ void setStep(Expr *Step) { *(getFinals().end()) = Step; }
+
+ /// \brief Sets the expression to calculate linear step for clause.
+ void setCalcStep(Expr *CalcStep) { *(getFinals().end() + 1) = CalcStep; }
/// \brief Build 'linear' clause with given number of variables \a NumVars.
///
@@ -1383,6 +1386,40 @@
NumVars),
ColonLoc(SourceLocation()) {}
+ /// \brief Gets the list of initial values for linear variables.
+ MutableArrayRef<Expr *> getInits() {
+ return MutableArrayRef<Expr *>(varlist_end(), varlist_size());
+ }
+ ArrayRef<const Expr *> getInits() const {
+ return llvm::makeArrayRef(varlist_end(), varlist_size());
+ }
+
+ /// \brief Sets the list of update expressions for linear variables.
+ MutableArrayRef<Expr *> getUpdates() {
+ return MutableArrayRef<Expr *>(getInits().end(), varlist_size());
+ }
+ ArrayRef<const Expr *> getUpdates() const {
+ return llvm::makeArrayRef(getInits().end(), varlist_size());
+ }
+
+ /// \brief Sets the list of final update expressions for linear variables.
+ MutableArrayRef<Expr *> getFinals() {
+ return MutableArrayRef<Expr *>(getUpdates().end(), varlist_size());
+ }
+ ArrayRef<const Expr *> getFinals() const {
+ return llvm::makeArrayRef(getUpdates().end(), varlist_size());
+ }
+
+ /// \brief Sets the list of the initial values for linear variables.
+ /// \param IL List of expressions.
+ void setInits(ArrayRef<Expr *> IL);
+
+ /// \brief Sets the list of update expressions to nullptr.
+ void clearUpdates();
+
+ /// \brief Sets the list of final update expressions to nullptr.
+ void clearFinals();
+
public:
/// \brief Creates clause with a list of variables \a VL and a linear step
/// \a Step.
@@ -1393,11 +1430,14 @@
/// \param ColonLoc Location of ':'.
/// \param EndLoc Ending location of the clause.
/// \param VL List of references to the variables.
+ /// \param IL List of initial values for the variables.
/// \param Step Linear step.
+ /// \param CalcStep Calculation of the linear step.
static OMPLinearClause *Create(const ASTContext &C, SourceLocation StartLoc,
SourceLocation LParenLoc,
SourceLocation ColonLoc, SourceLocation EndLoc,
- ArrayRef<Expr *> VL, Expr *Step);
+ ArrayRef<Expr *> VL, ArrayRef<Expr *> IL,
+ Expr *Step, Expr *CalcStep);
/// \brief Creates an empty clause with the place for \a NumVars variables.
///
@@ -1412,13 +1452,61 @@
SourceLocation getColonLoc() const { return ColonLoc; }
/// \brief Returns linear step.
- Expr *getStep() { return *varlist_end(); }
+ Expr *getStep() { return *(getFinals().end()); }
/// \brief Returns linear step.
- const Expr *getStep() const { return *varlist_end(); }
+ const Expr *getStep() const { return *(getFinals().end()); }
+ /// \brief Returns expression to calculate linear step.
+ Expr *getCalcStep() { return *(getFinals().end() + 1); }
+ /// \brief Returns expression to calculate linear step.
+ const Expr *getCalcStep() const { return *(getFinals().end() + 1); }
+
+ /// \brief Sets the list of update expressions for linear variables.
+ /// \param UL List of expressions.
+ void setUpdates(ArrayRef<Expr *> UL);
+
+ /// \brief Sets the list of final update expressions for linear variables.
+ /// \param FL List of expressions.
+ void setFinals(ArrayRef<Expr *> FL);
+
+ typedef MutableArrayRef<Expr *>::iterator inits_iterator;
+ typedef ArrayRef<const Expr *>::iterator inits_const_iterator;
+ typedef llvm::iterator_range<inits_iterator> inits_range;
+ typedef llvm::iterator_range<inits_const_iterator> inits_const_range;
+
+ inits_range inits() {
+ return inits_range(getInits().begin(), getInits().end());
+ }
+ inits_const_range inits() const {
+ return inits_const_range(getInits().begin(), getInits().end());
+ }
+
+ typedef MutableArrayRef<Expr *>::iterator updates_iterator;
+ typedef ArrayRef<const Expr *>::iterator updates_const_iterator;
+ typedef llvm::iterator_range<updates_iterator> updates_range;
+ typedef llvm::iterator_range<updates_const_iterator> updates_const_range;
+
+ updates_range updates() {
+ return updates_range(getUpdates().begin(), getUpdates().end());
+ }
+ updates_const_range updates() const {
+ return updates_const_range(getUpdates().begin(), getUpdates().end());
+ }
+
+ typedef MutableArrayRef<Expr *>::iterator finals_iterator;
+ typedef ArrayRef<const Expr *>::iterator finals_const_iterator;
+ typedef llvm::iterator_range<finals_iterator> finals_range;
+ typedef llvm::iterator_range<finals_const_iterator> finals_const_range;
+
+ finals_range finals() {
+ return finals_range(getFinals().begin(), getFinals().end());
+ }
+ finals_const_range finals() const {
+ return finals_const_range(getFinals().begin(), getFinals().end());
+ }
StmtRange children() {
return StmtRange(reinterpret_cast<Stmt **>(varlist_begin()),
- reinterpret_cast<Stmt **>(varlist_end() + 1));
+ reinterpret_cast<Stmt **>(getFinals().end() + 2));
}
static bool classof(const OMPClause *T) {
Index: tools/libclang/CIndex.cpp
===================================================================
--- tools/libclang/CIndex.cpp
+++ tools/libclang/CIndex.cpp
@@ -2031,7 +2031,17 @@
}
void OMPClauseEnqueue::VisitOMPLinearClause(const OMPLinearClause *C) {
VisitOMPClauseList(C);
+ for (const auto *E : C->inits()) {
+ Visitor->AddStmt(E);
+ }
+ for (const auto *E : C->updates()) {
+ Visitor->AddStmt(E);
+ }
+ for (const auto *E : C->finals()) {
+ Visitor->AddStmt(E);
+ }
Visitor->AddStmt(C->getStep());
+ Visitor->AddStmt(C->getCalcStep());
}
void OMPClauseEnqueue::VisitOMPAlignedClause(const OMPAlignedClause *C) {
VisitOMPClauseList(C);
Index: lib/Sema/SemaOpenMP.cpp
===================================================================
--- lib/Sema/SemaOpenMP.cpp
+++ lib/Sema/SemaOpenMP.cpp
@@ -622,6 +622,10 @@
PopExpressionEvaluationContext();
}
+static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
+ Expr *NumIterations, Sema &SemaRef,
+ Scope *S);
+
namespace {
class VarDeclFilterCCC : public CorrectionCandidateCallback {
@@ -2915,6 +2919,7 @@
// Save results
Built.IterationVarRef = IV.get();
Built.LastIteration = LastIteration.get();
+ Built.NumIterations = NumIterations.get();
Built.CalcLastIteration = CalcLastIteration.get();
Built.PreCond = PreCond.get();
Built.Cond = Cond.get();
@@ -2958,6 +2963,16 @@
assert((CurContext->isDependentContext() || B.builtAll()) &&
"omp simd loop exprs were not built");
+ if (!CurContext->isDependentContext()) {
+ // Finalize the clauses that need pre-built expressions for CodeGen.
+ for (auto C : Clauses) {
+ if (auto LC = dyn_cast<OMPLinearClause>(C))
+ if (FinishOpenMPLinearClause(*LC, cast<DeclRefExpr>(B.IterationVarRef),
+ B.NumIterations, *this, CurScope))
+ return StmtError();
+ }
+ }
+
getCurFunction()->setHasBranchProtectedScope();
return OMPSimdDirective::Create(Context, StartLoc, EndLoc, NestedLoopCount,
Clauses, AStmt, B);
@@ -5231,11 +5246,13 @@
SourceLocation ColonLoc,
SourceLocation EndLoc) {
SmallVector<Expr *, 8> Vars;
+ SmallVector<Expr *, 8> Inits;
for (auto &RefExpr : VarList) {
assert(RefExpr && "NULL expr in OpenMP linear clause.");
if (isa<DependentScopeDeclRefExpr>(RefExpr)) {
// It will be analyzed later.
Vars.push_back(RefExpr);
+ Inits.push_back(nullptr);
continue;
}
@@ -5277,6 +5294,7 @@
if (QType->isDependentType() || QType->isInstantiationDependentType()) {
// It will be analyzed later.
Vars.push_back(DE);
+ Inits.push_back(nullptr);
continue;
}
@@ -5322,14 +5340,27 @@
continue;
}
+ // Build var to save initial value.
+ VarDecl *Init = BuildVarDecl(*this, ELoc, DE->getType(), ".linear.start");
+ AddInitializerToDecl(Init, DefaultLvalueConversion(DE).get(),
+ /*DirectInit*/ false, /*TypeMayContainAuto*/ false);
+ CurContext->addDecl(Init);
+ Init->setIsUsed();
+ auto InitRef = DeclRefExpr::Create(
+ Context, /*QualifierLoc*/ NestedNameSpecifierLoc(),
+ /*TemplateKWLoc*/ SourceLocation(), Init,
+ /*isEnclosingLocal*/ false, DE->getLocStart(), DE->getType(),
+ /*VK*/ VK_LValue);
DSAStack->addDSA(VD, DE, OMPC_linear);
Vars.push_back(DE);
+ Inits.push_back(InitRef);
}
if (Vars.empty())
return nullptr;
Expr *StepExpr = Step;
+ Expr *CalcStepExpr = nullptr;
if (Step && !Step->isValueDependent() && !Step->isTypeDependent() &&
!Step->isInstantiationDependent() &&
!Step->containsUnexpandedParameterPack()) {
@@ -5339,17 +5370,85 @@
return nullptr;
StepExpr = Val.get();
+ // Build var to save the step value.
+ VarDecl *SaveVar =
+ BuildVarDecl(*this, StepLoc, StepExpr->getType(), ".linear.step");
+ CurContext->addDecl(SaveVar);
+ SaveVar->setIsUsed();
+ ExprResult SaveRef =
+ BuildDeclRefExpr(SaveVar, StepExpr->getType(), VK_LValue, StepLoc);
+ ExprResult CalcStep =
+ BuildBinOp(CurScope, StepLoc, BO_Assign, SaveRef.get(), StepExpr);
+
// Warn about zero linear step (it would be probably better specified as
// making corresponding variables 'const').
llvm::APSInt Result;
- if (StepExpr->isIntegerConstantExpr(Result, Context) &&
- !Result.isNegative() && !Result.isStrictlyPositive())
+ bool IsConstant = StepExpr->isIntegerConstantExpr(Result, Context);
+ if (IsConstant && !Result.isNegative() && !Result.isStrictlyPositive())
Diag(StepLoc, diag::warn_omp_linear_step_zero) << Vars[0]
<< (Vars.size() > 1);
+ if (!IsConstant && CalcStep.isUsable()) {
+ // Calculate the step beforehand instead of doing this on each iteration.
+ // (This is not used if the number of iterations may be kfold-ed).
+ CalcStepExpr = CalcStep.get();
+ }
}
return OMPLinearClause::Create(Context, StartLoc, LParenLoc, ColonLoc, EndLoc,
- Vars, StepExpr);
+ Vars, Inits, StepExpr, CalcStepExpr);
+}
+
+static bool FinishOpenMPLinearClause(OMPLinearClause &Clause, DeclRefExpr *IV,
+ Expr *NumIterations, Sema &SemaRef,
+ Scope *S) {
+ // Walk the vars and build update/final expressions for the CodeGen.
+ SmallVector<Expr *, 8> Updates;
+ SmallVector<Expr *, 8> Finals;
+ Expr *Step = Clause.getStep();
+ Expr *CalcStep = Clause.getCalcStep();
+ // OpenMP [2.14.3.7, linear clause]
+ // If linear-step is not specified it is assumed to be 1.
+ if (Step == nullptr)
+ Step = SemaRef.ActOnIntegerConstant(SourceLocation(), 1).get();
+ else if (CalcStep)
+ Step = cast<BinaryOperator>(CalcStep)->getLHS();
+ bool HasErrors = false;
+ auto CurInit = Clause.inits().begin();
+ for (auto &RefExpr : Clause.varlists()) {
+ Expr *InitExpr = *CurInit;
+
+ // Build privatized reference to the current linear var.
+ auto DE = cast<DeclRefExpr>(RefExpr);
+ auto PrivateRef = DeclRefExpr::Create(
+ SemaRef.Context, /*QualifierLoc*/ DE->getQualifierLoc(),
+ /*TemplateKWLoc*/ SourceLocation(), DE->getDecl(),
+ /* RefersToEnclosingVariableOrCapture */ true, DE->getLocStart(),
+ DE->getType(), /*VK*/ VK_LValue);
+
+ // Build update: Var = InitExpr + IV * Step
+ ExprResult Update =
+ BuildCounterUpdate(SemaRef, S, RefExpr->getExprLoc(), PrivateRef,
+ InitExpr, IV, Step, /* Subtract */ false);
+ Update = SemaRef.ActOnFinishFullExpr(Update.get());
+
+ // Build final: Var = InitExpr + NumIterations * Step
+ ExprResult Final =
+ BuildCounterUpdate(SemaRef, S, RefExpr->getExprLoc(), RefExpr, InitExpr,
+ NumIterations, Step, /* Subtract */ false);
+ Final = SemaRef.ActOnFinishFullExpr(Final.get());
+ if (!Update.isUsable() || !Final.isUsable()) {
+ Updates.push_back(nullptr);
+ Finals.push_back(nullptr);
+ HasErrors = true;
+ } else {
+ Updates.push_back(Update.get());
+ Finals.push_back(Final.get());
+ }
+ ++CurInit;
+ }
+ Clause.setUpdates(Updates);
+ Clause.setFinals(Finals);
+ return HasErrors;
}
OMPClause *Sema::ActOnOpenMPAlignedClause(
Index: lib/AST/Stmt.cpp
===================================================================
--- lib/AST/Stmt.cpp
+++ lib/AST/Stmt.cpp
@@ -1291,27 +1291,56 @@
return new (Mem) OMPSharedClause(N);
}
-OMPLinearClause *OMPLinearClause::Create(const ASTContext &C,
- SourceLocation StartLoc,
- SourceLocation LParenLoc,
- SourceLocation ColonLoc,
- SourceLocation EndLoc,
- ArrayRef<Expr *> VL, Expr *Step) {
+void OMPLinearClause::setInits(ArrayRef<Expr *> IL) {
+ assert(IL.size() == varlist_size() &&
+ "Number of inits is not the same as the preallocated buffer");
+ std::copy(IL.begin(), IL.end(), varlist_end());
+}
+
+void OMPLinearClause::setUpdates(ArrayRef<Expr *> UL) {
+ assert(UL.size() == varlist_size() &&
+ "Number of updates is not the same as the preallocated buffer");
+ std::copy(UL.begin(), UL.end(), getInits().end());
+}
+
+void OMPLinearClause::setFinals(ArrayRef<Expr *> FL) {
+ assert(FL.size() == varlist_size() &&
+ "Number of final updates is not the same as the preallocated buffer");
+ std::copy(FL.begin(), FL.end(), getUpdates().end());
+}
+
+void OMPLinearClause::clearUpdates() {
+ std::fill(getInits().end(), getInits().end() + varlist_size(), nullptr);
+}
+
+void OMPLinearClause::clearFinals() {
+ std::fill(getUpdates().end(), getUpdates().end() + varlist_size(), nullptr);
+}
+
+OMPLinearClause *
+OMPLinearClause::Create(const ASTContext &C, SourceLocation StartLoc,
+ SourceLocation LParenLoc, SourceLocation ColonLoc,
+ SourceLocation EndLoc, ArrayRef<Expr *> VL,
+ ArrayRef<Expr *> IL, Expr *Step, Expr *CalcStep) {
void *Mem = C.Allocate(llvm::RoundUpToAlignment(sizeof(OMPLinearClause),
llvm::alignOf<Expr *>()) +
- sizeof(Expr *) * (VL.size() + 1));
+ 4 * sizeof(Expr *) * (VL.size() + 2));
OMPLinearClause *Clause = new (Mem)
OMPLinearClause(StartLoc, LParenLoc, ColonLoc, EndLoc, VL.size());
Clause->setVarRefs(VL);
+ Clause->setInits(IL);
+ Clause->clearUpdates();
+ Clause->clearFinals();
Clause->setStep(Step);
+ Clause->setCalcStep(CalcStep);
return Clause;
}
OMPLinearClause *OMPLinearClause::CreateEmpty(const ASTContext &C,
unsigned NumVars) {
void *Mem = C.Allocate(llvm::RoundUpToAlignment(sizeof(OMPLinearClause),
llvm::alignOf<Expr *>()) +
- sizeof(Expr *) * (NumVars + 1));
+ 4 * sizeof(Expr *) * (NumVars + 2));
return new (Mem) OMPLinearClause(NumVars);
}
Index: lib/AST/StmtProfile.cpp
===================================================================
--- lib/AST/StmtProfile.cpp
+++ lib/AST/StmtProfile.cpp
@@ -359,7 +359,17 @@
}
void OMPClauseProfiler::VisitOMPLinearClause(const OMPLinearClause *C) {
VisitOMPClauseList(C);
+ for (auto *E : C->inits()) {
+ Profiler->VisitStmt(E);
+ }
+ for (auto *E : C->updates()) {
+ Profiler->VisitStmt(E);
+ }
+ for (auto *E : C->finals()) {
+ Profiler->VisitStmt(E);
+ }
Profiler->VisitStmt(C->getStep());
+ Profiler->VisitStmt(C->getCalcStep());
}
void OMPClauseProfiler::VisitOMPAlignedClause(const OMPAlignedClause *C) {
VisitOMPClauseList(C);
Index: lib/CodeGen/CGStmtOpenMP.cpp
===================================================================
--- lib/CodeGen/CGStmtOpenMP.cpp
+++ lib/CodeGen/CGStmtOpenMP.cpp
@@ -266,6 +266,13 @@
for (auto I : S.updates()) {
EmitIgnoredExpr(I);
}
+ // Update the linear variables.
+ for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) {
+ for (auto U : C->updates()) {
+ EmitIgnoredExpr(U);
+ }
+ }
+
// On a continue in the body, jump to the end.
auto Continue = getJumpDestInCurrentScope("omp.body.continue");
BreakContinueStack.push_back(BreakContinue(JumpDest(), Continue));
@@ -336,6 +343,12 @@
}
++IC;
}
+ // Emit the final values of the linear variables.
+ for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) {
+ for (auto F : C->finals()) {
+ EmitIgnoredExpr(F);
+ }
+ }
}
static void EmitOMPAlignedClause(CodeGenFunction &CGF, CodeGenModule &CGM,
@@ -381,6 +394,25 @@
}
}
+static void
+EmitPrivateLinearVars(CodeGenFunction &CGF, const OMPExecutableDirective &D,
+ CodeGenFunction::OMPPrivateScope &PrivateScope) {
+ for (auto Clause : OMPExecutableDirective::linear_filter(D.clauses())) {
+ for (auto *E : Clause->varlists()) {
+ auto VD = cast<VarDecl>(cast<DeclRefExpr>(E)->getDecl());
+ bool IsRegistered = PrivateScope.addPrivate(VD, [&]()->llvm::Value * {
+ // Emit var without initialization.
+ auto VarEmission = CGF.EmitAutoVarAlloca(*VD);
+ CGF.EmitAutoVarCleanups(VarEmission);
+ return VarEmission.getAllocatedAddress();
+ });
+ assert(IsRegistered && "linear var already registered as private");
+ // Silence the warning about unused variable.
+ (void)IsRegistered;
+ }
+ }
+}
+
void CodeGenFunction::EmitOMPSimdDirective(const OMPSimdDirective &S) {
// Pragma 'simd' code depends on presence of 'lastprivate'.
// If present, we have to separate last iteration of the loop:
@@ -428,6 +460,14 @@
InlinedOpenMPRegionScopeRAII Region(*this, S);
+ // Emit inits for the linear variables.
+ for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) {
+ for (auto Init : C->inits()) {
+ auto *D = cast<VarDecl>(cast<DeclRefExpr>(Init)->getDecl());
+ EmitVarDecl(*D);
+ }
+ }
+
// Emit the loop iteration variable.
const Expr *IVExpr = S.getIterationVariable();
const VarDecl *IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());
@@ -443,6 +483,17 @@
EmitIgnoredExpr(S.getCalcLastIteration());
}
+ // Emit the linear steps for the linear clauses.
+ // If a step is not constant, it is pre-calculated before the loop.
+ for (auto C : OMPExecutableDirective::linear_filter(S.clauses())) {
+ if (auto CS = cast_or_null<BinaryOperator>(C->getCalcStep()))
+ if (auto SaveRef = cast<DeclRefExpr>(CS->getLHS())) {
+ EmitVarDecl(*cast<VarDecl>(SaveRef->getDecl()));
+ // Emit calculation of the linear step.
+ EmitIgnoredExpr(CS);
+ }
+ }
+
if (SeparateIter) {
// Emit: if (LastIteration > 0) - begin.
RegionCounter Cnt = getPGORegionCounter(&S);
@@ -455,6 +506,7 @@
{
OMPPrivateScope LoopScope(*this);
EmitPrivateLoopCounters(*this, LoopScope, S.counters());
+ EmitPrivateLinearVars(*this, S, LoopScope);
EmitOMPPrivateClause(S, LoopScope);
(void)LoopScope.Privatize();
EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
@@ -473,6 +525,7 @@
{
OMPPrivateScope LoopScope(*this);
EmitPrivateLoopCounters(*this, LoopScope, S.counters());
+ EmitPrivateLinearVars(*this, S, LoopScope);
EmitOMPPrivateClause(S, LoopScope);
(void)LoopScope.Privatize();
EmitOMPInnerLoop(S, LoopScope.requiresCleanups(),
Index: lib/Serialization/ASTReaderStmt.cpp
===================================================================
--- lib/Serialization/ASTReaderStmt.cpp
+++ lib/Serialization/ASTReaderStmt.cpp
@@ -1928,7 +1928,20 @@
for (unsigned i = 0; i != NumVars; ++i)
Vars.push_back(Reader->Reader.ReadSubExpr());
C->setVarRefs(Vars);
+ Vars.clear();
+ for (unsigned i = 0; i != NumVars; ++i)
+ Vars.push_back(Reader->Reader.ReadSubExpr());
+ C->setInits(Vars);
+ Vars.clear();
+ for (unsigned i = 0; i != NumVars; ++i)
+ Vars.push_back(Reader->Reader.ReadSubExpr());
+ C->setUpdates(Vars);
+ Vars.clear();
+ for (unsigned i = 0; i != NumVars; ++i)
+ Vars.push_back(Reader->Reader.ReadSubExpr());
+ C->setFinals(Vars);
C->setStep(Reader->Reader.ReadSubExpr());
+ C->setCalcStep(Reader->Reader.ReadSubExpr());
}
void OMPClauseReader::VisitOMPAlignedClause(OMPAlignedClause *C) {
Index: lib/Serialization/ASTWriterStmt.cpp
===================================================================
--- lib/Serialization/ASTWriterStmt.cpp
+++ lib/Serialization/ASTWriterStmt.cpp
@@ -1820,9 +1820,20 @@
Record.push_back(C->varlist_size());
Writer->Writer.AddSourceLocation(C->getLParenLoc(), Record);
Writer->Writer.AddSourceLocation(C->getColonLoc(), Record);
- for (auto *VE : C->varlists())
+ for (auto *VE : C->varlists()) {
+ Writer->Writer.AddStmt(VE);
+ }
+ for (auto *VE : C->inits()) {
Writer->Writer.AddStmt(VE);
+ }
+ for (auto *VE : C->updates()) {
+ Writer->Writer.AddStmt(VE);
+ }
+ for (auto *VE : C->finals()) {
+ Writer->Writer.AddStmt(VE);
+ }
Writer->Writer.AddStmt(C->getStep());
+ Writer->Writer.AddStmt(C->getCalcStep());
}
void OMPClauseWriter::VisitOMPAlignedClause(OMPAlignedClause *C) {
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
