Index: include/clang/AST/Stmt.h =================================================================== --- include/clang/AST/Stmt.h (revision 206803) +++ include/clang/AST/Stmt.h (working copy) @@ -20,6 +20,8 @@ #include "clang/Basic/IdentifierTable.h" #include "clang/Basic/LLVM.h" #include "clang/Basic/SourceLocation.h" +#include "clang/Basic/VectorizeHint.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/Support/Compiler.h" @@ -1015,13 +1017,42 @@ } }; +/// LoopVectorizeHints - This provides the interface +/// for specifying and retrieving vectorization hints +/// +class LoopVectorizeHints { + SmallVector CondBrVectorizeHints; +public: + /// Add a vectorization hint to the conditional branch of the loop + /// Operand[0] should be a string like llvm.vectorizer.enable + void addCondBrVectorizeHint(VectorizeHint Hint) { + CondBrVectorizeHints.push_back(Hint); + } + + /// Returns true if vectorization hints have been specified + bool hasCondBrVecHints() const { + return !CondBrVectorizeHints.empty(); + } + + /// Beginning of list of vectorization hints + SmallVectorImpl::const_iterator beginCondBrVecHints() const { + return CondBrVectorizeHints.begin(); + } + + /// Terminator of vectorization hints list + SmallVectorImpl::const_iterator endCondBrVecHints() const { + return CondBrVectorizeHints.end(); + } +}; + /// WhileStmt - This represents a 'while' stmt. /// -class WhileStmt : public Stmt { +class WhileStmt : public Stmt, public LoopVectorizeHints { enum { VAR, COND, BODY, END_EXPR }; Stmt* SubExprs[END_EXPR]; SourceLocation WhileLoc; + public: WhileStmt(const ASTContext &C, VarDecl *Var, Expr *cond, Stmt *body, SourceLocation WL); @@ -1030,6 +1061,7 @@ explicit WhileStmt(EmptyShell Empty) : Stmt(WhileStmtClass, Empty) { } /// \brief Retrieve the variable declared in this "while" statement, if any. + /// /// In the following example, "x" is the condition variable. /// \code @@ -1073,7 +1105,7 @@ /// DoStmt - This represents a 'do/while' stmt. /// -class DoStmt : public Stmt { +class DoStmt : public Stmt, public LoopVectorizeHints { enum { BODY, COND, END_EXPR }; Stmt* SubExprs[END_EXPR]; SourceLocation DoLoc; @@ -1124,7 +1156,7 @@ /// the init/cond/inc parts of the ForStmt will be null if they were not /// specified in the source. /// -class ForStmt : public Stmt { +class ForStmt : public Stmt, public LoopVectorizeHints { enum { INIT, CONDVAR, COND, INC, BODY, END_EXPR }; Stmt* SubExprs[END_EXPR]; // SubExprs[INIT] is an expression or declstmt. SourceLocation ForLoc; Index: include/clang/Basic/TokenKinds.def =================================================================== --- include/clang/Basic/TokenKinds.def (revision 206803) +++ include/clang/Basic/TokenKinds.def (working copy) @@ -701,6 +701,11 @@ ANNOTATION(pragma_openmp) ANNOTATION(pragma_openmp_end) +// Annotations for vectorization pragma directives #pragma vectorize enable +// The lexer produces these so that they only take effect when the parser +// handles #pragma vectorize ... directives. +ANNOTATION(pragma_vectorize_hint) + // Annotations for module import translated from #include etc. ANNOTATION(module_include) ANNOTATION(module_begin) Index: include/clang/Basic/VectorizeHint.h =================================================================== --- include/clang/Basic/VectorizeHint.h (revision 0) +++ include/clang/Basic/VectorizeHint.h (working copy) @@ -0,0 +1,34 @@ +//===--- VectorizeHint.h - Types for VectorizeHint --------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + + +#ifndef LLVM_CLANG_BASIC_VECTORIZEHINT_H +#define LLVM_CLANG_BASIC_VECTORIZEHINT_H + +namespace clang { + +enum VectorizeHintKind { + VH_UNKNOWN, + VH_ENABLE, + VH_DISABLE, + VH_WIDTH, + VH_UNROLL +}; + +/// \brief Vectorization hint specified by a pragma vectorize +/// and used by codegen to attach metadata to the IR +struct VectorizeHint { + VectorizeHintKind Kind; + uint64_t Value; +}; + + +} // end namespace clang + +#endif // LLVM_CLANG_BASIC_VECTORIZEHINT_H Index: include/clang/Parse/Parser.h =================================================================== --- include/clang/Parse/Parser.h (revision 206803) +++ include/clang/Parse/Parser.h (working copy) @@ -17,6 +17,7 @@ #include "clang/Basic/OpenMPKinds.h" #include "clang/Basic/OperatorPrecedence.h" #include "clang/Basic/Specifiers.h" +#include "clang/Basic/VectorizeHint.h" #include "clang/Lex/CodeCompletionHandler.h" #include "clang/Lex/Preprocessor.h" #include "clang/Sema/DeclSpec.h" @@ -160,6 +161,7 @@ std::unique_ptr MSConstSeg; std::unique_ptr MSCodeSeg; std::unique_ptr MSSection; + std::unique_ptr VectorizeHintHandler; std::unique_ptr CommentSemaHandler; @@ -517,6 +519,10 @@ /// #pragma clang __debug captured StmtResult HandlePragmaCaptured(); + /// \brief Handle the annotation token produced for + /// #pragma vectorize... + VectorizeHint HandlePragmaVectorizeHint(); + /// GetLookAheadToken - This peeks ahead N tokens and returns that token /// without consuming any tokens. LookAhead(0) returns 'Tok', LookAhead(1) /// returns the token after Tok, etc. @@ -1598,6 +1604,7 @@ StmtResult ParseReturnStatement(); StmtResult ParseAsmStatement(bool &msAsm); StmtResult ParseMicrosoftAsmStatement(SourceLocation AsmLoc); + StmtResult ParsePragmaVectorizeHint(); /// \brief Describes the behavior that should be taken for an __if_exists /// block. Index: lib/CodeGen/CGStmt.cpp =================================================================== --- lib/CodeGen/CGStmt.cpp (revision 206803) +++ lib/CodeGen/CGStmt.cpp (working copy) @@ -18,6 +18,7 @@ #include "clang/AST/StmtVisitor.h" #include "clang/Basic/PrettyStackTrace.h" #include "clang/Basic/TargetInfo.h" +#include "clang/Basic/VectorizeHint.h" #include "clang/Sema/SemaDiagnostic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/IR/CallSite.h" @@ -500,6 +501,59 @@ EmitBlock(ContBlock, true); } +llvm::MDNode *CodeGenFunction::getVectorizeHints(llvm::LLVMContext &Context, + VectorizeHintIterator Begin, + VectorizeHintIterator End) { + llvm::IntegerType *BoolTy = llvm::Type::getInt1Ty(Context); + llvm::IntegerType *IntTy = llvm::Type::getInt32Ty(Context); + + // If there are any vectorize hints on this loop + // add them as metadata on the conditional branch + SmallVectorImpl::const_iterator I; + SmallVector Metadata(1); + + for (I = Begin; I != End; ++I) { + // Replace name with MDString + SmallVector OpValues; + + llvm::MDString *Name; + llvm::Value *Value; + + switch(I->Kind) { + case VH_ENABLE: + Name = llvm::MDString::get(Context, "llvm.vectorizer.enable"); + Value = llvm::ConstantInt::get(BoolTy, true); + break; + case VH_DISABLE: + Name = llvm::MDString::get(Context, "llvm.vectorizer.enable"); + Value = llvm::ConstantInt::get(BoolTy, false); + break; + case VH_WIDTH: + Name = llvm::MDString::get(Context, "llvm.vectorizer.width"); + Value = llvm::ConstantInt::get(IntTy, I->Value); + break; + case VH_UNROLL: + Name = llvm::MDString::get(Context, "llvm.vectorizer.unroll"); + Value = llvm::ConstantInt::get(IntTy, I->Value); + break; + default: + continue; + } + + OpValues.push_back(Name); + OpValues.push_back(Value); + + // Set or overwrite metadata indicated by Name + Metadata.push_back(llvm::MDNode::get(Context, OpValues)); + } + + // Add llvm.loop MDNode to CondBr + llvm::MDNode *LoopID = llvm::MDNode::get(Context, Metadata); + LoopID->replaceOperandWith(0, LoopID); //First op points to itself + + return LoopID; +} + void CodeGenFunction::EmitWhileStmt(const WhileStmt &S) { RegionCounter Cnt = getPGORegionCounter(&S); @@ -545,13 +599,24 @@ llvm::BasicBlock *ExitBlock = LoopExit.getBlock(); if (ConditionScope.requiresCleanups()) ExitBlock = createBasicBlock("while.exit"); - Builder.CreateCondBr(BoolCondVal, LoopBody, ExitBlock, - PGO.createLoopWeights(S.getCond(), Cnt)); + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, LoopBody, ExitBlock, + PGO.createLoopWeights(S.getCond(), Cnt)); if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); } + + // Attach metadata to loop body conditional branch + if (S.hasCondBrVecHints()) + { + llvm::MDNode *LoopID = getVectorizeHints(LoopBody->getContext(), + S.beginCondBrVecHints(), + S.endCondBrVecHints()); + + CondBr->setMetadata("llvm.loop", LoopID); + } } // Emit the loop body. We have to emit this in a cleanup scope @@ -617,10 +682,22 @@ EmitBoolCondBranch = false; // As long as the condition is true, iterate the loop. - if (EmitBoolCondBranch) - Builder.CreateCondBr(BoolCondVal, LoopBody, LoopExit.getBlock(), - PGO.createLoopWeights(S.getCond(), Cnt)); + if (EmitBoolCondBranch) { + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, LoopBody, LoopExit.getBlock(), + PGO.createLoopWeights(S.getCond(), Cnt)); + // Attach metadata to loop body conditional branch + if (S.hasCondBrVecHints()) + { + llvm::MDNode *LoopID = getVectorizeHints(LoopBody->getContext(), + S.beginCondBrVecHints(), + S.endCondBrVecHints()); + + CondBr->setMetadata("llvm.loop", LoopID); + } + } + // Emit the exit block. EmitBlock(LoopExit.getBlock()); @@ -684,9 +761,20 @@ // C99 6.8.5p2/p4: The first substatement is executed if the expression // compares unequal to 0. The condition must be a scalar type. llvm::Value *BoolCondVal = EvaluateExprAsBool(S.getCond()); - Builder.CreateCondBr(BoolCondVal, ForBody, ExitBlock, - PGO.createLoopWeights(S.getCond(), Cnt)); + llvm::BranchInst *CondBr = Builder.CreateCondBr( + BoolCondVal, ForBody, ExitBlock, + PGO.createLoopWeights(S.getCond(), Cnt)); + // Attach metadata to loop body conditional branch + if (S.hasCondBrVecHints()) + { + llvm::MDNode *LoopID = getVectorizeHints(ForBody->getContext(), + S.beginCondBrVecHints(), + S.endCondBrVecHints()); + + CondBr->setMetadata("llvm.loop", LoopID); + } + if (ExitBlock != LoopExit.getBlock()) { EmitBlock(ExitBlock); EmitBranchThroughCleanup(LoopExit); Index: lib/CodeGen/CodeGenFunction.h =================================================================== --- lib/CodeGen/CodeGenFunction.h (revision 206803) +++ lib/CodeGen/CodeGenFunction.h (working copy) @@ -2518,6 +2518,10 @@ void SetFPAccuracy(llvm::Value *Val, float Accuracy); private: + typedef SmallVectorImpl::const_iterator VectorizeHintIterator; + llvm::MDNode *getVectorizeHints(llvm::LLVMContext &Context, + VectorizeHintIterator Begin, + VectorizeHintIterator End); llvm::MDNode *getRangeForLoadFromType(QualType Ty); void EmitReturnOfRValue(RValue RV, QualType Ty); Index: lib/Parse/ParsePragma.cpp =================================================================== --- lib/Parse/ParsePragma.cpp (revision 206803) +++ lib/Parse/ParsePragma.cpp (working copy) @@ -16,6 +16,7 @@ #include "clang/Parse/ParseDiagnostic.h" #include "clang/Parse/Parser.h" #include "clang/Sema/Scope.h" +#include "clang/Basic/VectorizeHint.h" #include "llvm/ADT/StringSwitch.h" using namespace clang; @@ -131,6 +132,12 @@ Token &FirstToken) override; }; +struct PragmaVectorizeHintHandler : public PragmaHandler { + PragmaVectorizeHintHandler() : PragmaHandler("vectorize") {} + void HandlePragma(Preprocessor &PP, PragmaIntroducerKind Introducer, + Token &FirstToken) override; +}; + } // end namespace void Parser::initializePragmaHandlers() { @@ -195,6 +202,9 @@ MSSection.reset(new PragmaMSPragma("section")); PP.AddPragmaHandler(MSSection.get()); } + + VectorizeHintHandler.reset(new PragmaVectorizeHintHandler()); + PP.AddPragmaHandler(VectorizeHintHandler.get()); } void Parser::resetPragmaHandlers() { @@ -249,6 +259,9 @@ PP.RemovePragmaHandler("STDC", FPContractHandler.get()); FPContractHandler.reset(); + + PP.RemovePragmaHandler(VectorizeHintHandler.get()); + VectorizeHintHandler.reset(); } /// \brief Handle the annotation token produced for #pragma unused(...) @@ -570,6 +583,14 @@ DiagnosticsEngine::Error, "'#pragma %0' not implemented."); } +VectorizeHint Parser::HandlePragmaVectorizeHint() { + assert(Tok.is(tok::annot_pragma_vectorize_hint)); + VectorizeHint Hint = + *static_cast(Tok.getAnnotationValue()); + + return Hint; +} + // #pragma GCC visibility comes in two variants: // 'push' '(' [visibility] ')' // 'pop' @@ -1531,3 +1552,102 @@ Actions.ActOnPragmaMSComment(Kind, ArgumentString); } + +/// \brief Handle the \#pragma vectorize hint +/// +/// The syntax is: +/// \code +/// #pragma vectorize enable/disable +/// #pragma vectorize width(_value_) +/// #pragma vectorize unroll(_value_) +/// #pragma vectorize _option_(_value_) _option_(_value_) ... +/// \endcode +/// Specifying "enable" forces the loop-vectorizer to consider vectorizing +/// the subsequent loop, but it does not guarantee it. Values specified with +/// "width" or "unroll" are used by the loop-vectorizer only. Specifying +/// "disable" prevents vectorization of the subsequent loop. An inner-loop must +/// follow the pragma vectorize statements. +void PragmaVectorizeHintHandler::HandlePragma(Preprocessor &PP, + PragmaIntroducerKind Introducer, + Token &Tok) { + SourceLocation VecHintLoc = Tok.getLocation(); + + SmallVector TokenList; + + // Read the hint identifier + PP.Lex(Tok); + if (Tok.isNot(tok::identifier)) { + PP.Diag(Tok.getLocation(), diag::err_expected) << + "enable, disable, width, or unroll"; + return; + } + + while(Tok.is(tok::identifier)) { + + // Verify that this is one of the whitelisted vectorize hints + IdentifierInfo *II = Tok.getIdentifierInfo(); + VectorizeHintKind Kind = + llvm::StringSwitch(II->getName()) + .Case("enable", VH_ENABLE) + .Case("disable", VH_DISABLE) + .Case("width", VH_WIDTH) + .Case("unroll", VH_UNROLL) + .Default(VH_UNKNOWN); + + if (Kind == VH_UNKNOWN) { + PP.Diag(Tok.getLocation(), diag::err_expected) << + "enable, disable, width, or unroll"; + return; + } + + VectorizeHint *Hint = + (VectorizeHint*) PP.getPreprocessorAllocator().Allocate( + sizeof(VectorizeHint), llvm::alignOf()); + + Hint->Kind = Kind; + Hint->Value = 1; + + if (Kind == VH_WIDTH || Kind == VH_UNROLL) { + // Read width/unroll amount + PP.Lex(Tok); + if (Tok.isNot(tok::l_paren)) { + PP.Diag(Tok.getLocation(), diag::err_expected) << tok::l_paren; + return; + } + + PP.Lex(Tok); + if (Tok.isNot(tok::numeric_constant) || + !PP.parseSimpleIntegerLiteral(Tok, Hint->Value) || + Hint->Value <= 1) { + PP.Diag(Tok.getLocation(), diag::err_expected) << "positive integer"; + } + + if (Tok.isNot(tok::r_paren)) { + PP.Diag(Tok.getLocation(), diag::err_expected) << tok::r_paren; + return; + } + } + + // Get next token + PP.Lex(Tok); + + // Vectorization Hint Token + Token VecHintTok; + VecHintTok.startToken(); + VecHintTok.setKind(tok::annot_pragma_vectorize_hint); + VecHintTok.setLocation(VecHintLoc); + VecHintTok.setAnnotationValue(static_cast(Hint)); + TokenList.push_back(VecHintTok); + } + + if (Tok.isNot(tok::eod)) { + PP.Diag(Tok.getLocation(), diag::warn_pragma_extra_tokens_at_eol) << + "vectorize"; + return; + } + + while (!TokenList.empty()) { + PP.EnterToken(TokenList.back()); + TokenList.pop_back(); + } +} Index: lib/Parse/ParseStmt.cpp =================================================================== --- lib/Parse/ParseStmt.cpp (revision 206803) +++ lib/Parse/ParseStmt.cpp (working copy) @@ -19,6 +19,7 @@ #include "clang/Basic/PrettyStackTrace.h" #include "clang/Basic/SourceManager.h" #include "clang/Basic/TargetInfo.h" +#include "clang/Basic/VectorizeHint.h" #include "clang/Sema/DeclSpec.h" #include "clang/Sema/PrettyDeclStackTrace.h" #include "clang/Sema/Scope.h" @@ -354,6 +355,10 @@ ProhibitAttributes(Attrs); HandlePragmaMSPragma(); return StmtEmpty(); + + case tok::annot_pragma_vectorize_hint: + ProhibitAttributes(Attrs); + return ParsePragmaVectorizeHint(); } // If we reached this code, the statement must end in a semicolon. @@ -1750,6 +1755,45 @@ return Actions.ActOnReturnStmt(ReturnLoc, R.take()); } +StmtResult Parser::ParsePragmaVectorizeHint() +{ + // Get vectorize hint and consume annotated token + VectorizeHint Hint = HandlePragmaVectorizeHint(); + SourceLocation PragmaLoc = ConsumeToken(); + + // Get next statement + if (Tok.is(tok::annot_pragma_vectorize_hint) || + Tok.is(tok::kw_while) || Tok.is(tok::kw_do) || Tok.is(tok::kw_for)) { + StmtResult NextStmt = ParseStatement(); + + if (!NextStmt.isUsable()) + return NextStmt; + + Stmt *Loop = NextStmt.get(); + if (isa(Loop)) + Loop = cast(Loop)->getSubStmt(); + + // Verify it is a loop + if (isa(Loop)) { + WhileStmt *While = cast(Loop); + While->addCondBrVectorizeHint(Hint); + } else if (isa(Loop)) { + DoStmt *Do = cast(Loop); + Do->addCondBrVectorizeHint(Hint); + } else if (isa(Loop)) { + ForStmt *For = cast(Loop); + For->addCondBrVectorizeHint(Hint); + } + + return NextStmt; + } else { + Diag(PragmaLoc, diag::err_expected) << + "for, while, or do-while loop to immediately follow pragma vectorize"; + } + + return StmtEmpty(); +} + namespace { class ClangAsmParserCallback : public llvm::MCAsmParserSemaCallback { Parser &TheParser; Index: test/CodeGen/pragma-vectorize.c =================================================================== --- test/CodeGen/pragma-vectorize.c (revision 0) +++ test/CodeGen/pragma-vectorize.c (working copy) @@ -0,0 +1,54 @@ +// RUN: %clang -emit-llvm -S -o - %s | FileCheck %s + +// CHECK: br i1 %cmp, label %while.body, label %while.end, !llvm.loop !1 +// CHECK: br i1 %cmp, label %do.body, label %do.end, !llvm.loop !5 +// CHECK: br i1 %cmp, label %for.body, label %for.end, !llvm.loop !6 +// CHECK: br i1 %cmp, label %for.body, label %for.end, !llvm.loop !8 + +// CHECK: !1 = metadata !{metadata !1, metadata !2, metadata !3, metadata !4} +// CHECK: !2 = metadata !{metadata !"llvm.vectorizer.width", i32 4} +// CHECK: !3 = metadata !{metadata !"llvm.vectorizer.unroll", i32 8} +// CHECK: !4 = metadata !{metadata !"llvm.vectorizer.enable", i1 true} +// CHECK: !5 = metadata !{metadata !5, metadata !2, metadata !3} +// CHECK: !6 = metadata !{metadata !6, metadata !7, metadata !4} +// CHECK: !7 = metadata !{metadata !"llvm.vectorizer.unroll", i32 4} +// CHECK: !8 = metadata !{metadata !8, metadata !9} +// CHECK: !9 = metadata !{metadata !"llvm.vectorizer.enable", i1 false} + + +void while_test(int *List, int Length) { + int i = 0; + + #pragma vectorize enable + #pragma vectorize unroll(8) + #pragma vectorize width(4) + while(i < Length) { + List[i] = i*2; + i++; + } +} + +void do_test(int *List, int Length) { + int i = 0; + + #pragma vectorize unroll(8) width(4) + do { + List[i] = i*2; + i++; + } while (i < Length); +} + +void for_test(int *List, int Length) { + #pragma vectorize enable + #pragma vectorize unroll(4) + for(int i = 0; i < Length; i++) { + List[i] = i*2; + } +} + +void disable_test(int *List, int Length) { + #pragma vectorize disable + for(int i = 0; i < Length; i++) { + List[i] = i*2; + } +} Index: test/Parser/pragma-vectorize.c =================================================================== --- test/Parser/pragma-vectorize.c (revision 0) +++ test/Parser/pragma-vectorize.c (working copy) @@ -0,0 +1,50 @@ +// RUN: %clang_cc1 -fsyntax-only -verify %s + +// Note that this puts the expected lines before the directives to work around +// limitations in the -verify mode. + +void test(int *List, int Length) +{ + int i = 0; + + #pragma vectorize enable + #pragma vectorize disable + #pragma vectorize width(4) + #pragma vectorize unroll(8) + while(i < Length) { + List[i] = i; + } + + #pragma vectorize enable width(4) unroll(8) + while(i < Length) { + List[i] = i; + } + +/* expected-error {{expected '('}} */ #pragma vectorize width +/* expected-error {{expected '('}} */ #pragma vectorize unroll + +/* expected-error {{expected ')'}} */ #pragma vectorize width(4 +/* expected-error {{expected ')'}} */ #pragma vectorize unroll(4 + +/* expected-error {{expected positive integer}} */ #pragma vectorize width(0) +/* expected-error {{expected positive integer}} */ #pragma vectorize unroll(0) + +/* expected-error {{expected enable, disable, width, or unroll}} */ #pragma vectorize +/* expected-error {{expected enable, disable, width, or unroll}} */ #pragma vectorize badkeyword +/* expected-error {{expected enable, disable, width, or unroll}} */ #pragma vectorize enable badkeyword +/* expected-warning {{extra tokens at end of '#pragma vectorize' - ignored}} */ #pragma vectorize enable , + + while(i < Length) { + List[i] = i; + } + +/* expected-error {{expected for, while, or do-while loop to immediately follow pragma vectorize}} */ #pragma vectorize enable + int j = Length; + List[0] = List[1]; + + while(j < Length) { + List[j] = j; + } + +/* expected-error {{expected for, while, or do-while loop to immediately follow pragma vectorize}} */ #pragma vectorize enable +}