Hi djasper, klimek,
FormatTokenLexer is here, FormatTokenBuffer is on the way. This will allow to
re-parse unwrapped lines when needed.
http://llvm-reviews.chandlerc.com/D186
Files:
lib/Format/Format.cpp
lib/Format/UnwrappedLineParser.cpp
lib/Format/UnwrappedLineParser.h
Index: lib/Format/Format.cpp
===================================================================
--- lib/Format/Format.cpp
+++ lib/Format/Format.cpp
@@ -762,7 +762,8 @@
}
tooling::Replacements format() {
- UnwrappedLineParser Parser(Style, Lex, SourceMgr, *this);
+ FormatTokenLexer Tokens(Lex, SourceMgr);
+ UnwrappedLineParser Parser(Style, Tokens, *this);
StructuralError = Parser.parse();
for (std::vector<UnwrappedLine>::iterator I = UnwrappedLines.begin(),
E = UnwrappedLines.end();
Index: lib/Format/UnwrappedLineParser.cpp
===================================================================
--- lib/Format/UnwrappedLineParser.cpp
+++ lib/Format/UnwrappedLineParser.cpp
@@ -22,27 +22,23 @@
namespace clang {
namespace format {
-UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style, Lexer &Lex,
- SourceManager &SourceMgr,
+UnwrappedLineParser::UnwrappedLineParser(const FormatStyle &Style,
+ FormatTokenSource &Tokens,
UnwrappedLineConsumer &Callback)
- : GreaterStashed(false),
- Style(Style),
- Lex(Lex),
- SourceMgr(SourceMgr),
- IdentTable(Lex.getLangOpts()),
+ : Style(Style),
+ Tokens(Tokens),
Callback(Callback) {
- Lex.SetKeepWhitespaceMode(true);
}
bool UnwrappedLineParser::parse() {
- parseToken();
+ Tokens.parseToken();
return parseLevel();
}
bool UnwrappedLineParser::parseLevel() {
bool Error = false;
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (Tokens.tok().Tok.getKind()) {
case tok::hash:
parsePPDirective();
break;
@@ -60,12 +56,12 @@
parseStatement();
break;
}
- } while (!eof());
+ } while (!Tokens.eof());
return Error;
}
bool UnwrappedLineParser::parseBlock(unsigned AddLevels) {
- assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
+ assert(Tokens.tok().Tok.is(tok::l_brace) && "'{' expected");
nextToken();
addUnwrappedLine();
@@ -75,43 +71,43 @@
Line.Level -= AddLevels;
// FIXME: Add error handling.
- if (!FormatTok.Tok.is(tok::r_brace))
+ if (!Tokens.tok().Tok.is(tok::r_brace))
return true;
nextToken();
- if (FormatTok.Tok.is(tok::semi))
+ if (Tokens.tok().Tok.is(tok::semi))
nextToken();
return false;
}
void UnwrappedLineParser::parsePPDirective() {
- while (!eof()) {
+ while (!Tokens.eof()) {
nextToken();
- if (FormatTok.NewlinesBefore > 0) {
+ if (Tokens.tok().NewlinesBefore > 0) {
addUnwrappedLine();
return;
}
}
}
void UnwrappedLineParser::parseComment() {
- while (!eof()) {
+ while (!Tokens.eof()) {
nextToken();
- if (FormatTok.NewlinesBefore > 0) {
+ if (Tokens.tok().NewlinesBefore > 0) {
addUnwrappedLine();
return;
}
}
}
void UnwrappedLineParser::parseStatement() {
// Consume leading line comments, e.g. for branches without compounds.
- while (FormatTok.Tok.is(tok::comment)) {
+ while (Tokens.tok().Tok.is(tok::comment)) {
nextToken();
addUnwrappedLine();
}
- switch (FormatTok.Tok.getKind()) {
+ switch (Tokens.tok().Tok.getKind()) {
case tok::kw_namespace:
parseNamespace();
return;
@@ -146,7 +142,7 @@
int TokenNumber = 0;
do {
++TokenNumber;
- switch (FormatTok.Tok.getKind()) {
+ switch (Tokens.tok().Tok.getKind()) {
case tok::kw_enum:
parseEnum();
return;
@@ -163,23 +159,23 @@
return;
case tok::identifier:
nextToken();
- if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
+ if (TokenNumber == 1 && Tokens.tok().Tok.is(tok::colon)) {
parseLabel();
return;
}
break;
default:
nextToken();
break;
}
- } while (!eof());
+ } while (!Tokens.eof());
}
void UnwrappedLineParser::parseParens() {
- assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
+ assert(Tokens.tok().Tok.is(tok::l_paren) && "'(' expected.");
nextToken();
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (Tokens.tok().Tok.getKind()) {
case tok::l_paren:
parseParens();
break;
@@ -190,29 +186,29 @@
nextToken();
break;
}
- } while (!eof());
+ } while (!Tokens.eof());
}
void UnwrappedLineParser::parseIfThenElse() {
- assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
+ assert(Tokens.tok().Tok.is(tok::kw_if) && "'if' expected");
nextToken();
parseParens();
bool NeedsUnwrappedLine = false;
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock();
NeedsUnwrappedLine = true;
} else {
addUnwrappedLine();
++Line.Level;
parseStatement();
--Line.Level;
}
- if (FormatTok.Tok.is(tok::kw_else)) {
+ if (Tokens.tok().Tok.is(tok::kw_else)) {
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock();
addUnwrappedLine();
- } else if (FormatTok.Tok.is(tok::kw_if)) {
+ } else if (Tokens.tok().Tok.is(tok::kw_if)) {
parseIfThenElse();
} else {
addUnwrappedLine();
@@ -226,23 +222,23 @@
}
void UnwrappedLineParser::parseNamespace() {
- assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
+ assert(Tokens.tok().Tok.is(tok::kw_namespace) && "'namespace' expected");
nextToken();
- if (FormatTok.Tok.is(tok::identifier))
+ if (Tokens.tok().Tok.is(tok::identifier))
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock(0);
addUnwrappedLine();
}
// FIXME: Add error handling.
}
void UnwrappedLineParser::parseForOrWhileLoop() {
- assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
+ assert((Tokens.tok().Tok.is(tok::kw_for) || Tokens.tok().Tok.is(tok::kw_while)) &&
"'for' or 'while' expected");
nextToken();
parseParens();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock();
addUnwrappedLine();
} else {
@@ -254,9 +250,9 @@
}
void UnwrappedLineParser::parseDoWhile() {
- assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
+ assert(Tokens.tok().Tok.is(tok::kw_do) && "'do' expected");
nextToken();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock();
} else {
addUnwrappedLine();
@@ -266,7 +262,7 @@
}
// FIXME: Add error handling.
- if (!FormatTok.Tok.is(tok::kw_while)) {
+ if (!Tokens.tok().Tok.is(tok::kw_while)) {
addUnwrappedLine();
return;
}
@@ -277,32 +273,32 @@
void UnwrappedLineParser::parseLabel() {
// FIXME: remove all asserts.
- assert(FormatTok.Tok.is(tok::colon) && "':' expected");
+ assert(Tokens.tok().Tok.is(tok::colon) && "':' expected");
nextToken();
unsigned OldLineLevel = Line.Level;
if (Line.Level > 0)
--Line.Level;
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock();
}
addUnwrappedLine();
Line.Level = OldLineLevel;
}
void UnwrappedLineParser::parseCaseLabel() {
- assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
+ assert(Tokens.tok().Tok.is(tok::kw_case) && "'case' expected");
// FIXME: fix handling of complex expressions here.
do {
nextToken();
- } while (!eof() && !FormatTok.Tok.is(tok::colon));
+ } while (!Tokens.eof() && !Tokens.tok().Tok.is(tok::colon));
parseLabel();
}
void UnwrappedLineParser::parseSwitch() {
- assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
+ assert(Tokens.tok().Tok.is(tok::kw_switch) && "'switch' expected");
nextToken();
parseParens();
- if (FormatTok.Tok.is(tok::l_brace)) {
+ if (Tokens.tok().Tok.is(tok::l_brace)) {
parseBlock(Style.IndentCaseLabels ? 2 : 1);
addUnwrappedLine();
} else {
@@ -322,7 +318,7 @@
void UnwrappedLineParser::parseEnum() {
bool HasContents = false;
do {
- switch (FormatTok.Tok.getKind()) {
+ switch (Tokens.tok().Tok.getKind()) {
case tok::l_brace:
nextToken();
addUnwrappedLine();
@@ -350,31 +346,35 @@
nextToken();
break;
}
- } while (!eof());
+ } while (!Tokens.eof());
}
void UnwrappedLineParser::addUnwrappedLine() {
// Consume trailing comments.
- while (!eof() && FormatTok.NewlinesBefore == 0 &&
- FormatTok.Tok.is(tok::comment)) {
+ while (!Tokens.eof() && Tokens.tok().NewlinesBefore == 0 &&
+ Tokens.tok().Tok.is(tok::comment)) {
nextToken();
}
Callback.consumeUnwrappedLine(Line);
Line.Tokens.clear();
}
-bool UnwrappedLineParser::eof() const {
- return FormatTok.Tok.is(tok::eof);
-}
-
void UnwrappedLineParser::nextToken() {
- if (eof())
+ if (Tokens.eof())
return;
- Line.Tokens.push_back(FormatTok);
- parseToken();
+ Line.Tokens.push_back(Tokens.tok());
+ Tokens.parseToken();
+}
+
+FormatTokenLexer::FormatTokenLexer(Lexer &Lex, SourceManager &SourceMgr)
+ : GreaterStashed(false),
+ Lex(Lex),
+ SourceMgr(SourceMgr),
+ IdentTable(Lex.getLangOpts()) {
+ Lex.SetKeepWhitespaceMode(true);
}
-void UnwrappedLineParser::parseToken() {
+void FormatTokenLexer::parseToken() {
if (GreaterStashed) {
FormatTok.NewlinesBefore = 0;
FormatTok.WhiteSpaceStart = FormatTok.Tok.getLocation().getLocWithOffset(1);
@@ -408,10 +408,9 @@
}
}
-StringRef UnwrappedLineParser::tokenText() {
- StringRef Data(SourceMgr.getCharacterData(FormatTok.Tok.getLocation()),
- FormatTok.Tok.getLength());
- return Data;
+StringRef FormatTokenLexer::tokenText() {
+ return StringRef(SourceMgr.getCharacterData(FormatTok.Tok.getLocation()),
+ FormatTok.Tok.getLength());
}
} // end namespace format
Index: lib/Format/UnwrappedLineParser.h
===================================================================
--- lib/Format/UnwrappedLineParser.h
+++ lib/Format/UnwrappedLineParser.h
@@ -77,10 +77,36 @@
virtual void consumeUnwrappedLine(const UnwrappedLine &Line) = 0;
};
+class FormatTokenSource {
+protected:
+ FormatToken FormatTok;
+public:
+ const FormatToken &tok() const {
+ return FormatTok;
+ }
+ bool eof() const {
+ return FormatTok.Tok.is(tok::eof);
+ }
+
+ virtual void parseToken() = 0;
+};
+
+class FormatTokenLexer : public FormatTokenSource {
+ bool GreaterStashed;
+ Lexer &Lex;
+ SourceManager &SourceMgr;
+ IdentifierTable IdentTable;
+
+ /// Returns the text of \c FormatTok.
+ StringRef tokenText();
+public:
+ FormatTokenLexer(Lexer &Lex, SourceManager &SourceMgr);
+ virtual void parseToken();
+};
+
class UnwrappedLineParser {
public:
- UnwrappedLineParser(const FormatStyle &Style, Lexer &Lex,
- SourceManager &SourceMgr,
+ UnwrappedLineParser(const FormatStyle &Style, FormatTokenSource &Tokens,
UnwrappedLineConsumer &Callback);
/// Returns true in case of a structural error.
@@ -103,21 +129,12 @@
void parseAccessSpecifier();
void parseEnum();
void addUnwrappedLine();
- bool eof() const;
void nextToken();
- void parseToken();
-
- /// Returns the text of \c FormatTok.
- StringRef tokenText();
UnwrappedLine Line;
- FormatToken FormatTok;
- bool GreaterStashed;
const FormatStyle &Style;
- Lexer &Lex;
- SourceManager &SourceMgr;
- IdentifierTable IdentTable;
+ FormatTokenSource &Tokens;
UnwrappedLineConsumer &Callback;
};
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.cs.uiuc.edu/mailman/listinfo/cfe-commits
