subject:"\[PATCH\] D83218\: Hand Allocator and IdentifierTable into FormatTokenLexer."

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

2020-07-07 Thread Manuel Klimek via Phabricator via cfe-commits

This revision was automatically updated to reflect the committed changes.
Closed by commit rG8c2a61397607: Hand Allocator and IdentifierTable into 
FormatTokenLexer. (authored by klimek).

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D83218/new/

https://reviews.llvm.org/D83218

Files:
  clang/lib/Format/FormatTokenLexer.cpp
  clang/lib/Format/FormatTokenLexer.h
  clang/lib/Format/TokenAnalyzer.cpp


Index: clang/lib/Format/TokenAnalyzer.cpp
===
--- clang/lib/Format/TokenAnalyzer.cpp
+++ clang/lib/Format/TokenAnalyzer.cpp
@@ -64,11 +64,16 @@
 
 std::pair TokenAnalyzer::process() {
   tooling::Replacements Result;
-  FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
-  Env.getFirstStartColumn(), Style, Encoding);
+  llvm::SpecificBumpPtrAllocator Allocator;
+  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
+   Env.getFirstStartColumn(), Style, Encoding, Allocator,
 
-  UnwrappedLineParser Parser(Style, Tokens.getKeywords(),
- Env.getFirstStartColumn(), Tokens.lex(), *this);
+   IdentTable);
+  ArrayRef Toks(Lex.lex());
+  SmallVector Tokens(Toks.begin(), Toks.end());
+  UnwrappedLineParser Parser(Style, Lex.getKeywords(),
+ Env.getFirstStartColumn(), Tokens, *this);
   Parser.parse();
   assert(UnwrappedLines.rbegin()->empty());
   unsigned Penalty = 0;
@@ -76,14 +81,14 @@
 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
 SmallVector AnnotatedLines;
 
-TokenAnnotator Annotator(Style, Tokens.getKeywords());
+TokenAnnotator Annotator(Style, Lex.getKeywords());
 for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) {
   AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i]));
   Annotator.annotate(*AnnotatedLines.back());
 }
 
 std::pair RunResult =
-analyze(Annotator, AnnotatedLines, Tokens);
+analyze(Annotator, AnnotatedLines, Lex);
 
 LLVM_DEBUG({
   llvm::dbgs() << "Replacements for run " << Run << ":\n";
Index: clang/lib/Format/FormatTokenLexer.h
===
--- clang/lib/Format/FormatTokenLexer.h
+++ clang/lib/Format/FormatTokenLexer.h
@@ -38,7 +38,9 @@
 class FormatTokenLexer {
 public:
   FormatTokenLexer(const SourceManager , FileID ID, unsigned Column,
-   const FormatStyle , encoding::Encoding Encoding);
+   const FormatStyle , encoding::Encoding Encoding,
+   llvm::SpecificBumpPtrAllocator ,
+   IdentifierTable );
 
   ArrayRef lex();
 
@@ -103,10 +105,10 @@
   const SourceManager 
   FileID ID;
   const FormatStyle 
-  IdentifierTable IdentTable;
+  IdentifierTable 
   AdditionalKeywords Keywords;
   encoding::Encoding Encoding;
-  llvm::SpecificBumpPtrAllocator Allocator;
+  llvm::SpecificBumpPtrAllocator 
   // Index (in 'Tokens') of the last token that starts a new line.
   unsigned FirstInLineIndex;
   SmallVector Tokens;
Index: clang/lib/Format/FormatTokenLexer.cpp
===
--- clang/lib/Format/FormatTokenLexer.cpp
+++ clang/lib/Format/FormatTokenLexer.cpp
@@ -22,13 +22,15 @@
 namespace clang {
 namespace format {
 
-FormatTokenLexer::FormatTokenLexer(const SourceManager , FileID ID,
-   unsigned Column, const FormatStyle ,
-   encoding::Encoding Encoding)
+FormatTokenLexer::FormatTokenLexer(
+const SourceManager , FileID ID, unsigned Column,
+const FormatStyle , encoding::Encoding Encoding,
+llvm::SpecificBumpPtrAllocator ,
+IdentifierTable )
 : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
   Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
-  Style(Style), IdentTable(getFormattingLangOpts(Style)),
-  Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
+  Style(Style), IdentTable(IdentTable), Keywords(IdentTable),
+  Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0),
   FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
   MacroBlockEndRegex(Style.MacroBlockEnd) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,


Index: clang/lib/Format/TokenAnalyzer.cpp
===
--- clang/lib/Format/TokenAnalyzer.cpp
+++ clang/lib/Format/TokenAnalyzer.cpp
@@ -64,11 +64,16 @@
 
 std::pair TokenAnalyzer::process() {
   tooling::Replacements Result;
-  FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
-  Env.getFirstStartColumn(), Style, Encoding);
+  llvm::SpecificBumpPtrAllocator Allocator;
+  IdentifierTable

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

2020-07-07 Thread Manuel Klimek via Phabricator via cfe-commits

klimek updated this revision to Diff 275964.
klimek added a comment.

Address review comment.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D83218/new/

https://reviews.llvm.org/D83218

Files:
  clang/lib/Format/FormatTokenLexer.cpp
  clang/lib/Format/FormatTokenLexer.h
  clang/lib/Format/TokenAnalyzer.cpp


Index: clang/lib/Format/TokenAnalyzer.cpp
===
--- clang/lib/Format/TokenAnalyzer.cpp
+++ clang/lib/Format/TokenAnalyzer.cpp
@@ -64,11 +64,16 @@
 
 std::pair TokenAnalyzer::process() {
   tooling::Replacements Result;
-  FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
-  Env.getFirstStartColumn(), Style, Encoding);
+  llvm::SpecificBumpPtrAllocator Allocator;
+  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
+   Env.getFirstStartColumn(), Style, Encoding, Allocator,
 
-  UnwrappedLineParser Parser(Style, Tokens.getKeywords(),
- Env.getFirstStartColumn(), Tokens.lex(), *this);
+   IdentTable);
+  ArrayRef Toks(Lex.lex());
+  SmallVector Tokens(Toks.begin(), Toks.end());
+  UnwrappedLineParser Parser(Style, Lex.getKeywords(),
+ Env.getFirstStartColumn(), Tokens, *this);
   Parser.parse();
   assert(UnwrappedLines.rbegin()->empty());
   unsigned Penalty = 0;
@@ -76,14 +81,14 @@
 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
 SmallVector AnnotatedLines;
 
-TokenAnnotator Annotator(Style, Tokens.getKeywords());
+TokenAnnotator Annotator(Style, Lex.getKeywords());
 for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) {
   AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i]));
   Annotator.annotate(*AnnotatedLines.back());
 }
 
 std::pair RunResult =
-analyze(Annotator, AnnotatedLines, Tokens);
+analyze(Annotator, AnnotatedLines, Lex);
 
 LLVM_DEBUG({
   llvm::dbgs() << "Replacements for run " << Run << ":\n";
Index: clang/lib/Format/FormatTokenLexer.h
===
--- clang/lib/Format/FormatTokenLexer.h
+++ clang/lib/Format/FormatTokenLexer.h
@@ -38,7 +38,9 @@
 class FormatTokenLexer {
 public:
   FormatTokenLexer(const SourceManager , FileID ID, unsigned Column,
-   const FormatStyle , encoding::Encoding Encoding);
+   const FormatStyle , encoding::Encoding Encoding,
+   llvm::SpecificBumpPtrAllocator ,
+   IdentifierTable );
 
   ArrayRef lex();
 
@@ -103,10 +105,10 @@
   const SourceManager 
   FileID ID;
   const FormatStyle 
-  IdentifierTable IdentTable;
+  IdentifierTable 
   AdditionalKeywords Keywords;
   encoding::Encoding Encoding;
-  llvm::SpecificBumpPtrAllocator Allocator;
+  llvm::SpecificBumpPtrAllocator 
   // Index (in 'Tokens') of the last token that starts a new line.
   unsigned FirstInLineIndex;
   SmallVector Tokens;
Index: clang/lib/Format/FormatTokenLexer.cpp
===
--- clang/lib/Format/FormatTokenLexer.cpp
+++ clang/lib/Format/FormatTokenLexer.cpp
@@ -22,13 +22,15 @@
 namespace clang {
 namespace format {
 
-FormatTokenLexer::FormatTokenLexer(const SourceManager , FileID ID,
-   unsigned Column, const FormatStyle ,
-   encoding::Encoding Encoding)
+FormatTokenLexer::FormatTokenLexer(
+const SourceManager , FileID ID, unsigned Column,
+const FormatStyle , encoding::Encoding Encoding,
+llvm::SpecificBumpPtrAllocator ,
+IdentifierTable )
 : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
   Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
-  Style(Style), IdentTable(getFormattingLangOpts(Style)),
-  Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
+  Style(Style), IdentTable(IdentTable), Keywords(IdentTable),
+  Encoding(Encoding), Allocator(Allocator), FirstInLineIndex(0),
   FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
   MacroBlockEndRegex(Style.MacroBlockEnd) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,


Index: clang/lib/Format/TokenAnalyzer.cpp
===
--- clang/lib/Format/TokenAnalyzer.cpp
+++ clang/lib/Format/TokenAnalyzer.cpp
@@ -64,11 +64,16 @@
 
 std::pair TokenAnalyzer::process() {
   tooling::Replacements Result;
-  FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
-  Env.getFirstStartColumn(), Style, Encoding);
+  llvm::SpecificBumpPtrAllocator Allocator;
+  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  FormatTokenLexer Lex(Env.getSourceManager(),

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

2020-07-07 Thread Sam McCall via Phabricator via cfe-commits

sammccall accepted this revision.
sammccall added inline comments.
This revision is now accepted and ready to land.



Comment at: clang/lib/Format/FormatTokenLexer.h:121
   llvm::Regex MacroBlockEndRegex;
+  llvm::SpecificBumpPtrAllocator 
 

this seems a little odd, like it fit better where it was, up to you.


Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D83218/new/

https://reviews.llvm.org/D83218



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

2020-07-06 Thread Manuel Klimek via Phabricator via cfe-commits

klimek created this revision.
klimek added a reviewer: sammccall.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.

This allows us to share the allocator in the future so we can create tokens 
while parsing.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D83218

Files:
  clang/lib/Format/FormatTokenLexer.cpp
  clang/lib/Format/FormatTokenLexer.h
  clang/lib/Format/TokenAnalyzer.cpp

Index: clang/lib/Format/TokenAnalyzer.cpp
===
--- clang/lib/Format/TokenAnalyzer.cpp
+++ clang/lib/Format/TokenAnalyzer.cpp
@@ -64,11 +64,16 @@
 
 std::pair TokenAnalyzer::process() {
   tooling::Replacements Result;
-  FormatTokenLexer Tokens(Env.getSourceManager(), Env.getFileID(),
-  Env.getFirstStartColumn(), Style, Encoding);
+  llvm::SpecificBumpPtrAllocator Allocator;
+  IdentifierTable IdentTable(getFormattingLangOpts(Style));
+  FormatTokenLexer Lex(Env.getSourceManager(), Env.getFileID(),
+   Env.getFirstStartColumn(), Style, Encoding, Allocator,
 
-  UnwrappedLineParser Parser(Style, Tokens.getKeywords(),
- Env.getFirstStartColumn(), Tokens.lex(), *this);
+   IdentTable);
+  ArrayRef Toks(Lex.lex());
+  SmallVector Tokens(Toks.begin(), Toks.end());
+  UnwrappedLineParser Parser(Style, Lex.getKeywords(),
+ Env.getFirstStartColumn(), Tokens, *this);
   Parser.parse();
   assert(UnwrappedLines.rbegin()->empty());
   unsigned Penalty = 0;
@@ -76,14 +81,14 @@
 LLVM_DEBUG(llvm::dbgs() << "Run " << Run << "...\n");
 SmallVector AnnotatedLines;
 
-TokenAnnotator Annotator(Style, Tokens.getKeywords());
+TokenAnnotator Annotator(Style, Lex.getKeywords());
 for (unsigned i = 0, e = UnwrappedLines[Run].size(); i != e; ++i) {
   AnnotatedLines.push_back(new AnnotatedLine(UnwrappedLines[Run][i]));
   Annotator.annotate(*AnnotatedLines.back());
 }
 
 std::pair RunResult =
-analyze(Annotator, AnnotatedLines, Tokens);
+analyze(Annotator, AnnotatedLines, Lex);
 
 LLVM_DEBUG({
   llvm::dbgs() << "Replacements for run " << Run << ":\n";
Index: clang/lib/Format/FormatTokenLexer.h
===
--- clang/lib/Format/FormatTokenLexer.h
+++ clang/lib/Format/FormatTokenLexer.h
@@ -38,7 +38,9 @@
 class FormatTokenLexer {
 public:
   FormatTokenLexer(const SourceManager , FileID ID, unsigned Column,
-   const FormatStyle , encoding::Encoding Encoding);
+   const FormatStyle , encoding::Encoding Encoding,
+   llvm::SpecificBumpPtrAllocator ,
+   IdentifierTable );
 
   ArrayRef lex();
 
@@ -103,10 +105,9 @@
   const SourceManager 
   FileID ID;
   const FormatStyle 
-  IdentifierTable IdentTable;
+  IdentifierTable 
   AdditionalKeywords Keywords;
   encoding::Encoding Encoding;
-  llvm::SpecificBumpPtrAllocator Allocator;
   // Index (in 'Tokens') of the last token that starts a new line.
   unsigned FirstInLineIndex;
   SmallVector Tokens;
@@ -117,6 +118,7 @@
 
   llvm::Regex MacroBlockBeginRegex;
   llvm::Regex MacroBlockEndRegex;
+  llvm::SpecificBumpPtrAllocator 
 
   // Targets that may appear inside a C# attribute.
   static const llvm::StringSet<> CSharpAttributeTargets;
Index: clang/lib/Format/FormatTokenLexer.cpp
===
--- clang/lib/Format/FormatTokenLexer.cpp
+++ clang/lib/Format/FormatTokenLexer.cpp
@@ -22,15 +22,17 @@
 namespace clang {
 namespace format {
 
-FormatTokenLexer::FormatTokenLexer(const SourceManager , FileID ID,
-   unsigned Column, const FormatStyle ,
-   encoding::Encoding Encoding)
+FormatTokenLexer::FormatTokenLexer(
+const SourceManager , FileID ID, unsigned Column,
+const FormatStyle , encoding::Encoding Encoding,
+llvm::SpecificBumpPtrAllocator ,
+IdentifierTable )
 : FormatTok(nullptr), IsFirstToken(true), StateStack({LexerState::NORMAL}),
   Column(Column), TrailingWhitespace(0), SourceMgr(SourceMgr), ID(ID),
-  Style(Style), IdentTable(getFormattingLangOpts(Style)),
-  Keywords(IdentTable), Encoding(Encoding), FirstInLineIndex(0),
-  FormattingDisabled(false), MacroBlockBeginRegex(Style.MacroBlockBegin),
-  MacroBlockEndRegex(Style.MacroBlockEnd) {
+  Style(Style), IdentTable(IdentTable), Keywords(IdentTable),
+  Encoding(Encoding), FirstInLineIndex(0), FormattingDisabled(false),
+  MacroBlockBeginRegex(Style.MacroBlockBegin),
+  MacroBlockEndRegex(Style.MacroBlockEnd), Allocator(Allocator) {
   Lex.reset(new Lexer(ID, SourceMgr.getBuffer(ID), SourceMgr,
   getFormattingLangOpts(Style)));
   Lex->SetKeepWhitespaceMode(true);
___
cfe-commits mailing list

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

[PATCH] D83218: Hand Allocator and IdentifierTable into FormatTokenLexer.

4 matches

Site Navigation

Mail list logo

Footer information