Author: ibiryukov Date: Tue Aug 6 10:07:58 2019 New Revision: 368062 URL: http://llvm.org/viewvc/llvm-project?rev=368062&view=rev Log: [Syntax] Do not add a node for 'eof' into the tree
Summary: While useful as a sentinel value when iterating over tokens, having 'eof' in the tree, seems to do more harm than good. Reviewers: sammccall Reviewed By: sammccall Subscribers: javed.absar, kristof.beyls, cfe-commits Tags: #clang Differential Revision: https://reviews.llvm.org/D64576 Modified: cfe/trunk/lib/Tooling/Syntax/BuildTree.cpp cfe/trunk/unittests/Tooling/Syntax/TreeTest.cpp Modified: cfe/trunk/lib/Tooling/Syntax/BuildTree.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Syntax/BuildTree.cpp?rev=368062&r1=368061&r2=368062&view=diff ============================================================================== --- cfe/trunk/lib/Tooling/Syntax/BuildTree.cpp (original) +++ cfe/trunk/lib/Tooling/Syntax/BuildTree.cpp Tue Aug 6 10:07:58 2019 @@ -58,8 +58,11 @@ public: /// Finish building the tree and consume the root node. syntax::TranslationUnit *finalize() && { auto Tokens = Arena.tokenBuffer().expandedTokens(); + assert(!Tokens.empty()); + assert(Tokens.back().kind() == tok::eof); + // Build the root of the tree, consuming all the children. - Pending.foldChildren(Tokens, + Pending.foldChildren(Tokens.drop_back(), new (Arena.allocator()) syntax::TranslationUnit); return cast<syntax::TranslationUnit>(std::move(Pending).finalize()); @@ -96,10 +99,11 @@ private: /// Ensures that added nodes properly nest and cover the whole token stream. struct Forest { Forest(syntax::Arena &A) { - // FIXME: do not add 'eof' to the tree. - + assert(!A.tokenBuffer().expandedTokens().empty()); + assert(A.tokenBuffer().expandedTokens().back().kind() == tok::eof); // Create all leaf nodes. - for (auto &T : A.tokenBuffer().expandedTokens()) + // Note that we do not have 'eof' in the tree. + for (auto &T : A.tokenBuffer().expandedTokens().drop_back()) Trees.insert(Trees.end(), {&T, NodeAndRole{new (A.allocator()) syntax::Leaf(&T)}}); } Modified: cfe/trunk/unittests/Tooling/Syntax/TreeTest.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/unittests/Tooling/Syntax/TreeTest.cpp?rev=368062&r1=368061&r2=368062&view=diff ============================================================================== --- cfe/trunk/unittests/Tooling/Syntax/TreeTest.cpp (original) +++ cfe/trunk/unittests/Tooling/Syntax/TreeTest.cpp Tue Aug 6 10:07:58 2019 @@ -138,15 +138,14 @@ void foo() {} | `-CompoundStatement | |-2: { | `-3: } -|-TopLevelDeclaration -| |-void -| |-foo -| |-( -| |-) -| `-CompoundStatement -| |-2: { -| `-3: } -`-<eof> +`-TopLevelDeclaration + |-void + |-foo + |-( + |-) + `-CompoundStatement + |-2: { + `-3: } )txt"}, }; _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits