Revision: 18026
Author: [email protected]
Date: Fri Nov 22 15:56:11 2013 UTC
Log: Experimentar scanner: Unify the API to Scanner API some more.
This doesn't yet contain the unicode line terminator fix.
Also added a test file.
[email protected]
BUG=
Review URL: https://codereview.chromium.org/83693002
http://code.google.com/p/v8/source/detail?r=18026
Added:
/branches/experimental/parser/test/lexer/cornercases/multiline-and-weird-html-comment.js
Modified:
/branches/experimental/parser/src/lexer/experimental-scanner.h
/branches/experimental/parser/src/lexer/lexer_py.re
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
=======================================
--- /dev/null
+++
/branches/experimental/parser/test/lexer/cornercases/multiline-and-weird-html-comment.js
Fri Nov 22 15:56:11 2013 UTC
@@ -0,0 +1,32 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// The "-->" at the beginning is a weird SpiderMonkey compatibility hack.
+
+/* here is a multiline comment with
+a newline*/ --> this is now inside a comment too!
+but --> this is not
=======================================
--- /branches/experimental/parser/src/lexer/experimental-scanner.h Fri Nov
22 14:54:41 2013 UTC
+++ /branches/experimental/parser/src/lexer/experimental-scanner.h Fri Nov
22 15:56:11 2013 UTC
@@ -84,26 +84,55 @@
~ExperimentalScanner();
+ // Returns the next token and advances input.
Token::Value Next() {
+ has_line_terminator_before_next_ = false;
current_ = next_;
Scan(); // will fill in next_.
return current_.token;
}
+ // Returns the current token again.
+ Token::Value current_token() { return current_.token; }
+
+ // Returns the location information for the current token
+ // (the token last returned by Next()).
Location location() {
return Location(current_.beg_pos, current_.end_pos);
}
- void SetHarmonyNumericLiterals(bool numeric_literals) {
- harmony_numeric_literals_ = numeric_literals;
+ // One token look-ahead (past the token returned by Next()).
+ Token::Value peek() const { return next_.token; }
+
+ Location peek_location() const { return next_.location; }
+
+ UnicodeCache* unicode_cache() { return unicode_cache_; }
+
+ bool HarmonyScoping() const {
+ return harmony_scoping_;
+ }
+ void SetHarmonyScoping(bool scoping) {
+ harmony_scoping_ = scoping;
+ }
+ bool HarmonyModules() const {
+ return harmony_modules_;
}
-
void SetHarmonyModules(bool modules) {
harmony_modules_ = modules;
}
+ bool HarmonyNumericLiterals() const {
+ return harmony_numeric_literals_;
+ }
+ void SetHarmonyNumericLiterals(bool numeric_literals) {
+ harmony_numeric_literals_ = numeric_literals;
+ }
- void SetHarmonyScoping(bool scoping) {
- harmony_scoping_ = scoping;
+ // Returns true if there was a line terminator before the peek'ed token,
+ // possibly inside a multi-line comment.
+ bool HasAnyLineTerminatorBeforeNext() const {
+ return has_line_terminator_before_next_;
+ // FIXME: do we need to distinguish between newlines inside and outside
+ // multiline comments? Atm doesn't look like we need to.
}
private:
@@ -127,7 +156,7 @@
YYCTYPE* start_;
YYCTYPE* cursor_;
YYCTYPE* marker_;
- bool just_seen_line_terminator_;
+ bool has_line_terminator_before_next_;
YYCTYPE yych;
@@ -147,7 +176,7 @@
YYCTYPE* source_end,
Isolate* isolate)
: unicode_cache_(isolate->unicode_cache()),
- just_seen_line_terminator_(true),
+ has_line_terminator_before_next_(true),
harmony_numeric_literals_(false),
harmony_modules_(false),
harmony_scoping_(false) {
=======================================
--- /branches/experimental/parser/src/lexer/lexer_py.re Fri Nov 22 13:16:25
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer_py.re Fri Nov 22 15:56:11
2013 UTC
@@ -79,7 +79,7 @@
"-->" <{
- if (!just_seen_line_terminator_) {
+ if (!has_line_terminator_before_next_) {
BACKWARD(1);
PUSH_TOKEN(Token::DEC);
}
=======================================
---
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Fri Nov 22 14:54:41 2013 UTC
+++
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Fri Nov 22 15:56:11 2013 UTC
@@ -193,7 +193,6 @@
next_.beg_pos = start_ - buffer_; \
next_.end_pos = cursor_ - buffer_; \
start_ = cursor_; \
- just_seen_line_terminator_ = false; \
}
#define PUSH_TOKEN(T) { \
@@ -216,7 +215,7 @@
#define PUSH_LINE_TERMINATOR(s) { \
start_ = cursor_; \
- just_seen_line_terminator_ = true; \
+ has_line_terminator_before_next_ = true; \
}
#define FORWARD() { \
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
