Reviewers: dcarney,
Message:
Committed patchset #1 manually as r18023.
Description:
Experimental parser: continue unifying the API with Scanner.
This CL adds storing next and current tokens separately and the Location
struct.
Also, kills additional layer of Next() in lexer-shell, to make it easier
to see that the comparsion is fair.
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=18023
Please review this at https://codereview.chromium.org/83633002/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+51, -17 lines):
M src/lexer/experimental-scanner.h
M src/lexer/lexer-shell.cc
M tools/lexer_generator/code_generator.jinja
Index: src/lexer/experimental-scanner.h
diff --git a/src/lexer/experimental-scanner.h
b/src/lexer/experimental-scanner.h
index
2f70e1c39679268afaec018fdbded0da57c2d188..b9ce4826e4d54c123e570630ea9b51fa93e3ac75
100644
--- a/src/lexer/experimental-scanner.h
+++ b/src/lexer/experimental-scanner.h
@@ -63,6 +63,20 @@ class UnicodeCache;
template<typename YYCTYPE>
class ExperimentalScanner {
public:
+ struct Location {
+ Location(int b, int e) : beg_pos(b), end_pos(e) { }
+ Location() : beg_pos(0), end_pos(0) { }
+
+ bool IsValid() const {
+ return beg_pos >= 0 && end_pos >= beg_pos;
+ }
+
+ static Location invalid() { return Location(-1, -1); }
+
+ int beg_pos;
+ int end_pos;
+ };
+
explicit ExperimentalScanner(
YYCTYPE* source,
YYCTYPE* source_end,
@@ -70,7 +84,15 @@ class ExperimentalScanner {
~ExperimentalScanner();
- Token::Value Next(int* beg_pos, int* end_pos);
+ Token::Value Next() {
+ current_ = next_;
+ Scan(); // will fill in next_.
+ return current_.token;
+ }
+
+ Location location() {
+ return Location(current_.beg_pos, current_.end_pos);
+ }
void SetHarmonyNumericLiterals(bool numeric_literals) {
harmony_numeric_literals_ = numeric_literals;
@@ -85,6 +107,15 @@ class ExperimentalScanner {
}
private:
+ struct TokenDesc {
+ Token::Value token;
+ int beg_pos;
+ int end_pos;
+ LiteralBuffer* literal_chars;
+ };
+
+ void Scan();
+
bool ValidIdentifierStart();
bool ValidIdentifierPart();
uc32 ScanHexNumber(int length);
@@ -100,6 +131,9 @@ class ExperimentalScanner {
YYCTYPE yych;
+ TokenDesc current_; // desc for current token (as returned by Next())
+ TokenDesc next_; // desc for next token (one token look-ahead)
+
bool harmony_numeric_literals_;
bool harmony_modules_;
bool harmony_scoping_;
@@ -122,6 +156,7 @@ ExperimentalScanner<YYCTYPE>::ExperimentalScanner(
start_ = buffer_;
cursor_ = buffer_;
marker_ = buffer_;
+ Scan();
}
Index: src/lexer/lexer-shell.cc
diff --git a/src/lexer/lexer-shell.cc b/src/lexer/lexer-shell.cc
index
6c72590e2745dfd728bb1b6ad4beba74192d0527..7b05e40df8e50fea0680ca965bce81a4d3f8cf31
100644
--- a/src/lexer/lexer-shell.cc
+++ b/src/lexer/lexer-shell.cc
@@ -156,16 +156,10 @@ class BaselineScanner {
delete[] source_;
}
- Token::Value Next(int* beg_pos, int* end_pos) {
- Token::Value res = scanner_->Next();
- *beg_pos = scanner_->location().beg_pos;
- *end_pos = scanner_->location().end_pos;
- return res;
- }
+ Scanner* scanner_;
private:
UnicodeCache* unicode_cache_;
- Scanner* scanner_;
const byte* source_;
BufferedUtf16CharacterStream* stream_;
};
@@ -205,7 +199,9 @@ TimeDelta RunBaselineScanner(const char* fname,
Token::Value token;
int beg, end;
do {
- token = scanner.Next(&beg, &end);
+ token = scanner.scanner_->Next();
+ beg = scanner.scanner_->location().beg_pos;
+ end = scanner.scanner_->location().end_pos;
if (dump_tokens) {
tokens->push_back(TokenWithLocation(token, beg, end));
}
@@ -227,17 +223,19 @@ TimeDelta RunExperimentalScanner(const char* fname,
YYCTYPE* buffer = reinterpret_cast<YYCTYPE*>(
ReadFile(fname, &buffer_end, repeat, encoding == UTF8TO16));
+ timer.Start();
ExperimentalScanner<YYCTYPE> scanner(
buffer, reinterpret_cast<YYCTYPE*>(buffer_end), isolate);
scanner.SetHarmonyNumericLiterals(harmony_settings.numeric_literals);
scanner.SetHarmonyModules(harmony_settings.modules);
scanner.SetHarmonyScoping(harmony_settings.scoping);
- timer.Start();
Token::Value token;
int beg, end;
do {
- token = scanner.Next(&beg, &end);
+ token = scanner.Next();
+ beg = scanner.location().beg_pos;
+ end = scanner.location().end_pos;
if (dump_tokens) {
tokens->push_back(TokenWithLocation(token, beg, end));
}
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
e131b7cd1393289f3caa65d78b24f307cec0f88e..00cbf9b1fffbee8403850b0a03a4b2ea19f35b5c
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -190,21 +190,23 @@
{%- endmacro %}
#define PREPARE_PUSH_TOKEN() { \
- *beg_pos_to_return = start_ - buffer_; \
- *end_pos_to_return = cursor_ - buffer_; \
+ next_.beg_pos = start_ - buffer_; \
+ next_.end_pos = cursor_ - buffer_; \
start_ = cursor_; \
just_seen_line_terminator_ = false; \
}
#define PUSH_TOKEN(T) { \
PREPARE_PUSH_TOKEN(); \
- return T; \
+ next_.token = T; \
+ return; \
}
#define PUSH_TOKEN_AND_GO_FORWARD(T) {\
PREPARE_PUSH_TOKEN(); \
FORWARD(); \
- return T; \
+ next_.token = T; \
+ return; \
}
#define PUSH_EOS() { \
@@ -235,8 +237,7 @@
namespace v8 {
namespace internal {
template<>
-Token::Value ExperimentalScanner<{{char_type}}>::Next(int*
beg_pos_to_return,
- int*
end_pos_to_return) {
+void ExperimentalScanner<{{char_type}}>::Scan() {
// Setup environment.
{{char_type}} primary_char;
if (cursor_ >= buffer_end_) primary_char = 0;
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
