Revision: 18023
Author: [email protected]
Date: Fri Nov 22 14:54:41 2013 UTC
Log: Experimental parser: continue unifying the API with Scanner.
This CL adds storing next and current tokens separately and the Location
struct.
Also, kills additional layer of Next() in lexer-shell, to make it easier
to see that the comparsion is fair.
[email protected]
BUG=
Review URL: https://codereview.chromium.org/83633002
http://code.google.com/p/v8/source/detail?r=18023
Modified:
/branches/experimental/parser/src/lexer/experimental-scanner.h
/branches/experimental/parser/src/lexer/lexer-shell.cc
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
=======================================
--- /branches/experimental/parser/src/lexer/experimental-scanner.h Fri Nov
22 12:59:16 2013 UTC
+++ /branches/experimental/parser/src/lexer/experimental-scanner.h Fri Nov
22 14:54:41 2013 UTC
@@ -63,6 +63,20 @@
template<typename YYCTYPE>
class ExperimentalScanner {
public:
+ struct Location {
+ Location(int b, int e) : beg_pos(b), end_pos(e) { }
+ Location() : beg_pos(0), end_pos(0) { }
+
+ bool IsValid() const {
+ return beg_pos >= 0 && end_pos >= beg_pos;
+ }
+
+ static Location invalid() { return Location(-1, -1); }
+
+ int beg_pos;
+ int end_pos;
+ };
+
explicit ExperimentalScanner(
YYCTYPE* source,
YYCTYPE* source_end,
@@ -70,7 +84,15 @@
~ExperimentalScanner();
- Token::Value Next(int* beg_pos, int* end_pos);
+ Token::Value Next() {
+ current_ = next_;
+ Scan(); // will fill in next_.
+ return current_.token;
+ }
+
+ Location location() {
+ return Location(current_.beg_pos, current_.end_pos);
+ }
void SetHarmonyNumericLiterals(bool numeric_literals) {
harmony_numeric_literals_ = numeric_literals;
@@ -85,6 +107,15 @@
}
private:
+ struct TokenDesc {
+ Token::Value token;
+ int beg_pos;
+ int end_pos;
+ LiteralBuffer* literal_chars;
+ };
+
+ void Scan();
+
bool ValidIdentifierStart();
bool ValidIdentifierPart();
uc32 ScanHexNumber(int length);
@@ -100,6 +131,9 @@
YYCTYPE yych;
+ TokenDesc current_; // desc for current token (as returned by Next())
+ TokenDesc next_; // desc for next token (one token look-ahead)
+
bool harmony_numeric_literals_;
bool harmony_modules_;
bool harmony_scoping_;
@@ -122,6 +156,7 @@
start_ = buffer_;
cursor_ = buffer_;
marker_ = buffer_;
+ Scan();
}
=======================================
--- /branches/experimental/parser/src/lexer/lexer-shell.cc Fri Nov 22
14:18:15 2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer-shell.cc Fri Nov 22
14:54:41 2013 UTC
@@ -156,16 +156,10 @@
delete[] source_;
}
- Token::Value Next(int* beg_pos, int* end_pos) {
- Token::Value res = scanner_->Next();
- *beg_pos = scanner_->location().beg_pos;
- *end_pos = scanner_->location().end_pos;
- return res;
- }
+ Scanner* scanner_;
private:
UnicodeCache* unicode_cache_;
- Scanner* scanner_;
const byte* source_;
BufferedUtf16CharacterStream* stream_;
};
@@ -205,7 +199,9 @@
Token::Value token;
int beg, end;
do {
- token = scanner.Next(&beg, &end);
+ token = scanner.scanner_->Next();
+ beg = scanner.scanner_->location().beg_pos;
+ end = scanner.scanner_->location().end_pos;
if (dump_tokens) {
tokens->push_back(TokenWithLocation(token, beg, end));
}
@@ -227,17 +223,19 @@
YYCTYPE* buffer = reinterpret_cast<YYCTYPE*>(
ReadFile(fname, &buffer_end, repeat, encoding == UTF8TO16));
+ timer.Start();
ExperimentalScanner<YYCTYPE> scanner(
buffer, reinterpret_cast<YYCTYPE*>(buffer_end), isolate);
scanner.SetHarmonyNumericLiterals(harmony_settings.numeric_literals);
scanner.SetHarmonyModules(harmony_settings.modules);
scanner.SetHarmonyScoping(harmony_settings.scoping);
- timer.Start();
Token::Value token;
int beg, end;
do {
- token = scanner.Next(&beg, &end);
+ token = scanner.Next();
+ beg = scanner.location().beg_pos;
+ end = scanner.location().end_pos;
if (dump_tokens) {
tokens->push_back(TokenWithLocation(token, beg, end));
}
=======================================
---
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Fri Nov 22 14:39:45 2013 UTC
+++
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Fri Nov 22 14:54:41 2013 UTC
@@ -190,21 +190,23 @@
{%- endmacro %}
#define PREPARE_PUSH_TOKEN() { \
- *beg_pos_to_return = start_ - buffer_; \
- *end_pos_to_return = cursor_ - buffer_; \
+ next_.beg_pos = start_ - buffer_; \
+ next_.end_pos = cursor_ - buffer_; \
start_ = cursor_; \
just_seen_line_terminator_ = false; \
}
#define PUSH_TOKEN(T) { \
PREPARE_PUSH_TOKEN(); \
- return T; \
+ next_.token = T; \
+ return; \
}
#define PUSH_TOKEN_AND_GO_FORWARD(T) {\
PREPARE_PUSH_TOKEN(); \
FORWARD(); \
- return T; \
+ next_.token = T; \
+ return; \
}
#define PUSH_EOS() { \
@@ -235,8 +237,7 @@
namespace v8 {
namespace internal {
template<>
-Token::Value ExperimentalScanner<{{char_type}}>::Next(int*
beg_pos_to_return,
- int*
end_pos_to_return) {
+void ExperimentalScanner<{{char_type}}>::Scan() {
// Setup environment.
{{char_type}} primary_char;
if (cursor_ >= buffer_end_) primary_char = 0;
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
