Revision: 19880
Author: [email protected]
Date: Thu Mar 13 08:29:31 2014 UTC
Log: move remaining uses of scanner literals into scanner
[email protected]
BUG=
Review URL: https://codereview.chromium.org/198713002
http://code.google.com/p/v8/source/detail?r=19880
Modified:
/branches/bleeding_edge/src/preparser.cc
/branches/bleeding_edge/src/preparser.h
/branches/bleeding_edge/src/scanner.cc
/branches/bleeding_edge/src/scanner.h
/branches/bleeding_edge/test/cctest/test-parsing.cc
=======================================
--- /branches/bleeding_edge/src/preparser.cc Wed Mar 12 14:03:25 2014 UTC
+++ /branches/bleeding_edge/src/preparser.cc Thu Mar 13 08:29:31 2014 UTC
@@ -1164,14 +1164,7 @@
reserved_error_loc = scanner()->location();
}
- int prev_value;
- if (scanner()->is_literal_one_byte()) {
- prev_value = duplicate_finder.AddAsciiSymbol(
- scanner()->literal_one_byte_string(), 1);
- } else {
- prev_value =
-
duplicate_finder.AddUtf16Symbol(scanner()->literal_utf16_string(), 1);
- }
+ int prev_value = scanner()->FindSymbol(&duplicate_finder, 1);
if (!dupe_error_loc.IsValid() && prev_value != 0) {
dupe_error_loc = scanner()->location();
@@ -1273,12 +1266,7 @@
void PreParser::LogSymbol() {
- int identifier_pos = position();
- if (scanner()->is_literal_one_byte()) {
- log_->LogAsciiSymbol(identifier_pos,
scanner()->literal_one_byte_string());
- } else {
- log_->LogUtf16Symbol(identifier_pos,
scanner()->literal_utf16_string());
- }
+ scanner()->LogSymbol(log_, position());
}
=======================================
--- /branches/bleeding_edge/src/preparser.h Wed Mar 12 19:15:17 2014 UTC
+++ /branches/bleeding_edge/src/preparser.h Thu Mar 13 08:29:31 2014 UTC
@@ -1549,11 +1549,9 @@
bool* ok) {
int old;
if (property == Token::NUMBER) {
- old = finder_.AddNumber(scanner()->literal_one_byte_string(), type);
- } else if (scanner()->is_literal_one_byte()) {
- old = finder_.AddAsciiSymbol(scanner()->literal_one_byte_string(),
type);
+ old = scanner()->FindNumber(&finder_, type);
} else {
- old = finder_.AddUtf16Symbol(scanner()->literal_utf16_string(), type);
+ old = scanner()->FindSymbol(&finder_, type);
}
PropertyKind old_type = static_cast<PropertyKind>(old);
if (HasConflict(old_type, type)) {
=======================================
--- /branches/bleeding_edge/src/scanner.cc Wed Mar 12 14:03:25 2014 UTC
+++ /branches/bleeding_edge/src/scanner.cc Thu Mar 13 08:29:31 2014 UTC
@@ -36,6 +36,7 @@
#include "conversions-inl.h"
#include "list-inl.h"
#include "v8.h"
+#include "parser.h"
namespace v8 {
namespace internal {
@@ -1113,18 +1114,6 @@
next_.location.end_pos = source_pos() - 1;
return true;
}
-
-
-Handle<String> Scanner::AllocateLiteralString(Isolate* isolate,
- PretenureFlag tenured) {
- if (is_literal_one_byte()) {
- return isolate->factory()->NewStringFromOneByte(
- Vector<const uint8_t>::cast(literal_one_byte_string()), tenured);
- } else {
- return isolate->factory()->NewStringFromTwoByte(
- literal_utf16_string(), tenured);
- }
-}
Handle<String> Scanner::AllocateNextLiteralString(Isolate* isolate,
@@ -1156,6 +1145,28 @@
unicode_cache_, literal_one_byte_string(),
ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY);
}
+
+
+int Scanner::FindNumber(DuplicateFinder* finder, int value) {
+ return finder->AddNumber(literal_one_byte_string(), value);
+}
+
+
+int Scanner::FindSymbol(DuplicateFinder* finder, int value) {
+ if (is_literal_one_byte()) {
+ return finder->AddAsciiSymbol(literal_one_byte_string(), value);
+ }
+ return finder->AddUtf16Symbol(literal_utf16_string(), value);
+}
+
+
+void Scanner::LogSymbol(ParserRecorder* log, int position) {
+ if (is_literal_one_byte()) {
+ log->LogAsciiSymbol(position, literal_one_byte_string());
+ } else {
+ log->LogUtf16Symbol(position, literal_utf16_string());
+ }
+}
int DuplicateFinder::AddAsciiSymbol(Vector<const char> key, int value) {
=======================================
--- /branches/bleeding_edge/src/scanner.h Wed Mar 12 14:03:25 2014 UTC
+++ /branches/bleeding_edge/src/scanner.h Thu Mar 13 08:29:31 2014 UTC
@@ -44,6 +44,9 @@
namespace internal {
+class ParserRecorder;
+
+
// Returns the value (0 .. 15) of a hexadecimal character c.
// If c is not a legal hexadecimal character, returns a value < 0.
inline int HexValue(uc32 c) {
@@ -370,32 +373,13 @@
// Returns the location information for the current token
// (the token last returned by Next()).
Location location() const { return current_.location; }
- // Returns the literal string, if any, for the current token (the
- // token last returned by Next()). The string is 0-terminated.
- // Literal strings are collected for identifiers, strings, and
- // numbers.
- // These functions only give the correct result if the literal
- // was scanned between calls to StartLiteral() and TerminateLiteral().
- Vector<const char> literal_one_byte_string() {
- ASSERT_NOT_NULL(current_.literal_chars);
- return current_.literal_chars->one_byte_literal();
- }
- Vector<const uc16> literal_utf16_string() {
- ASSERT_NOT_NULL(current_.literal_chars);
- return current_.literal_chars->utf16_literal();
- }
- bool is_literal_one_byte() {
- ASSERT_NOT_NULL(current_.literal_chars);
- return current_.literal_chars->is_one_byte();
- }
- bool is_literal_contextual_keyword(Vector<const char> keyword) {
- ASSERT_NOT_NULL(current_.literal_chars);
- return current_.literal_chars->is_contextual_keyword(keyword);
- }
- int literal_length() const {
- ASSERT_NOT_NULL(current_.literal_chars);
- return current_.literal_chars->length();
- }
+
+ // Similar functions for the upcoming token.
+
+ // One token look-ahead (past the token returned by Next()).
+ Token::Value peek() const { return next_.token; }
+
+ Location peek_location() const { return next_.location; }
bool literal_contains_escapes() const {
Location location = current_.location;
@@ -406,38 +390,15 @@
}
return current_.literal_chars->length() != source_length;
}
-
- // Similar functions for the upcoming token.
-
- // One token look-ahead (past the token returned by Next()).
- Token::Value peek() const { return next_.token; }
-
- Location peek_location() const { return next_.location; }
-
- // Returns the literal string for the next token (the token that
- // would be returned if Next() were called).
- Vector<const char> next_literal_one_byte_string() {
- ASSERT_NOT_NULL(next_.literal_chars);
- return next_.literal_chars->one_byte_literal();
- }
- Vector<const uc16> next_literal_utf16_string() {
- ASSERT_NOT_NULL(next_.literal_chars);
- return next_.literal_chars->utf16_literal();
- }
- bool is_next_literal_one_byte() {
- ASSERT_NOT_NULL(next_.literal_chars);
- return next_.literal_chars->is_one_byte();
+ bool is_literal_contextual_keyword(Vector<const char> keyword) {
+ ASSERT_NOT_NULL(current_.literal_chars);
+ return current_.literal_chars->is_contextual_keyword(keyword);
}
bool is_next_contextual_keyword(Vector<const char> keyword) {
ASSERT_NOT_NULL(next_.literal_chars);
return next_.literal_chars->is_contextual_keyword(keyword);
}
- int next_literal_length() const {
- ASSERT_NOT_NULL(next_.literal_chars);
- return next_.literal_chars->length();
- }
- Handle<String> AllocateLiteralString(Isolate* isolate, PretenureFlag
tenured);
Handle<String> AllocateNextLiteralString(Isolate* isolate,
PretenureFlag tenured);
Handle<String> AllocateInternalizedString(Isolate* isolate);
@@ -461,12 +422,12 @@
}
}
- UnicodeCache* unicode_cache() { return unicode_cache_; }
+ int FindNumber(DuplicateFinder* finder, int value);
+ int FindSymbol(DuplicateFinder* finder, int value);
- static const int kCharacterLookaheadBufferSize = 1;
+ void LogSymbol(ParserRecorder* log, int position);
- // Scans octal escape sequence. Also accepts "\0" decimal escape
sequence.
- uc32 ScanOctalEscape(uc32 c, int length);
+ UnicodeCache* unicode_cache() { return unicode_cache_; }
// Returns the location of the last seen octal literal.
Location octal_position() const { return octal_pos_; }
@@ -519,6 +480,11 @@
LiteralBuffer* literal_chars;
};
+ static const int kCharacterLookaheadBufferSize = 1;
+
+ // Scans octal escape sequence. Also accepts "\0" decimal escape
sequence.
+ uc32 ScanOctalEscape(uc32 c, int length);
+
// Call this after setting source_ to the input.
void Init() {
// Set c0_ (one character ahead)
@@ -578,6 +544,47 @@
return else_;
}
}
+
+ // Returns the literal string, if any, for the current token (the
+ // token last returned by Next()). The string is 0-terminated.
+ // Literal strings are collected for identifiers, strings, and
+ // numbers.
+ // These functions only give the correct result if the literal
+ // was scanned between calls to StartLiteral() and TerminateLiteral().
+ Vector<const char> literal_one_byte_string() {
+ ASSERT_NOT_NULL(current_.literal_chars);
+ return current_.literal_chars->one_byte_literal();
+ }
+ Vector<const uc16> literal_utf16_string() {
+ ASSERT_NOT_NULL(current_.literal_chars);
+ return current_.literal_chars->utf16_literal();
+ }
+ bool is_literal_one_byte() {
+ ASSERT_NOT_NULL(current_.literal_chars);
+ return current_.literal_chars->is_one_byte();
+ }
+ int literal_length() const {
+ ASSERT_NOT_NULL(current_.literal_chars);
+ return current_.literal_chars->length();
+ }
+ // Returns the literal string for the next token (the token that
+ // would be returned if Next() were called).
+ Vector<const char> next_literal_one_byte_string() {
+ ASSERT_NOT_NULL(next_.literal_chars);
+ return next_.literal_chars->one_byte_literal();
+ }
+ Vector<const uc16> next_literal_utf16_string() {
+ ASSERT_NOT_NULL(next_.literal_chars);
+ return next_.literal_chars->utf16_literal();
+ }
+ bool is_next_literal_one_byte() {
+ ASSERT_NOT_NULL(next_.literal_chars);
+ return next_.literal_chars->is_one_byte();
+ }
+ int next_literal_length() const {
+ ASSERT_NOT_NULL(next_.literal_chars);
+ return next_.literal_chars->length();
+ }
uc32 ScanHexNumber(int expected_length);
=======================================
--- /branches/bleeding_edge/test/cctest/test-parsing.cc Wed Mar 12 14:03:25
2014 UTC
+++ /branches/bleeding_edge/test/cctest/test-parsing.cc Thu Mar 13 08:29:31
2014 UTC
@@ -795,6 +795,7 @@
i::Utf8ToUtf16CharacterStream stream(
reinterpret_cast<const i::byte*>(re_source),
static_cast<unsigned>(strlen(re_source)));
+ i::HandleScope scope(CcTest::i_isolate());
i::Scanner scanner(CcTest::i_isolate()->unicode_cache());
scanner.Initialize(&stream);
@@ -802,8 +803,12 @@
CHECK(start == i::Token::DIV || start == i::Token::ASSIGN_DIV);
CHECK(scanner.ScanRegExpPattern(start == i::Token::ASSIGN_DIV));
scanner.Next(); // Current token is now the regexp literal.
- CHECK(scanner.is_literal_one_byte());
- i::Vector<const char> actual = scanner.literal_one_byte_string();
+ i::Handle<i::String> val =
+ scanner.AllocateInternalizedString(CcTest::i_isolate());
+ i::DisallowHeapAllocation no_alloc;
+ i::String::FlatContent content = val->GetFlatContent();
+ CHECK(content.IsAscii());
+ i::Vector<const uint8_t> actual = content.ToOneByteVector();
for (int i = 0; i < actual.length(); i++) {
CHECK_NE('\0', expected[i]);
CHECK_EQ(expected[i], actual[i]);
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
