Revision: 19665
Author: [email protected]
Date: Wed Mar 5 08:32:50 2014 UTC
Log: Experimental parser: more cleanup after rebase
[email protected]
BUG=
Review URL: https://codereview.chromium.org/180743019
http://code.google.com/p/v8/source/detail?r=19665
Modified:
/branches/experimental/parser/src/lexer/lexer-shell.cc
/branches/experimental/parser/src/lexer/lexer.cc
/branches/experimental/parser/src/lexer/lexer.h
=======================================
--- /branches/experimental/parser/src/lexer/lexer-shell.cc Tue Mar 4
15:50:47 2014 UTC
+++ /branches/experimental/parser/src/lexer/lexer-shell.cc Wed Mar 5
08:32:50 2014 UTC
@@ -361,6 +361,7 @@
int main(int argc, char* argv[]) {
+ return 0;
v8::V8::InitializeICU();
v8::V8::SetFlagsFromCommandLine(&argc, argv, true);
std::vector<std::string> fnames;
=======================================
--- /branches/experimental/parser/src/lexer/lexer.cc Tue Mar 4 15:50:47
2014 UTC
+++ /branches/experimental/parser/src/lexer/lexer.cc Wed Mar 5 08:32:50
2014 UTC
@@ -113,12 +113,16 @@
isolate_->heap()->AddGCEpilogueCallback(
&i::UpdateLexersAfterGC, kGCTypeAll, true);
}
- lexers_.insert(lexer);
+ std::pair<LexerSet::iterator, bool> res = lexers_.insert(lexer);
+ USE(res);
+ ASSERT(res.second);
}
void LexerGCHandler::RemoveLexer(LexerBase* lexer) {
- lexers_.erase(lexer);
+ LexerSet::iterator it = lexers_.find(lexer);
+ ASSERT(it != lexers_.end());
+ lexers_.erase(it);
if (lexers_.empty()) {
isolate_->heap()->RemoveGCEpilogueCallback(&i::UpdateLexersAfterGC);
}
@@ -131,6 +135,18 @@
(*it)->UpdateBufferBasedOnHandle();
}
}
+
+
+LexerBase::LexerBase(UnicodeCache* unicode_cache)
+ : unicode_cache_(unicode_cache),
+ has_line_terminator_before_next_(true),
+ has_multiline_comment_before_next_(false),
+ current_literal_(&literals_[0]),
+ next_literal_(&literals_[1]),
+ harmony_numeric_literals_(false),
+ harmony_modules_(false),
+ harmony_scoping_(false) {
+}
LexerBase::~LexerBase() {}
@@ -188,8 +204,6 @@
cursor_ = buffer_ + start_position;
buffer_end_ = buffer_ + end_position;
start_ = cursor_;
- has_line_terminator_before_next_ = false;
- has_multiline_comment_before_next_ = false;
}
@@ -209,12 +223,6 @@
has_multiline_comment_before_next_ = false;
Scan(); // Fills in next_.
}
-
-
-template<typename Char>
-void Lexer<Char>::SetEnd(int pos) {
- buffer_end_ = buffer_ + pos;
-}
template<typename Char>
@@ -518,12 +526,11 @@
--end;
}
if (IsSubstringOfSource(token)) {
- literal->is_ascii = true;
+ literal->is_one_byte = true;
literal->is_in_buffer = false;
literal->offset = start - buffer_;
literal->length = end - start;
- literal->ascii_string = Vector<const char>(
- reinterpret_cast<const char*>(start), literal->length);
+ literal->one_byte_string = Vector<const uint8_t>(start,
literal->length);
return true;
}
return CopyToLiteralBuffer(start, end, token, literal);
@@ -541,11 +548,11 @@
--end;
}
if (IsSubstringOfSource(token)) {
- literal->is_ascii = false;
+ literal->is_one_byte = false;
literal->is_in_buffer = false;
literal->offset = start - buffer_;
literal->length = end - start;
- literal->utf16_string = Vector<const uint16_t>(start, literal->length);
+ literal->two_byte_string = Vector<const uint16_t>(start,
literal->length);
return true;
}
return CopyToLiteralBuffer(start, end, token, literal);
@@ -588,13 +595,14 @@
literal->buffer.AddChar(*cursor++);
}
}
- literal->is_ascii = literal->buffer.is_ascii();
+ literal->is_one_byte = literal->buffer.is_ascii();
literal->is_in_buffer = true;
literal->length = literal->buffer.length();
- if (literal->is_ascii) {
- literal->ascii_string = literal->buffer.ascii_literal();
+ if (literal->is_one_byte) {
+ literal->one_byte_string =
+ Vector<const uint8_t>::cast(literal->buffer.ascii_literal());
} else {
- literal->utf16_string = literal->buffer.utf16_literal();
+ literal->two_byte_string = literal->buffer.utf16_literal();
}
return true;
}
@@ -605,10 +613,10 @@
LiteralDesc* literal) {
Factory* factory = isolate_->factory();
if (literal->is_in_buffer) {
- return literal->is_ascii
+ return literal->is_one_byte
? factory->InternalizeOneByteString(
- Vector<const uint8_t>::cast(literal->ascii_string))
- : factory->InternalizeTwoByteString(literal->utf16_string);
+ Vector<const uint8_t>::cast(literal->one_byte_string))
+ : factory->InternalizeTwoByteString(literal->two_byte_string);
}
if (sizeof(Char) == 1) {
SubStringKey<uint8_t> key(
@@ -627,9 +635,9 @@
LiteralDesc* literal, PretenureFlag pretenured) {
Factory* factory = isolate_->factory();
if (literal->is_in_buffer) {
- return literal->is_ascii
- ? factory->NewStringFromAscii(literal->ascii_string, pretenured)
- : factory->NewStringFromTwoByte(literal->utf16_string, pretenured);
+ return literal->is_one_byte
+ ? factory->NewStringFromOneByte(literal->one_byte_string,
pretenured)
+ : factory->NewStringFromTwoByte(literal->two_byte_string,
pretenured);
}
int from = literal->offset;
int length = literal->length;
@@ -647,9 +655,9 @@
LiteralDesc* literal, PretenureFlag pretenured) {
Factory* factory = isolate_->factory();
if (literal->is_in_buffer) {
- return literal->is_ascii
- ? factory->NewStringFromAscii(literal->ascii_string, pretenured)
- : factory->NewStringFromTwoByte(literal->utf16_string, pretenured);
+ return literal->is_one_byte
+ ? factory->NewStringFromOneByte(literal->one_byte_string,
pretenured)
+ : factory->NewStringFromTwoByte(literal->two_byte_string,
pretenured);
}
// Save the offset and the length before allocating the string as it may
// cause a GC, invalidate the literal, and move the source.
=======================================
--- /branches/experimental/parser/src/lexer/lexer.h Tue Mar 4 14:45:34
2014 UTC
+++ /branches/experimental/parser/src/lexer/lexer.h Wed Mar 5 08:32:50
2014 UTC
@@ -45,8 +45,9 @@
void UpdateLexersAfterGC();
private:
+ typedef std::set<LexerBase*> LexerSet;
Isolate* isolate_;
- std::set<LexerBase*> lexers_;
+ LexerSet lexers_;
};
@@ -66,16 +67,7 @@
int end_pos;
};
- explicit LexerBase(UnicodeCache* unicode_cache)
- : unicode_cache_(unicode_cache),
- has_line_terminator_before_next_(true),
- has_multiline_comment_before_next_(false),
- current_literal_(&literals_[0]),
- next_literal_(&literals_[1]),
- harmony_numeric_literals_(false),
- harmony_modules_(false),
- harmony_scoping_(false) {
- }
+ explicit LexerBase(UnicodeCache* unicode_cache);
virtual ~LexerBase();
@@ -105,8 +97,6 @@
// we need to decide if pos is counted in characters or in bytes.
virtual void SeekForward(int pos) = 0;
- virtual void SetEnd(int pos) = 0;
-
// Scans the input as a regular expression pattern, previous
character(s) must
// be /(=). Returns true if a pattern is scanned. FIXME: this won't work
for
// utf-8 newlines.
@@ -128,29 +118,14 @@
has_multiline_comment_before_next_;
}
- Handle<String> GetLiteralSymbol() {
+ Vector<const uint8_t> literal_one_byte_string() {
EnsureCurrentLiteralIsValid();
- return InternalizeLiteral(current_literal_);
+ return current_literal_->one_byte_string;
}
- Handle<String> GetLiteralString(PretenureFlag tenured) {
+ Vector<const uint16_t> literal_two_byte_string() {
EnsureCurrentLiteralIsValid();
- return AllocateLiteral(current_literal_, tenured);
- }
-
- Handle<String> GetNextLiteralString(PretenureFlag tenured) {
- EnsureNextLiteralIsValid();
- return AllocateLiteral(next_literal_, tenured);
- }
-
- Vector<const char> literal_ascii_string() {
- EnsureCurrentLiteralIsValid();
- return current_literal_->ascii_string;
- }
-
- Vector<const uc16> literal_utf16_string() {
- EnsureCurrentLiteralIsValid();
- return current_literal_->utf16_string;
+ return current_literal_->two_byte_string;
}
int literal_length() {
@@ -158,14 +133,14 @@
return current_literal_->length;
}
- bool is_literal_ascii() {
+ bool is_literal_one_byte() {
EnsureCurrentLiteralIsValid();
- return current_literal_->is_ascii;
+ return current_literal_->is_one_byte;
}
- bool is_literal_contextual_keyword(Vector<const char> keyword) {
- if (!is_literal_ascii()) return false;
- Vector<const char> literal = literal_ascii_string();
+ bool is_literal_contextual_keyword(Vector<const uint8_t> keyword) {
+ if (!is_literal_one_byte()) return false;
+ Vector<const uint8_t> literal = literal_one_byte_string();
return literal.length() == keyword.length() &&
(memcmp(literal.start(), keyword.start(), literal.length()) == 0);
}
@@ -174,14 +149,14 @@
return current_.has_escapes;
}
- Vector<const char> next_literal_ascii_string() {
+ Vector<const uint8_t> next_literal_one_byte_string() {
EnsureNextLiteralIsValid();
- return next_literal_->ascii_string;
+ return next_literal_->one_byte_string;
}
- Vector<const uc16> next_literal_utf16_string() {
+ Vector<const uint16_t> next_literal_two_byte_string() {
EnsureNextLiteralIsValid();
- return next_literal_->utf16_string;
+ return next_literal_->two_byte_string;
}
int next_literal_length() {
@@ -189,14 +164,14 @@
return next_literal_->length;
}
- bool is_next_literal_ascii() {
+ bool is_next_literal_one_byte() {
EnsureNextLiteralIsValid();
- return next_literal_->is_ascii;
+ return next_literal_->is_one_byte;
}
- bool is_next_contextual_keyword(Vector<const char> keyword) {
- if (!is_next_literal_ascii()) return false;
- Vector<const char> literal = next_literal_ascii_string();
+ bool is_next_contextual_keyword(Vector<const uint8_t> keyword) {
+ if (!is_next_literal_one_byte()) return false;
+ Vector<const uint8_t> literal = next_literal_one_byte_string();
return literal.length() == keyword.length() &&
(memcmp(literal.start(), keyword.start(), literal.length()) == 0);
}
@@ -238,14 +213,14 @@
struct LiteralDesc {
int beg_pos;
- bool is_ascii;
+ bool is_one_byte;
bool is_in_buffer;
int offset;
int length;
- Vector<const char> ascii_string;
- Vector<const uc16> utf16_string;
+ Vector<const uint8_t> one_byte_string;
+ Vector<const uint16_t> two_byte_string;
LiteralBuffer buffer;
- LiteralDesc() : beg_pos(-1), is_ascii(false), is_in_buffer(false),
+ LiteralDesc() : beg_pos(-1), is_one_byte(false), is_in_buffer(false),
offset(0), length(0) { }
bool Valid(int pos) { return beg_pos == pos; }
};
@@ -303,13 +278,12 @@
public:
Lexer(UnicodeCache* unicode_cache,
Handle<String> source,
- int start_position_,
- int end_position_);
+ int start_position,
+ int end_position);
Lexer(UnicodeCache* unicode_cache, const Char* source_ptr, int length);
virtual ~Lexer();
virtual void SeekForward(int pos);
- virtual void SetEnd(int pos);
virtual bool ScanRegExpPattern(bool seen_equal);
virtual bool ScanRegExpFlags();
virtual Location octal_position() const;
@@ -353,16 +327,18 @@
const TokenDesc& token,
LiteralDesc* literal);
+ // One of source_handle_ or source_ptr_ is set.
+ // If source_ptr_ is set, isolate_ is 0 and no isolate accesses are
allowed.
Isolate* isolate_;
const Handle<String> source_handle_;
const Char* const source_ptr_;
const int start_position_;
const int end_position_;
+ // Stream variables.
const Char* buffer_;
const Char* buffer_end_;
const Char* start_;
const Char* cursor_;
-
// Where we have seen the last octal number or an octal escape inside a
// string. Used by octal_position().
const Char* last_octal_end_;
@@ -384,8 +360,6 @@
void Initialize(Utf16CharacterStream* source);
inline void SeekForward(int pos) { lexer_->SeekForward(pos); }
-
- inline void SetEnd(int pos) { lexer_->SetEnd(pos); }
inline bool ScanRegExpPattern(bool seen_equal) {
return lexer_->ScanRegExpPattern(seen_equal);
@@ -439,25 +413,13 @@
inline bool HasAnyLineTerminatorBeforeNext() const {
return lexer_->HasAnyLineTerminatorBeforeNext();
}
-
- inline Handle<String> GetLiteralSymbol() {
- return lexer_->GetLiteralSymbol();
- }
-
- inline Handle<String> GetLiteralString(PretenureFlag tenured) {
- return lexer_->GetLiteralString(tenured);
- }
-
- inline Handle<String> GetNextLiteralString(PretenureFlag tenured) {
- return lexer_->GetNextLiteralString(tenured);
- }
inline Vector<const char> literal_ascii_string() {
- return lexer_->literal_ascii_string();
+ return Vector<const char>::cast(lexer_->literal_one_byte_string());
}
inline Vector<const uc16> literal_utf16_string() {
- return lexer_->literal_utf16_string();
+ return lexer_->literal_two_byte_string();
}
inline int literal_length() {
@@ -465,11 +427,13 @@
}
inline bool is_literal_ascii() {
- return lexer_->is_literal_ascii();
+ return lexer_->is_literal_one_byte();
}
- inline bool is_literal_contextual_keyword(Vector<const char> keyword) {
- return lexer_->is_literal_contextual_keyword(keyword);
+ inline bool is_literal_contextual_keyword(
+ Vector<const char>& keyword) { // NOLINT
+ return lexer_->is_literal_contextual_keyword(
+ Vector<const uint8_t>::cast(keyword));
}
inline bool literal_contains_escapes() const {
@@ -477,11 +441,11 @@
}
inline Vector<const char> next_literal_ascii_string() {
- return lexer_->next_literal_ascii_string();
+ return Vector<const
char>::cast(lexer_->next_literal_one_byte_string());
}
inline Vector<const uc16> next_literal_utf16_string() {
- return lexer_->next_literal_utf16_string();
+ return lexer_->next_literal_two_byte_string();
}
inline int next_literal_length() {
@@ -489,11 +453,13 @@
}
inline bool is_next_literal_ascii() {
- return lexer_->is_next_literal_ascii();
+ return lexer_->is_next_literal_one_byte();
}
- inline bool is_next_contextual_keyword(Vector<const char> keyword) {
- return lexer_->is_next_contextual_keyword(keyword);
+ inline bool is_next_contextual_keyword(
+ Vector<const char>& keyword) { // NOLINT
+ return lexer_->is_next_contextual_keyword(
+ Vector<const uint8_t>::cast(keyword));
}
private:
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
