Reviewers: Rico,
Description:
Fix bug when the scanner does a pushback at the end of input.
We don't advance the input cursor past the end of input, so we shouldn't
decrease it when we pushback the kEndOfInput marker.
Please review this at http://codereview.chromium.org/6246004/
Affected files:
M src/preparser-api.cc
M src/scanner-base.h
M src/scanner.h
M src/scanner.cc
Index: src/preparser-api.cc
diff --git a/src/preparser-api.cc b/src/preparser-api.cc
index
dba30265f6aaef24b19966260c2251b8635e6583..3817935f8f8050f5327ff3a9fa964af1a2cb88cd
100644
--- a/src/preparser-api.cc
+++ b/src/preparser-api.cc
@@ -69,8 +69,12 @@ class InputStreamUTF16Buffer : public
UC16CharacterStream {
}
}
- virtual void PushBack(uc16 ch) {
+ virtual void PushBack(uc32 ch) {
ASSERT(pos_ > 0);
+ if (ch == kEndOfInput) {
+ pos_--;
+ return;
+ }
if (buffer_cursor_ <= pushback_buffer_) {
// No more room in the current buffer to do pushbacks.
if (pushback_buffer_end_cache_ == NULL) {
@@ -98,7 +102,8 @@ class InputStreamUTF16Buffer : public
UC16CharacterStream {
buffer_end_ = pushback_buffer_backing_ +
pushback_buffer_backing_size_;
}
}
- pushback_buffer_[buffer_cursor_ - pushback_buffer_- 1] = ch;
+ pushback_buffer_[buffer_cursor_ - pushback_buffer_- 1] =
+ static_cast<uc16>(ch);
pos_--;
}
Index: src/scanner-base.h
diff --git a/src/scanner-base.h b/src/scanner-base.h
index
b668df5046dae63903c4a2fd6445336235a24024..1024ad18582e3c078793d2295fe3cb8a92d23b18
100644
--- a/src/scanner-base.h
+++ b/src/scanner-base.h
@@ -64,10 +64,10 @@ class UC16CharacterStream {
// Returns and advances past the next UC16 character in the input
// stream. If there are no more characters, it returns a negative
// value.
- inline int32_t Advance() {
+ inline uc32 Advance() {
if (buffer_cursor_ < buffer_end_ || ReadBlock()) {
pos_++;
- return *(buffer_cursor_++);
+ return static_cast<uc32>(*(buffer_cursor_++));
}
// Note: currently the following increment is necessary to avoid a
// parser problem! The scanner treats the final kEndOfInput as
@@ -97,13 +97,14 @@ class UC16CharacterStream {
return SlowSeekForward(character_count);
}
- // Pushes back the most recently read UC16 character, i.e.,
- // the value returned by the most recent call to Advance.
+ // Pushes back the most recently read UC16 character (or negative
+ // value if at end of input), i.e., the value returned by the most recent
+ // call to Advance.
// Must not be used right after calling SeekForward.
- virtual void PushBack(uc16 character) = 0;
+ virtual void PushBack(int32_t character) = 0;
protected:
- static const int32_t kEndOfInput = -1;
+ static const uc32 kEndOfInput = -1;
// Ensures that the buffer_cursor_ points to the character at
// position pos_ of the input, if possible. If the position
Index: src/scanner.cc
diff --git a/src/scanner.cc b/src/scanner.cc
index
7fd6ef22e7b7537fcc6400ea892a9f46c06f46d4..b66d10b9882c982d99398e28cf2de785e3a1d7ce
100755
--- a/src/scanner.cc
+++ b/src/scanner.cc
@@ -48,14 +48,18 @@
BufferedUC16CharacterStream::BufferedUC16CharacterStream()
BufferedUC16CharacterStream::~BufferedUC16CharacterStream() { }
-void BufferedUC16CharacterStream::PushBack(uc16 character) {
+void BufferedUC16CharacterStream::PushBack(uc32 character) {
+ if (character == kEndOfInput) {
+ pos_--;
+ return;
+ }
if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) {
// buffer_ is writable, buffer_cursor_ is const pointer.
- buffer_[--buffer_cursor_ - buffer_] = character;
+ buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character);
pos_--;
return;
}
- SlowPushBack(character);
+ SlowPushBack(static_cast<uc16>(character));
}
Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index
bdf899b54df4fb53b699a2f5780e672b03f86bda..d7621825a975e5f2b5a91a266007b45ddbaff9f0
100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -43,7 +43,7 @@ class BufferedUC16CharacterStream: public
UC16CharacterStream {
BufferedUC16CharacterStream();
virtual ~BufferedUC16CharacterStream();
- virtual void PushBack(uc16 character);
+ virtual void PushBack(uc32 character);
protected:
static const unsigned kBufferSize = 512;
@@ -107,11 +107,12 @@ class ExternalTwoByteStringUC16CharacterStream:
public UC16CharacterStream {
int end_position);
virtual ~ExternalTwoByteStringUC16CharacterStream();
- virtual void PushBack(uc16 character) {
+ virtual void PushBack(uc32 character) {
ASSERT(buffer_cursor_ > raw_data_);
buffer_cursor_--;
pos_--;
}
+
protected:
virtual unsigned SlowSeekForward(unsigned delta) {
// Fast case always handles seeking.
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
