Reviewers: Jakob, jochen,
Description:
Fix DCHECK on SetBookmark.
The DCHECK was a lie. The idea was that - when a bookmark is set -
the scanner must clearly be at a character boundary and hence the
bookmark does not need to save a 'partial' UTF-8 code point. The
first part is true - the Scanner is always at a character boundary -
but the 'partial' UTF-8 code point is at the end of a block, not at
the current character position of the Scanner.
Hence, the 'partial' character needs to be saved as well.
jkummerow: Thanks for noticing.
BUG=chromium:470930
[email protected], [email protected]
LOG=N
Please review this at https://codereview.chromium.org/1154773004/
Base URL: https://chromium.googlesource.com/v8/v8.git@master
Affected files (+15, -3 lines):
M src/scanner-character-streams.h
M src/scanner-character-streams.cc
Index: src/scanner-character-streams.cc
diff --git a/src/scanner-character-streams.cc
b/src/scanner-character-streams.cc
index
98e79aad76fd458715ad69649b692f3083a4ee40..b5124fac25a45fd862e87a522651db709a56318e
100644
--- a/src/scanner-character-streams.cc
+++ b/src/scanner-character-streams.cc
@@ -388,14 +388,13 @@ size_t ExternalStreamingStream::FillBuffer(size_t
position) {
bool ExternalStreamingStream::SetBookmark() {
- DCHECK(utf8_split_char_buffer_length_ == 0); // We can't be within a
char.
-
// Bookmarking for this stream is a bit more complex than expected, since
// the stream state is distributed over several places:
// - pos_ (inherited from Utf16CharacterStream)
// - buffer_cursor_ and buffer_end_ (also from Utf16CharacterStream)
// - buffer_ (from BufferedUtf16CharacterStream)
// - current_data_ (+ .._offset_ and .._length) (this class)
+ // - utf8_split_char_buffer_* (a partial utf8 symbol at the block
boundary)
//
// The underlying source_stream_ instance likely could re-construct this
// local data for us, but with the given interfaces we have no way of
@@ -405,6 +404,7 @@ bool ExternalStreamingStream::SetBookmark() {
// - pos_ => bookmark_
// - buffer_[buffer_cursor_ .. buffer_end_] => bookmark_buffer_
// - current_data_[.._offset_ .. .._length_] => bookmark_data_
+ // - utf8_split_char_buffer_* => bookmark_utf8_split...
bookmark_ = pos_;
@@ -419,6 +419,10 @@ bool ExternalStreamingStream::SetBookmark() {
CopyBytes(bookmark_data_.start(), current_data_ + current_data_offset_,
data_length);
+ bookmark_utf8_split_char_buffer_length_ = utf8_split_char_buffer_length_;
+ for (size_t i = 0; i < utf8_split_char_buffer_length_; i++)
+ bookmark_utf8_split_char_buffer_[i] = utf8_split_char_buffer_[i];
+
return source_stream_->SetBookmark();
}
@@ -439,6 +443,11 @@ void ExternalStreamingStream::ResetToBookmark() {
bookmark_buffer_.length());
buffer_cursor_ = buffer_;
buffer_end_ = buffer_ + bookmark_buffer_.length();
+
+ // utf8 split char buffer
+ utf8_split_char_buffer_length_ = bookmark_utf8_split_char_buffer_length_;
+ for (size_t i = 0; i < bookmark_utf8_split_char_buffer_length_; i++)
+ utf8_split_char_buffer_[i] = bookmark_utf8_split_char_buffer_[i];
}
Index: src/scanner-character-streams.h
diff --git a/src/scanner-character-streams.h
b/src/scanner-character-streams.h
index
e9047d3212e101675e777151e7a70f58ce290124..582165710db84c1171f74a163e3dcfdeaa8170ac
100644
--- a/src/scanner-character-streams.h
+++ b/src/scanner-character-streams.h
@@ -93,7 +93,8 @@ class ExternalStreamingStream : public
BufferedUtf16CharacterStream {
current_data_offset_(0),
current_data_length_(0),
utf8_split_char_buffer_length_(0),
- bookmark_(0) {}
+ bookmark_(0),
+ bookmark_utf8_split_char_buffer_length_(0) {}
virtual ~ExternalStreamingStream() {
delete[] current_data_;
@@ -133,6 +134,8 @@ class ExternalStreamingStream : public
BufferedUtf16CharacterStream {
size_t bookmark_;
Vector<uint16_t> bookmark_buffer_;
Vector<uint8_t> bookmark_data_;
+ uint8_t bookmark_utf8_split_char_buffer_[4];
+ size_t bookmark_utf8_split_char_buffer_length_;
};
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/d/optout.
