Revision: 17274
Author: [email protected]
Date: Fri Oct 18 14:33:27 2013 UTC
Log: Experimental parser: Starting to unify the ExperimentalScanner
iface with Scanner iface.
.. so that we can replace Scanner with ExperimentalScanner.
BUG=
[email protected]
Review URL: https://codereview.chromium.org/28763003
http://code.google.com/p/v8/source/detail?r=17274
Added:
/branches/experimental/parser/src/lexer/experimental-scanner.cc
/branches/experimental/parser/src/lexer/experimental-scanner.h
Modified:
/branches/experimental/parser/src/lexer/lexer-shell.cc
/branches/experimental/parser/src/lexer/lexer.gyp
/branches/experimental/parser/src/lexer/lexer.h
/branches/experimental/parser/src/lexer/lexer.re
=======================================
--- /dev/null
+++ /branches/experimental/parser/src/lexer/experimental-scanner.cc Fri Oct
18 14:33:27 2013 UTC
@@ -0,0 +1,131 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include "experimental-scanner.h"
+
+#include "lexer.h"
+
+namespace v8 {
+namespace internal {
+
+namespace {
+
+// Will go away.
+const byte* ReadFile(const char* name, Isolate* isolate, int* size) {
+ FILE* file = fopen(name, "rb");
+ *size = 0;
+ if (file == NULL) return NULL;
+
+ fseek(file, 0, SEEK_END);
+ *size = ftell(file);
+ rewind(file);
+
+ byte* chars = new byte[*size + 1];
+ chars[*size] = 0;
+ for (int i = 0; i < *size;) {
+ int read = static_cast<int>(fread(&chars[i], 1, *size - i, file));
+ i += read;
+ }
+ fclose(file);
+ return chars;
+}
+
+}
+
+ExperimentalScanner::ExperimentalScanner(const char* fname,
+ bool read_all_at_once)
+ : current_(0),
+ fetched_(0),
+ read_all_at_once_(read_all_at_once),
+ source_(0),
+ length_(0) {
+ file_ = fopen(fname, "rb");
+ scanner_ = new PushScanner(this);
+ if (read_all_at_once_) {
+ source_ = ReadFile(fname, NULL, &length_);
+ token_.resize(1500);
+ beg_.resize(1500);
+ end_.resize(1500);
+ } else {
+ token_.resize(BUFFER_SIZE);
+ beg_.resize(BUFFER_SIZE);
+ end_.resize(BUFFER_SIZE);
+ }
+}
+
+
+ExperimentalScanner::~ExperimentalScanner() {
+ fclose(file_);
+ delete[] source_;
+}
+
+
+void ExperimentalScanner::FillTokens() {
+ current_ = 0;
+ fetched_ = 0;
+ if (read_all_at_once_) {
+ scanner_->push(source_, length_ + 1);
+ } else {
+ uint8_t chars[BUFFER_SIZE];
+ int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
+ for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
+ scanner_->push(chars, BUFFER_SIZE);
+ }
+}
+
+
+Token::Value ExperimentalScanner::Next() {
+ while (current_ == fetched_)
+ FillTokens();
+ return token_[current_++];
+}
+
+
+Token::Value ExperimentalScanner::current_token() {
+ return token_[current_ - 1];
+}
+
+
+ExperimentalScanner::Location ExperimentalScanner::location() {
+ return Location(beg_[current_ - 1], end_[current_ - 1]);
+}
+
+
+void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
+ if (token == Token::EOS) end--;
+ if (fetched_ >= token_.size()) {
+ token_.resize(token_.size() * 2);
+ beg_.resize(beg_.size() * 2);
+ end_.resize(end_.size() * 2);
+ }
+ token_[fetched_] = token;
+ beg_[fetched_] = beg;
+ end_[fetched_] = end;
+ fetched_++;
+}
+
+} } // namespace v8::internal
=======================================
--- /dev/null
+++ /branches/experimental/parser/src/lexer/experimental-scanner.h Fri Oct
18 14:33:27 2013 UTC
@@ -0,0 +1,83 @@
+// Copyright 2013 the V8 project authors. All rights reserved.
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+// * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following
+// disclaimer in the documentation and/or other materials provided
+// with the distribution.
+// * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived
+// from this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#ifndef V8_LEXER_EXPERIMENTAL_SCANNER_H
+#define V8_LEXER_EXPERIMENTAL_SCANNER_H
+
+#include <vector>
+
+#include "flags.h"
+#include "token.h"
+
+namespace v8 {
+namespace internal {
+
+class PushScanner;
+
+class ExperimentalScanner {
+ public:
+ struct Location {
+ Location(int b, int e) : beg_pos(b), end_pos(e) { }
+ Location() : beg_pos(0), end_pos(0) { }
+
+ bool IsValid() const {
+ return beg_pos >= 0 && end_pos >= beg_pos;
+ }
+
+ static Location invalid() { return Location(-1, -1); }
+
+ int beg_pos;
+ int end_pos;
+ };
+
+ ExperimentalScanner(const char* fname, bool read_all_at_once);
+ ~ExperimentalScanner();
+
+ Token::Value Next();
+ Token::Value current_token();
+ Location location();
+
+ void Record(v8::internal::Token::Value token, int beg_pos, int end_pos);
+
+ private:
+ void FillTokens();
+ static const int BUFFER_SIZE = 256;
+ std::vector<v8::internal::Token::Value> token_;
+ std::vector<int> beg_;
+ std::vector<int> end_;
+ size_t current_;
+ size_t fetched_;
+ FILE* file_;
+ PushScanner* scanner_;
+ bool read_all_at_once_;
+ const v8::internal::byte* source_;
+ int length_;
+};
+
+} } // namespace v8::internal
+
+#endif
=======================================
--- /branches/experimental/parser/src/lexer/lexer-shell.cc Fri Oct 18
09:01:34 2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer-shell.cc Fri Oct 18
14:33:27 2013 UTC
@@ -42,6 +42,8 @@
#include "scopeinfo.h"
#include "string-stream.h"
#include "scanner.h"
+
+#include "experimental-scanner.h"
#include "lexer.h"
using namespace v8::internal;
@@ -98,73 +100,6 @@
Utf8ToUtf16CharacterStream* stream_;
};
-ExperimentalScanner::ExperimentalScanner(const char* fname,
- bool read_all_at_once)
- : current_(0),
- fetched_(0),
- read_all_at_once_(read_all_at_once),
- source_(0),
- length_(0) {
- file_ = fopen(fname, "rb");
- scanner_ = new PushScanner(this);
- if (read_all_at_once_) {
- source_ = ReadFile(fname, NULL, &length_);
- token_.resize(1500);
- beg_.resize(1500);
- end_.resize(1500);
- } else {
- token_.resize(BUFFER_SIZE);
- beg_.resize(BUFFER_SIZE);
- end_.resize(BUFFER_SIZE);
- }
-}
-
-
-ExperimentalScanner::~ExperimentalScanner() {
- fclose(file_);
- delete[] source_;
-}
-
-
-void ExperimentalScanner::FillTokens() {
- current_ = 0;
- fetched_ = 0;
- if (read_all_at_once_) {
- scanner_->push(source_, length_ + 1);
- } else {
- uint8_t chars[BUFFER_SIZE];
- int n = static_cast<int>(fread(&chars, 1, BUFFER_SIZE, file_));
- for (int i = n; i < BUFFER_SIZE; i++) chars[i] = 0;
- scanner_->push(chars, BUFFER_SIZE);
- }
-}
-
-
-Token::Value ExperimentalScanner::Next(int* beg_pos, int* end_pos) {
- while (current_ == fetched_)
- FillTokens();
- *beg_pos = beg_[current_];
- *end_pos = end_[current_];
- Token::Value res = token_[current_];
- if (res != Token::Token::EOS)
- current_++;
- return res;
-}
-
-
-void ExperimentalScanner::Record(Token::Value token, int beg, int end) {
- if (token == Token::EOS) end--;
- if (fetched_ >= token_.size()) {
- token_.resize(token_.size() * 2);
- beg_.resize(beg_.size() * 2);
- end_.resize(end_.size() * 2);
- }
- token_[fetched_] = token;
- beg_[fetched_] = beg;
- end_[fetched_] = end;
- fetched_++;
-}
-
int main(int argc, char* argv[]) {
v8::V8::InitializeICU();
@@ -204,10 +139,11 @@
{
timer.Start();
do {
- token = experimental.Next(&beg, &end);
+ token = experimental.Next();
experimental_tokens.push_back(token);
- experimental_beg.push_back(beg);
- experimental_end.push_back(end);
+ ExperimentalScanner::Location location = experimental.location();
+ experimental_beg.push_back(location.beg_pos);
+ experimental_end.push_back(location.end_pos);
} while (token != Token::EOS);
experimental_time = timer.Elapsed();
}
=======================================
--- /branches/experimental/parser/src/lexer/lexer.gyp Wed Oct 16 09:06:56
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer.gyp Fri Oct 18 14:33:27
2013 UTC
@@ -44,6 +44,8 @@
'../../src/lexer',
],
'sources': [
+ 'experimental-scanner.cc',
+ 'experimental-scanner.h',
'lexer-shell.cc',
'lexer.h',
'<(SHARED_INTERMEDIATE_DIR)/lexer.cc',
=======================================
--- /branches/experimental/parser/src/lexer/lexer.h Thu Oct 17 13:56:53
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer.h Fri Oct 18 14:33:27
2013 UTC
@@ -30,11 +30,12 @@
#ifndef V8_LEXER_LEXER_H
#define V8_LEXER_LEXER_H
-#include <vector>
-
#include "token.h"
#include "flags.h"
+namespace v8 {
+namespace internal {
+
class ExperimentalScanner;
class PushScanner {
@@ -66,26 +67,6 @@
ExperimentalScanner* sink_;
};
-class ExperimentalScanner {
- public:
- ExperimentalScanner(const char* fname, bool read_all_at_once);
- ~ExperimentalScanner();
- v8::internal::Token::Value Next(int* beg_pos, int* end_pos);
- void Record(v8::internal::Token::Value token, int beg_pos, int end_pos);
-
- private:
- void FillTokens();
- static const int BUFFER_SIZE = 256;
- std::vector<v8::internal::Token::Value> token_;
- std::vector<int> beg_;
- std::vector<int> end_;
- size_t current_;
- size_t fetched_;
- FILE* file_;
- PushScanner* scanner_;
- bool read_all_at_once_;
- const v8::internal::byte* source_;
- int length_;
-};
+} }
#endif // V8_LEXER_LEXER_H
=======================================
--- /branches/experimental/parser/src/lexer/lexer.re Fri Oct 18 11:41:17
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer.re Fri Oct 18 14:33:27
2013 UTC
@@ -70,6 +70,7 @@
#endif // defined(WIN32)
+#include "experimental-scanner.h"
#include "lexer.h"
using namespace v8::internal;
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
