Reviewers: ,
Message:
Committed patchset #1 manually as r18039.
Description:
Experimental lexer generator: utf line terminator fix.
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=18039
Please review this at https://codereview.chromium.org/85543002/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+4, -1 lines):
M src/scanner.h
M tools/lexer_generator/code_generator.py
Index: src/scanner.h
diff --git a/src/scanner.h b/src/scanner.h
index
4a58b1e366b6c3019444897cd111e0c15e2a8567..f6f797e22f1e08dce696b9141fb7d7440195a85d
100644
--- a/src/scanner.h
+++ b/src/scanner.h
@@ -139,6 +139,9 @@ class UnicodeCache {
bool IsIdentifierPart(unibrow::uchar c) { return
kIsIdentifierPart.get(c); }
bool IsLineTerminator(unibrow::uchar c) { return
kIsLineTerminator.get(c); }
bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); }
+ bool IsWhiteSpaceNotLineTerminator(unibrow::uchar c) {
+ return !kIsLineTerminator.get(c) && kIsWhiteSpace.get(c);
+ }
bool IsLetter(unibrow::uchar c) { return kIsLetter.get(c); }
bool IsIdentifierPartNotLetter(unibrow::uchar c) {
return kIsIdentifierPartNotLetter.get(c);
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
ba0ec0daf538afa14c9c8c0e190f1f6cd5185f45..b609c238b49a46f3fa3ac724c413ad0d9546d70a
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -200,7 +200,7 @@ class CodeGenerator:
return split_count + (0 if no_switch else 1)
__call_map = {
- 'non_primary_whitespace' : 'IsWhiteSpace',
+ 'non_primary_whitespace' : 'IsWhiteSpaceNotLineTerminator',
'non_primary_letter' : 'IsLetter',
'non_primary_identifier_part_not_letter' : 'IsIdentifierPartNotLetter',
'non_primary_line_terminator' : 'IsLineTerminator',
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
