Reviewers: marja,
Message:
Committed patchset #1 manually as r17994.
Description:
Experimental parser: cleanup after adding encodings
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=17994
Please review this at https://codereview.chromium.org/82953003/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+18, -18 lines):
M tools/lexer_generator/code_generator.jinja
M tools/lexer_generator/code_generator.py
M tools/lexer_generator/transition_keys.py
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
cadbdb0f6bf614531957de43ffd093d909b606d1..ae8d3273720c64f6133b9c2d8c121d78c24d2ce7
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -3,12 +3,12 @@
{%- macro do_key(key) -%}
{%- for r in key -%}
{%- if not loop.first %} || {% endif -%}
- {%- if r[0] == 'LATIN_1' -%}
+ {%- if r[0] == 'PRIMARY_RANGE' -%}
{%- if r[1][0] == r[1][1] -%}
yych == {{r[1][0]}}
{%- elif r[1][0] == 0 -%}
yych <= {{r[1][1]}}
- {%- elif r[1][1] == 255 and encoding == 'latin1'-%}
+ {%- elif r[1][1] == upper_bound and not encoding == 'utf16'-%}
yych >= {{r[1][0]}}
{%- else -%}
({{r[1][0]}} <= yych && yych <= {{r[1][1]}})
@@ -24,18 +24,17 @@
{%- if r[1] == 'byte_order_mark' -%}
(yych == 0xfffe || yych == 0xfeff)
{%- elif r[1] == 'non_latin_1_whitespace' -%}
- {# FIXME: Add and use unicode_cache_->InNonAsciiWhitespace #}
- (yych > 255 && unicode_cache_->IsWhiteSpace(yych))
+ (yych > {{upper_bound}} && unicode_cache_->IsWhiteSpace(yych))
{%- elif r[1] == 'non_latin_1_letter' -%}
- {# FIXME: Add and use unicode_cache_->InNonAsciiLetter #}
- (yych > 255 && unicode_cache_->IsLetter(yych))
+ (yych > {{upper_bound}} && unicode_cache_->IsLetter(yych))
{%- elif r[1] == 'non_latin_1_identifier_part_not_letter' -%}
- (yych > 255 && unicode_cache_->IsIdentifierPartNotLetter(yych))
+ (yych > {{upper_bound}} &&
+ unicode_cache_->IsIdentifierPartNotLetter(yych))
{%- elif r[1] == 'non_latin_1_line_terminator' -%}
- (yych > 255 && unicode_cache_->IsLineTerminator(yych))
+ (yych > {{upper_bound}} &&
unicode_cache_->IsLineTerminator(yych))
{%- elif r[1] == 'non_latin_1_everything_else' -%}
{# FIXME: Optimize this away #}
- (yych > 255 &&
+ (yych > {{upper_bound}} &&
!unicode_cache_->IsWhiteSpace(yych) &&
!unicode_cache_->IsLetter(yych) &&
!unicode_cache_->IsIdentifierPartNotLetter(yych) &&
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
e6d44662908f0e8bc32decc2d3a006cddebaae0f..67d80c2ad58f3f19e6bf6e5aa8fc709ddbb9313d
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -82,13 +82,13 @@ class CodeGenerator:
@staticmethod
def __range_cmp(left, right):
- if left[0] == 'LATIN_1':
- if right[0] == 'LATIN_1':
+ if left[0] == 'PRIMARY_RANGE':
+ if right[0] == 'PRIMARY_RANGE':
return cmp(left[1], right[1])
assert right[0] == 'CLASS'
return -1
assert left[0] == 'CLASS'
- if right[0] == 'LATIN_1':
+ if right[0] == 'PRIMARY_RANGE':
return 1
# TODO store numeric values and cmp
return cmp(left[1], right[1])
@@ -118,7 +118,7 @@ class CodeGenerator:
for (t, r) in disjoint_keys:
if t == 'CLASS':
class_keys += 1
- elif t == 'LATIN_1':
+ elif t == 'PRIMARY_RANGE':
distinct_keys += r[1] - r[0] + 1
ranges += 1
else:
@@ -246,14 +246,15 @@ class CodeGenerator:
undefined = jinja2.StrictUndefined)
template = template_env.get_template('code_generator.jinja')
- encoding = self.__dfa.encoding().name()
+ encoding = self.__dfa.encoding()
char_types =
{'latin1': 'uint8_t', 'utf16': 'uint16_t', 'utf8': 'int8_t'}
- char_type = char_types[encoding]
+ char_type = char_types[encoding.name()]
return template.render(
start_node_number = 0,
debug_print = self.__debug_print,
default_action = default_action,
dfa_states = dfa_states,
- encoding = encoding,
- char_type = char_type)
+ encoding = encoding.name(),
+ char_type = char_type,
+ upper_bound = encoding.primary_range()[1])
Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py
b/tools/lexer_generator/transition_keys.py
index
765970078549a7ebe89b0f2b08ed19ee60b9d2f7..82fb81b51463e906dff35c848ae1b50f41d51ad4
100644
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -285,7 +285,7 @@ class TransitionKey(object):
if encoding.is_class_range(r):
yield ('CLASS', TransitionKey.__class_name(encoding, r))
else:
- yield ('LATIN_1', r)
+ yield ('PRIMARY_RANGE', r)
__printable_cache = {
ord('\t') : '\\t',
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
