If you're interested, I think there is already a Python-based compiler somewhere deep in the SVN history (pre open-source).
--David [email protected] wrote: > Author: gstein > Date: Sat Jan 31 07:40:26 2009 > New Revision: 739520 > > URL: http://svn.apache.org/viewvc?rev=739520&view=rev > Log: > Snapshot (back up) my work-in-progress before I hop on a plane. > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/gen.py > incubator/thrift/branches/py-compiler/compiler/py/src/parser.py > incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/ > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/ > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt > > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt > > Added: incubator/thrift/branches/py-compiler/compiler/py/src/gen.py > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/gen.py?rev=739520&view=auto > ============================================================================== > --- incubator/thrift/branches/py-compiler/compiler/py/src/gen.py (added) > +++ incubator/thrift/branches/py-compiler/compiler/py/src/gen.py Sat Jan 31 > 07:40:26 2009 > @@ -0,0 +1,169 @@ > +# > +# simple generator for Thrift > +# > + > +import sys > +import os > +import cStringIO > +import operator > + > +import parser > +import ezt > + > + > +### temporary > +PATH = '/Users/gstein/src/asf/thrift/compiler/py/src/templates-py' > +t_py = ezt.Template(os.path.join(PATH, 'py.ezt'), > + compress_whitespace=False) > +t_py_ser = ezt.Template(os.path.join(PATH, 'py_ser.ezt'), > + compress_whitespace=False) > +t_py_deser = ezt.Template(os.path.join(PATH, 'py_deser.ezt'), > + compress_whitespace=False) > +t_py_cvalue = ezt.Template(os.path.join(PATH, 'py_cvalue.ezt'), > + compress_whitespace=False) > + > + > +def generate(program): > + t_py.generate(sys.stdout, Proxy(program)) > + > + > +class AutoVars(object): > + def __init__(self): > + self._counter = 0 > + self._mapping = { } > + self._saved = [ ] > + > + def open_context(self): > + self._saved.append(self._mapping) > + self._mapping = { } > + > + def close_context(self): > + self._mapping = self._saved.pop() > + > + def __getattr__(self, name): > + if name.startswith('__'): > + raise AttributeError(name) > + > + if name in self._mapping: > + return self._mapping[name] > + var = '%s%d' % (name, self._counter) > + self._counter += 1 > + self._mapping[name] = var > + return var > + > + > +class Proxy(object): > + def __init__(self, ob): > + self._ob = ob > + > + for name, value in vars(ob).items(): > + proxy = custom_proxy(value) > + if proxy: > + value = proxy(value) > + elif isinstance(value, list) and value: > + # lists are homogenous, so check the first item > + proxy = custom_proxy(value[0]) > + if proxy: > + value = [proxy(ob) for ob in value] > + elif hasattr(value[0], '__dict__'): > + value = [Proxy(ob) for ob in value] > + setattr(self, name, value) > + > + def __getattr__(self, name): > + if name == 'auto': > + return g_auto > + raise AttributeError(name) > + > + > +class ProxyFieldType(Proxy): > + def __getattr__(self, name): > + if name == 'serializer': > + return Subtemplate(t_py_ser, self) > + if name == 'deserializer': > + return Subtemplate(t_py_deser, self) > + return Proxy.__getattr__(self, name) > + > + > +class Subtemplate(object): > + def __init__(self, template, data): > + self._template = template > + self._data = data > + > + def __getattr__(self, name): > + # jam the name of the result variable into the data params > + self._data.result_var = getattr(g_auto, name) > + > + # use a new variable context for this template generation > + g_auto.open_context() > + value = gen_value(self._template, self._data) > + g_auto.close_context() > + > + return value > + > + > +class ProxyField(Proxy): > + def __getattr__(self, name): > + if name == 'type_enum': > + return TYPE_ENUM.get(self._ob.field_type.ident, > + self._ob.field_type.ident.tvalue) > + return Proxy.__getattr__(self, name) > + > + > +class ProxyStruct(Proxy): > + def __getattr__(self, name): > + if name == 'sorted_fields': > + highest = max(int(f.field_id or -1) for f in self._ob.fields) > + fields = [None] * (highest + 1) > + for field in self._ob.fields: > + if field.field_id: > + id = int(field.field_id) > + if id > 0: > + fields[id] = ProxyField(field) > + return fields > + return Proxy.__getattr__(self, name) > + > + > +class ProxyConstValue(Proxy): > + def __getattr__(self, name): > + if name == 'cvalue': > + return gen_value(t_py_cvalue, self) > + return Proxy.__getattr__(self, name) > + > + > +def custom_proxy(value): > + if isinstance(value, parser.FieldType): > + return ProxyFieldType > + if isinstance(value, parser.Field): > + return ProxyField > + if isinstance(value, parser.Struct): > + return ProxyStruct > + if isinstance(value, parser.ConstValue): > + return ProxyConstValue > + return None > + > + > +TYPE_ENUM = { > + parser.ID_STRING: 'TType.STRING', > + parser.ID_BOOL: 'TType.BOOL', > + parser.ID_BYTE: 'TType.BYTE', > + parser.ID_I16: 'TType.I16', > + parser.ID_I32: 'TType.I32', > + parser.ID_I64: 'TType.I64', > + parser.ID_DOUBLE: 'TType.DOUBLE', > + parser.ID_MAP: 'TType.MAP', > + parser.ID_SET: 'TType.SET', > + parser.ID_LIST: 'TType.LIST', > + # TType.STRUCT and TType.I32 for enums > + } > + > + > +def gen_value(template, ob): > + buf = cStringIO.StringIO() > + template.generate(buf, ob) > + return buf.getvalue() > + > + > +if __name__ == '__main__': > + import sys > + program = parser.parse(open(sys.argv[1]).read()) > + generate(program) > > Added: incubator/thrift/branches/py-compiler/compiler/py/src/parser.py > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/parser.py?rev=739520&view=auto > ============================================================================== > --- incubator/thrift/branches/py-compiler/compiler/py/src/parser.py (added) > +++ incubator/thrift/branches/py-compiler/compiler/py/src/parser.py Sat Jan > 31 07:40:26 2009 > @@ -0,0 +1,497 @@ > +# > +# simple parser for Thrift. > +# > + > +# Note: the scanner module is designed to allow this wildcard import > +from scanner import * > + > + > +def parse(contents): > + > + scanner = Scanner(contents) > + program = Program() > + > + while True: > + > + t = scanner.get() > + if t is None: > + return program > + > + ### delta: we don't enforce HeaderList followed by DefinitionList > + ### delta: deprecated namespaces are not parsed > + > + if t == ID_INCLUDE: > + inc = scanner.value_of(TYPE_LIT) > + program.add_include(inc) > + elif t == ID_NAMESPACE: > + lang = scanner.value_of(TYPE_ID) > + ns = scanner.value_of(TYPE_ID) > + program.add_namespace(lang, ns) > + elif t == ID_CPP_INCLUDE: > + inc = scanner.value_of(TYPE_LIT) > + program.add_cpp_include(inc) > + elif t == ID_PHP_NAMESPACE: > + ns = scanner.value_of(TYPE_ID) > + program.set_php_namespace(ns) > + elif t == ID_XSD_NAMESPACE: > + ns = scanner.value_of(TYPE_LIT) > + program.set_xsd_namespace(ns) > + elif t == ID_CONST: > + doc = scanner.doc > + ft = parse_field_type(scanner, True) > + ident = scanner.value_of(TYPE_ID) > + scanner.eat_expected(SYM_EQ) > + value = parse_const_value(scanner) > + scanner.eat_commasemi() > + program.add_const(ident, ft, value, doc) > + elif t == ID_TYPEDEF: > + doc = scanner.doc > + ft = parse_field_type(scanner, False) > + ident = scanner.value_of(TYPE_ID) > + program.add_typedef(ident, ft, doc) > + elif t == ID_ENUM: > + enum_doc = scanner.doc > + enum_ident = scanner.value_of(TYPE_ID) > + scanner.eat_expected(SYM_LBRACE) > + values = [ ] > + while True: > + t = scanner.get(eof_allowed=False) > + if t == SYM_RBRACE: > + break > + if t.ttype != TYPE_ID: > + raise ExpectedType(TYPE_ID, t.ttype, scanner.lineno) > + doc = scanner.doc > + ident = t.tvalue > + t = scanner.get(eof_allowed=False) > + if t == SYM_EQ: > + value = scanner.value_of(TYPE_INT) > + else: > + scanner.pushback(t) > + value = None > + scanner.eat_commasemi() > + values.append(EnumValue(ident, value, doc)) > + program.add_enum(enum_ident, values, enum_doc) > + elif t == ID_SENUM: > + doc = scanner.doc > + ident = scanner.value_of(TYPE_ID) > + scanner.eat_expected(SYM_LBRACE) > + values = [ ] > + while True: > + t = scanner.get(eof_allowed=False) > + if t == SYM_RBRACE: > + break > + if t.ttype != TYPE_LIT: > + raise ExpectedType(TYPE_LIT, t.ttype, scanner.lineno) > + scanner.eat_commasemi() > + values.append(t.tvalue) > + program.add_senum(ident, values, doc) > + elif t == ID_STRUCT: > + doc = scanner.doc > + ident = scanner.value_of(TYPE_ID) > + t = scanner.get(eof_allowed=False) > + if t == ID_XSD_ALL: > + xsd_all = True > + else: > + xsd_all = False > + scanner.pushback(t) > + fields = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE) > + annotations = parse_annotations(scanner) > + program.add_struct(ident, fields, annotations, doc) > + elif t == ID_EXCEPTION: > + doc = scanner.doc > + ident = scanner.value_of(TYPE_ID) > + fields = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE) > + program.add_exception(ident, fields, doc) > + elif t == ID_SERVICE: > + svc_doc = scanner.doc > + svc_ident = scanner.value_of(TYPE_ID) > + t = scanner.get(eof_allowed=False) > + if t == ID_EXTENDS: > + extends = t.tvalue > + t = scanner.get(eof_allowed=False) > + else: > + extends = None > + if t != SYM_LBRACE: > + raise ExpectedError(SYM_LBRACE, t, scanner.lineno) > + functions = [ ] > + while True: > + t = scanner.get(eof_allowed=False) > + doc = scanner.doc > + if t == SYM_RBRACE: > + break > + if t == ID_ASYNC: > + async = True > + t = scanner.get(eof_allowed=False) > + else: > + async = False > + if t == ID_VOID: > + ft = FieldType(ident=ID_VOID) > + else: > + scanner.pushback(t) > + ft = parse_field_type(scanner, True) > + ident = scanner.value_of(TYPE_ID) > + params = parse_field_list(scanner, SYM_LPAREN, SYM_RPAREN) > + t = scanner.get(eof_allowed=False) > + if t == ID_THROWS: > + throws = parse_field_list(scanner, SYM_LPAREN, SYM_RPAREN) > + else: > + throws = None > + scanner.pushback(t) > + scanner.eat_commasemi() > + functions.append(Function(ident, async, ft, params, throws, doc)) > + program.add_service(svc_ident, extends, functions, svc_doc) > + else: > + raise IncorrectSyntax(scanner.lineno) > + > + > +def parse_field_type(scanner, ident_allowed): > + ident = scanner.get_type(TYPE_ID) > + if ident in BASE_TYPES: > + return FieldType(ident=ident) > + > + cpp_type = None > + > + if ident == ID_MAP: > + t = scanner.get(eof_allowed=False) > + if t == ID_CPP_TYPE: > + cpp_type = scanner.value_of(TYPE_LITERAL) > + t = scanner.get() > + if t != SYM_LT: > + raise ExpectedError(SYM_LT, t, scanner.lineno) > + map_from = parse_field_type(scanner, True) > + scanner.eat_expected(SYM_COMMA) > + map_to = parse_field_type(scanner, True) > + scanner.eat_expected(SYM_GT) > + return FieldType(cpp_type=cpp_type, map_from=map_from, map_to=map_to, > + annotations=parse_annotations(scanner)) > + > + if ident == ID_SET: > + t = scanner.get(eof_allowed=False) > + if t == ID_CPP_TYPE: > + cpp_type = scanner.value_of(TYPE_LITERAL) > + t = scanner.get() > + if t != SYM_LT: > + raise ExpectedError(SYM_LT, t, scanner.lineno) > + set_of = parse_field_type(scanner, True) > + scanner.eat_expected(SYM_GT) > + return FieldType(cpp_type=cpp_type, set_of=set_of, > + annotations=parse_annotations(scanner)) > + > + if ident == ID_LIST: > + scanner.eat_expected(SYM_LT) > + list_of = parse_field_type(scanner, True) > + scanner.eat_expected(SYM_GT) > + t = scanner.get() > + if t == ID_CPP_TYPE: > + cpp_type = scanner.value_of(TYPE_LITERAL) > + elif t is not None: > + scanner.pushback(t) > + return FieldType(cpp_type=cpp_type, list_of=list_of, > + annotations=parse_annotations(scanner)) > + > + # random identifiers are allowed for FieldType, but not DefinitionType > + if ident_allowed: > + return FieldType(ident=ident) > + > + raise IncorrectSyntax(scanner.lineno) > + > + > +def parse_const_value(scanner): > + value = scanner.get(eof_allowed=False) > + if value.ttype in [TYPE_INT, TYPE_HEX, TYPE_DUB, TYPE_LIT, TYPE_ID]: > + return ConstValue(ConstValue.CTYPE_BASE, value) > + > + if value == SYM_LBRKT: > + values = [ ] > + while True: > + t = scanner.get(eof_allowed=False) > + if t == SYM_RBRKT: > + return ConstValue(ConstValue.CTYPE_LIST, values) > + scanner.pushback(t) > + scanner.eat_commasemi() > + values.append(parse_const_value(scanner)) > + > + if value == SYM_LBRACE: > + values = [ ] > + while True: > + t = scanner.get(eof_allowed=False) > + if t == SYM_RBRACE: > + return ConstValue(ConstValue.CTYPE_MAP, values) > + scanner.pushback(t) > + key = parse_const_value(scanner) > + scanner.eat_expected(SYM_COLON) > + value = parse_const_value(scanner) > + scanner.eat_commasemi() > + values.append(KeyValuePair(key, value)) > + > + raise IncorrectSyntax(scanner.lineno) > + > + > +def parse_field_list(scanner, start, end): > + scanner.eat_expected(start) > + > + fields = [ ] > + while True: > + t = scanner.get(eof_allowed=False) > + if t == end: > + return fields > + doc = scanner.doc > + if t.ttype == TYPE_INT: > + field_id = t.tvalue > + scanner.eat_expected(SYM_COLON) > + t = scanner.get(eof_allowed=False) > + else: > + field_id = None > + if t == ID_REQUIRED or t == ID_OPTIONAL: > + ### delta: we don't warn when this occurs in an arglist > + requiredness = t > + else: > + requiredness = None > + scanner.pushback(t) > + ft = parse_field_type(scanner, True) > + ident = scanner.value_of(TYPE_ID) > + t = scanner.get() > + if t == SYM_EQ: > + value = parse_const_value(scanner) > + t = scanner.get() > + else: > + value = None > + if t == ID_XSD_OPTIONAL: > + xsd_optional = True > + t = scanner.get() > + else: > + xsd_optional = False > + if t == ID_XSD_NILLABLE: > + xsd_nillable = True > + t = scanner.get() > + else: > + xsd_nillable = False > + if t == ID_XSD_ATTRS: > + xsd_attrs = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE) > + else: > + xsd_attrs = None > + if t is not None: > + scanner.pushback(t) > + scanner.eat_commasemi() > + fields.append(Field(ident, ft, doc, field_id, requiredness, value, > + xsd_optional, xsd_nillable, xsd_attrs)) > + > + > +def parse_annotations(scanner): > + t = scanner.get() > + if t is None: > + return None > + if t != SYM_LPAREN: > + scanner.pushback(t) > + return None > + annotations = [ ] > + while True: > + ident = scanner.value_of(TYPE_ID) > + scanner.eat_expected(SYM_EQ) > + value = scanner.value_of(TYPE_LIT) > + annotations.append(KeyValuePair(ident, value)) > + > + scanner.eat_commasemi() > + t = scanner.get() > + if t == SYM_RPAREN: > + return annotations > + scanner.pushback(t) > + > + > +class Program(object): > + def __init__(self): > + self.includes = [ ] > + self.namespaces = [ ] > + self.cpp_includes = [ ] > + self.php_namespace = None > + self.xsd_namespace = None > + self.consts = [ ] > + self.typedefs = [ ] > + self.enums = [ ] > + self.structs = [ ] > + self.exceptions = [ ] > + self.services = [ ] > + > + def add_include(self, include): > + self.includes.append(include) > + > + def add_namespace(self, lang, namespace): > + self.namespaces.append(Namespace(lang, namespace)) > + > + def add_cpp_include(self, include): > + self.cpp_includes.append(include) > + > + def set_php_namespace(self, namespace): > + self.php_namespace = namespace > + > + def set_xsd_namespace(self, namespace): > + self.xsd_namespace = namespace > + > + def add_const(self, ident, field_type, value, doc): > + self.consts.append(ConstDef(ident, field_type, value, doc)) > + > + def add_typedef(self, ident, field_type, doc): > + self.typedefs.append(Typedef(ident, field_type, doc)) > + > + def add_enum(self, ident, value, doc): > + self.enums.append(Enum(ident, value, doc)) > + > + def add_senum(self, ident, values, doc): > + self.typedefs.append(Typedef(ident, FieldType(values=values), doc)) > + > + def add_struct(self, ident, fields, annotations, doc): > + self.structs.append(Struct(ident, fields, annotations, doc)) > + > + def add_exception(self, ident, fields, doc): > + self.exceptions.append(Exception(ident, fields, doc)) > + > + def add_service(self, ident, extends, functions, doc): > + self.services.append(Service(ident, extends, functions, doc)) > + > + > +class Service(object): > + def __init__(self, ident, extends, functions, doc): > + self.ident = ident > + self.extends = extends > + self.functions = functions > + self.doc = doc > + > + > +class Function(object): > + def __init__(self, ident, async, field_type, params, throws, doc): > + self.ident = ident > + self.async = async > + self.field_type = field_type > + self.params = params > + self.throws = throws > + self.doc = doc > + > + > +class Enum(object): > + def __init__(self, ident, values, doc): > + self.ident = ident > + self.values = values > + self.doc = doc > + > + for i in range(1, len(values)): > + if values[i].value is None: > + ### keep as integer? > + values[i].value = str(int(values[i - 1].value) + 1) > + > + > +class EnumValue(object): > + def __init__(self, ident, value, doc): > + self.ident = ident > + self.value = value > + self.doc = doc > + > + > +class Field(object): > + def __init__(self, ident, field_type, doc, field_id, requiredness, value, > + xsd_optional, xsd_nillable, xsd_attrs): > + assert value is None or isinstance(value, ConstValue) > + > + self.ident = ident > + self.field_type = field_type > + self.doc = doc > + self.field_id = field_id > + self.requiredness = requiredness > + self.value = value > + self.xsd_optional = xsd_optional > + self.xsd_nillable = xsd_nillable > + self.xsd_attrs = xsd_attrs > + > + > +class FieldType(object): > + def __init__(self, ident=None, cpp_type=None, map_from=None, map_to=None, > + set_of=None, list_of=None, annotations=None, values=None): > + if map_from is not None: > + self.ident = ID_MAP > + elif set_of is not None: > + self.ident = ID_SET > + elif list_of is not None: > + self.ident = ID_LIST > + elif values is not None: > + self.ident = ID_STRING > + else: > + assert ident is not None > + self.ident = ident > + self.cpp_type = cpp_type > + self.map_from = map_from > + self.map_to = map_to > + self.set_of = set_of > + self.list_of = list_of > + self.annotations = annotations > + self.values = values > + > + > +class KeyValuePair(object): > + def __init__(self, key, value): > + self.key = key > + self.value = value > + > + > +class ConstDef(object): > + def __init__(self, ident, field_type, value, doc): > + assert isinstance(value, ConstValue) > + > + self.ident = ident > + self.field_type = field_type > + self.value = value > + self.doc = doc > + > + > +class ConstValue(object): > + CTYPE_BASE = 'base' > + CTYPE_LIST = 'list' > + CTYPE_MAP = 'map' > + > + def __init__(self, ctype, value): > + self.ctype = ctype > + self.value = value > + > + > +class Typedef(object): > + def __init__(self, ident, field_type, doc): > + self.ident = ident > + self.field_type = field_type > + self.doc = doc > + > + > +class Struct(object): > + def __init__(self, ident, fields, annotations, doc): > + self.ident = ident > + self.fields = fields > + self.annotations = annotations > + self.doc = doc > + > + > +class Exception(object): > + def __init__(self, ident, fields, doc): > + self.ident = ident > + self.fields = fields > + self.doc = doc > + > + > +class Namespace(object): > + def __init__(self, lang, namespace): > + self.lang = lang > + self.namespace = namespace > + > + > +BASE_TYPES = [ > + ID_STRING, > + ID_BINARY, > + ID_SLIST, > + ID_BOOL, > + ID_BYTE, > + ID_I16, > + ID_I32, > + ID_I64, > + ID_DOUBLE, > + ] > + > + > +if __name__ == '__main__': > + import sys > + parse(open(sys.argv[1]).read()) > > Added: incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py?rev=739520&view=auto > ============================================================================== > --- incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py (added) > +++ incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py Sat Jan > 31 07:40:26 2009 > @@ -0,0 +1,271 @@ > +# > +# simple scanner for Thrift. emits tokens. > +# > + > +__all__ = ['Scanner', 'SimpleScanner', 'Token', 'TYPE_INT', > + 'ExpectedError', 'ExpectedType', 'UnexpectedEOF', > + 'UnknownToken', 'IncorrectSyntax', > + ] > + > +import re > + > +re_int = re.compile('[+-]?[0-9]+$') # special handling > +re_hex = re.compile('0x[0-9A-Fa-f]+') > +re_dub = re.compile(r'[+-]?[0-9]*(\.[0-9]+)?([eE][+-]?[0-9]+)?') > + > +re_white = re.compile('[ \t\r\n]+') > +re_silly = re.compile(r'/\*+\*/') > +re_multi = re.compile(r'/\*[^*]/*([^*/]|[^*]/|\*[^/])*\*+/') > +re_comment = re.compile('//[^\n]*') > +re_unix = re.compile('#[^\n]*') > + > +re_doc = re.compile(r'/\*\*([^*/]|[^*]/|\*[^/])*\*+/') > + > +re_ident = re.compile('[a-zA-Z_][\.a-zA-Z_0-9]*') > +re_symbol = re.compile(r'[:;,{}()=<>\[\]]') > +re_dliteral = re.compile('"[^"]*"') > +re_sliteral = re.compile("'[^']*'") > +re_st_ident = re.compile('[a-zA-Z-][.a-zA-Z_0-9-]*') > + > +skip_re = [re_white, re_silly, re_multi, re_comment, re_unix] > + > +types = [ > + ('HEX', re_hex), # keep before re_dub > + ('DUB', re_dub), > + ('DOC', re_doc), > + ('ID', re_ident), > + ('SYM', re_symbol), > + ('LIT', re_dliteral), > + ('LIT', re_sliteral), > + ('STID', re_st_ident), > + ] > + > +for key, pattern in types: > + globals()['TYPE_' + key] = key > + __all__.append('TYPE_' + key) > +TYPE_INT = 'INT' > + > + > +class SimpleScanner(object): > + > + def __init__(self, contents): > + self.contents = contents > + self.lineno = 1 > + > + def get(self): > + """Get the next token. > + > + Consumes and returns the next token. Note that leading whitespace is > + skipped. > + > + Returns None if there are no more tokens. > + """ > + self._skip() > + > + if not self.contents: > + return None > + > + for ttype, pattern in types: > + m = pattern.match(self.contents) > + if m: > + if m.end() == 0: > + continue > + tvalue = m.group() > + if pattern is re_dub and re_int.match(tvalue): > + ttype = TYPE_INT > + elif ttype == TYPE_LIT: > + # strip quotes > + tvalue = tvalue[1:-1] > + ### fold TYPE_HEX into TYPE_INT? convert INT/DUB away from string? > + token = Token(ttype, tvalue) > + self._chomp(m.end()) > + return token > + > + raise UnknownToken(self.lineno) > + > + def _skip(self): > + "Skip over leading whitespace." > + > + while True: > + for pattern in skip_re: > + m = pattern.match(self.contents) > + if m: > + self._chomp(m.end()) > + break > + else: > + # nothing matched. all done. > + return > + > + def _chomp(self, amt): > + "Chomp AMT bytes off the front of the contents. Count newlines." > + self.lineno += self.contents[:amt].count('\n') > + self.contents = self.contents[amt:] > + > + > +class Scanner(SimpleScanner): > + def __init__(self, contents): > + SimpleScanner.__init__(self, contents) > + > + self.doc = None > + self.pending = None > + > + def get(self, eof_allowed=True): > + if self.pending is not None: > + token = self.pending > + self.pending = None > + return token > + > + self.doc = None > + while True: > + t = SimpleScanner.get(self) > + if t is None: > + if eof_allowed: > + return None > + raise UnexpectedEOF(self.lineno) > + if t.ttype != TYPE_DOC: > + #print 'TOKEN:', t > + return t > + self.doc = t > + > + def get_type(self, ttype): > + "Get the next token, ensuring it is of the given type." > + t = self.get(eof_allowed=False) > + if t.ttype != ttype: > + raise ExpectedType(ttype, t.ttype, self.lineno) > + return t > + > + def value_of(self, ttype): > + "Get the next token's value, ensuring it is of the given type." > + return self.get_type(ttype).tvalue > + > + def pushback(self, token): > + "Push a token back into the scanner; it was unused." > + assert token is not None > + assert self.pending is None > + self.pending = token > + > + def eat_commasemi(self): > + "Eat a comma or a semicolon, if present." > + t = self.get() > + if t != SYM_COMMA and t != SYM_SEMI: > + self.pushback(t) > + > + def eat_expected(self, token): > + "Eat the expected token, or raise a ExpectedError." > + t = self.get() > + if t != token: > + raise ExpectedError(token, t, self.lineno) > + > + > +class Token(object): > + def __init__(self, ttype, tvalue=None): > + self.ttype = ttype > + self.tvalue = tvalue > + > + def __str__(self): > + if self.tvalue is None: > + return 'T(%s)' % self.ttype > + return 'T(%s, "%s")' % (self.ttype, self.tvalue) > + > + def __eq__(self, other): > + return self.ttype == other.ttype and self.tvalue == other.tvalue > + > + def __ne__(self, other): > + return self.ttype != other.ttype or self.tvalue != other.tvalue > + > + def __hash__(self): > + return hash((self.ttype, self.tvalue)) > + > + > +for ident in ['namespace', > + 'cpp_namespace', > + 'cpp_include', > + 'cpp_type', > + 'java_package', > + 'cocoa_prefix', > + 'csharp_namespace', > + 'php_namespace', > + 'py_module', > + 'perl_package', > + 'ruby_namespace', > + 'smalltalk_category', > + 'smalltalk_prefix', > + 'xsd_all', > + 'xsd_optional', > + 'xsd_nillable', > + 'xsd_namespace', > + 'xsd_attrs', > + 'include', > + 'void', > + 'bool', > + 'byte', > + 'i16', > + 'i32', > + 'i64', > + 'double', > + 'string', > + 'binary', > + 'slist', > + 'senum', > + 'map', > + 'list', > + 'set', > + 'async', > + 'typedef', > + 'struct', > + 'exception', > + 'extends', > + 'throws', > + 'service', > + 'enum', > + 'const', > + 'required', > + 'optional', > + ]: > + name = 'ID_' + ident.upper() > + globals()[name] = Token(TYPE_ID, ident) > + __all__.append(name) > + > + > +for name, sym in [('COLON', ':'), > + ('SEMI', ';'), > + ('COMMA', ','), > + ('LBRACE', '{'), > + ('RBRACE', '}'), > + ('LPAREN', '('), > + ('RPAREN', ')'), > + ('LBRKT', '['), > + ('RBRKT', ']'), > + ('EQ', '='), > + ('LT', '<'), > + ('GT', '>'), > + ]: > + globals()['SYM_' + name] = Token(TYPE_SYM, sym) > + __all__.append('SYM_' + name) > + > + > +class ExpectedError(Exception): > + "Expected token was not present." > + > +class ExpectedType(Exception): > + "Expected token type was not present." > + > +class UnexpectedEOF(Exception): > + "EOF reached unexpectedly." > + > +class UnknownToken(Exception): > + "Unknown token encountered." > + > +class IncorrectSyntax(Exception): > + "Incorrect syntax encountered." > + > + > +if __name__ == '__main__': > + import sys > + > + s = Scanner(open(sys.argv[1]).read()) > + while True: > + token = s.get() > + if token is None: > + break > + print token > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt?rev=739520&view=auto > ============================================================================== > --- > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt > (added) > +++ > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt > Sat Jan 31 07:40:26 2009 > @@ -0,0 +1,33 @@ > +[if-any includes]Includes:[for includes] > + [includes][end] > + > +[end][if-any namespaces]Namespaces:[for namespaces] > + [namespaces.lang] [namespaces.namespace][end] > + > +[end][if-any cpp_includes]C++ Includes:[for cpp_includes] > + [cpp_includes][end] > + > +[end][if-any php_namespace]PHP Namespace: [php_namespace] > +[end][if-any xsd_namespace]XSD Namespace: [xsd_namespace] > +[end][if-any consts]Constants:[for consts] > + [consts.ident] [consts.field_type.serializer] = [consts.value.cvalue][end] > + > +[end][if-any typedefs]Typedefs:[for typedefs] > + [typedefs.ident] => [typedefs.field_type.serializer][end] > + > +[end][if-any enums]Enums:[for enums] > + [enums.ident] {[for enums.values] > + [enums.values.ident] = [enums.values.value],[end] > + }[end] > + > +[end][if-any structs]Structs:[for structs] > + [structs.ident] {[for structs.fields] > + [structs.fields.field_id]: [structs.fields.field_type.serializer] > [structs.fields.ident],[end] > + }[end] > + > +[end][if-any exceptions]Exceptions:[for exceptions] > + [exceptions.ident][end] > + > +[end][if-any services]Services:[for services] > + [services.ident][end] > +[end] > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt?rev=739520&view=auto > ============================================================================== > --- > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt > (added) > +++ > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt > Sat Jan 31 07:40:26 2009 > @@ -0,0 +1,3 @@ > +[is ctype "map"]{[for value][value.key.cvalue]: [value.value.cvalue],[end]}[# > +#][else][is ctype "list"][[][for value][value.cvalue],[end]][# > +#][else][value.tvalue][end][end] > \ No newline at end of file > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt?rev=739520&view=auto > ============================================================================== > --- > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt > (added) > +++ > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt > Sat Jan 31 07:40:26 2009 > @@ -0,0 +1 @@ > +deserializer > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt?rev=739520&view=auto > ============================================================================== > --- > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt > (added) > +++ > incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt > Sat Jan 31 07:40:26 2009 > @@ -0,0 +1,6 @@ > +[is ident.tvalue "map"]map<[map_from.serializer],[map_to.serializer]>[# > +][else][is ident.tvalue "set"]set<[set_of.serializer]>[# > +][else][is ident.tvalue "list"]list<[list_of.serializer]>[# > +][else][if-any values]string {[for values][values],[end]}[# > +][else][ident.tvalue][# > +][end][end][end][end] > \ No newline at end of file > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt?rev=739520&view=auto > ============================================================================== > --- incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt > (added) > +++ incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt > Sat Jan 31 07:40:26 2009 > @@ -0,0 +1,100 @@ > +# > +# Autogenerated by Thrift > +# > +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING > +# > + > +from thrift.Thrift import * > +[# ### probably need some massaging to really locate this module > +][for includes]import [includes].ttypes > +[end] > +from thrift.transport import TTransport > +from thrift.protocol import TBinaryProtocol > +try: > + from thrift.protocol import fastbinary > +except: > + fastbinary = None > + > + > +[# ### need gen_newstyle flag > +][for enums]class [enums.ident](object):[for enums.values] > + [enums.values.ident] = [enums.values.value][end] > +[end] > + > +[for consts][consts.ident] = [consts.value.cvalue] > +[end] > + > +[# ### need gen_newstyle flag > +][for structs]class [structs.ident](object): > + > + thrift_spec = ([# ### should sort fields. need None markers for gaps. > +][if-any structs.fields][for structs.sorted_fields] > + [if-any structs.sorted_fields]([structs.sorted_fields.field_id], > [structs.sorted_fields.type_enum], [# > +]'[structs.sorted_fields.ident]', [# > +]None, [# ### should have spec_args here > +][if-any > structs.sorted_fields.value][structs.sorted_fields.value.cvalue][else]None[end], > [# > +]),[else]None,[end] # ### list-index[# structs.sorted_fields.list-index][end] > + ) > +[else] thrift_spec = None > +[end] > +[if-any structs.fields] def __init__(self,[# > +][for structs.fields] [structs.fields.ident]=[# > +][if-any structs.fields.value]thrift_spec[[][structs.fields.field_id]][[]4][# > +][else]None[end],[end]): > +[for structs.fields][if-any ""][# ### complex test here > +] if [structs.fields.ident] is > self.thrift_spec[[]structs.fields.field_id][[]4]: > + [structs.fields.ident] = [structs.fields.value.cvalue] > +[end] self.[structs.fields.ident] = [structs.fields.ident] > +[end] > +[end][# close: if-any structs.fields] > + > + def read(self, iprot): > + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated [# > +]and isinstance(iprot.trans, TTransport.CReadableTransport) [# > +]and self.thrift_spec is not None and fastbinary is not None: > + fastbinary.decode_binary(self, iprot.trans, [# > +](self.__class__, self.thrift_spec)) > + return > + iprot.readStructBegin() > + while True: > + (fname, ftype, fid) = iprot.readFieldBegin() > + if ftype == TType.STOP: > + break[for structs.fields] > + [if-index structs.fields first]if[else]elif[end] fid == [# > +][structs.fields.field_id]: > + if ftype == [structs.fields.type_enum]: > + pass # deserialize > + else: > + iprot.skip(ftype)[end] > + else: > + iprot.skip(ftype) > + iprot.readFieldEnd() > + iprot.readStructEnd() > + > + def write(self, oprot): > + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated [# > +]and self.thrift_spec is not None and fastbinary is not None: > + oprot.trans.write(fastbinary.encode_binary(self, [# > +](self.__class__, self.thrift_spec))) > + return > + oprot.writeStructBegin('[structs.ident]')[for structs.fields] > + if self.[structs.fields.ident] != None: > + oprot.writeFieldBegin('[structs.fields.ident]', [# > +][structs.fields.type_enum], [structs.fields.field_id]) > + # serialize > + oprot.writeFieldEnd()[end] > + oprot.writeFieldStop() > + oprot.writeStructEnd() > + > + def __repr__(self): > + L = [[]'%s=%r' % (key, value) > + for key, value in self.__dict__.iteritems()] > + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) > + > + def __eq__(self, other): > + return isinstance(other, self.__class__) and self.__dict__ == > other.__dict__ > + > + def __ne__(self, other): > + return not (self == other) > + > +[end][# for structs] > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt?rev=739520&view=auto > ============================================================================== > --- > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt > (added) > +++ > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt > Sat Jan 31 07:40:26 2009 > @@ -0,0 +1,10 @@ > +[is ctype "map"]{ > + [for value][value.key.cvalue]: [value.value.cvalue], > +[end]}[# > +#][else][is ctype "set"]set([[] > + [for value][value.cvalue], > +[end]])[# > +#][else][is ctype "list"][[] > + [for value][value.cvalue], > +[end]][# > +#][else][value.tvalue][end][end][end] > \ No newline at end of file > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt?rev=739520&view=auto > ============================================================================== > (empty) > > Added: > incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt > URL: > http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt?rev=739520&view=auto > ============================================================================== > (empty) > >
