Oh! Neat. After I land back in California, I'll go digging around and see what I can find. Thanks for the pointer!
On Sat, Jan 31, 2009 at 09:03, David Reiss <[email protected]> wrote: > If you're interested, I think there is already a Python-based compiler > somewhere deep in the SVN history (pre open-source). > > --David > > [email protected] wrote: >> Author: gstein >> Date: Sat Jan 31 07:40:26 2009 >> New Revision: 739520 >> >> URL: http://svn.apache.org/viewvc?rev=739520&view=rev >> Log: >> Snapshot (back up) my work-in-progress before I hop on a plane. >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/gen.py >> incubator/thrift/branches/py-compiler/compiler/py/src/parser.py >> incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/ >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt >> >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt >> >> Added: incubator/thrift/branches/py-compiler/compiler/py/src/gen.py >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/gen.py?rev=739520&view=auto >> ============================================================================== >> --- incubator/thrift/branches/py-compiler/compiler/py/src/gen.py (added) >> +++ incubator/thrift/branches/py-compiler/compiler/py/src/gen.py Sat Jan 31 >> 07:40:26 2009 >> @@ -0,0 +1,169 @@ >> +# >> +# simple generator for Thrift >> +# >> + >> +import sys >> +import os >> +import cStringIO >> +import operator >> + >> +import parser >> +import ezt >> + >> + >> +### temporary >> +PATH = '/Users/gstein/src/asf/thrift/compiler/py/src/templates-py' >> +t_py = ezt.Template(os.path.join(PATH, 'py.ezt'), >> + compress_whitespace=False) >> +t_py_ser = ezt.Template(os.path.join(PATH, 'py_ser.ezt'), >> + compress_whitespace=False) >> +t_py_deser = ezt.Template(os.path.join(PATH, 'py_deser.ezt'), >> + compress_whitespace=False) >> +t_py_cvalue = ezt.Template(os.path.join(PATH, 'py_cvalue.ezt'), >> + compress_whitespace=False) >> + >> + >> +def generate(program): >> + t_py.generate(sys.stdout, Proxy(program)) >> + >> + >> +class AutoVars(object): >> + def __init__(self): >> + self._counter = 0 >> + self._mapping = { } >> + self._saved = [ ] >> + >> + def open_context(self): >> + self._saved.append(self._mapping) >> + self._mapping = { } >> + >> + def close_context(self): >> + self._mapping = self._saved.pop() >> + >> + def __getattr__(self, name): >> + if name.startswith('__'): >> + raise AttributeError(name) >> + >> + if name in self._mapping: >> + return self._mapping[name] >> + var = '%s%d' % (name, self._counter) >> + self._counter += 1 >> + self._mapping[name] = var >> + return var >> + >> + >> +class Proxy(object): >> + def __init__(self, ob): >> + self._ob = ob >> + >> + for name, value in vars(ob).items(): >> + proxy = custom_proxy(value) >> + if proxy: >> + value = proxy(value) >> + elif isinstance(value, list) and value: >> + # lists are homogenous, so check the first item >> + proxy = custom_proxy(value[0]) >> + if proxy: >> + value = [proxy(ob) for ob in value] >> + elif hasattr(value[0], '__dict__'): >> + value = [Proxy(ob) for ob in value] >> + setattr(self, name, value) >> + >> + def __getattr__(self, name): >> + if name == 'auto': >> + return g_auto >> + raise AttributeError(name) >> + >> + >> +class ProxyFieldType(Proxy): >> + def __getattr__(self, name): >> + if name == 'serializer': >> + return Subtemplate(t_py_ser, self) >> + if name == 'deserializer': >> + return Subtemplate(t_py_deser, self) >> + return Proxy.__getattr__(self, name) >> + >> + >> +class Subtemplate(object): >> + def __init__(self, template, data): >> + self._template = template >> + self._data = data >> + >> + def __getattr__(self, name): >> + # jam the name of the result variable into the data params >> + self._data.result_var = getattr(g_auto, name) >> + >> + # use a new variable context for this template generation >> + g_auto.open_context() >> + value = gen_value(self._template, self._data) >> + g_auto.close_context() >> + >> + return value >> + >> + >> +class ProxyField(Proxy): >> + def __getattr__(self, name): >> + if name == 'type_enum': >> + return TYPE_ENUM.get(self._ob.field_type.ident, >> + self._ob.field_type.ident.tvalue) >> + return Proxy.__getattr__(self, name) >> + >> + >> +class ProxyStruct(Proxy): >> + def __getattr__(self, name): >> + if name == 'sorted_fields': >> + highest = max(int(f.field_id or -1) for f in self._ob.fields) >> + fields = [None] * (highest + 1) >> + for field in self._ob.fields: >> + if field.field_id: >> + id = int(field.field_id) >> + if id > 0: >> + fields[id] = ProxyField(field) >> + return fields >> + return Proxy.__getattr__(self, name) >> + >> + >> +class ProxyConstValue(Proxy): >> + def __getattr__(self, name): >> + if name == 'cvalue': >> + return gen_value(t_py_cvalue, self) >> + return Proxy.__getattr__(self, name) >> + >> + >> +def custom_proxy(value): >> + if isinstance(value, parser.FieldType): >> + return ProxyFieldType >> + if isinstance(value, parser.Field): >> + return ProxyField >> + if isinstance(value, parser.Struct): >> + return ProxyStruct >> + if isinstance(value, parser.ConstValue): >> + return ProxyConstValue >> + return None >> + >> + >> +TYPE_ENUM = { >> + parser.ID_STRING: 'TType.STRING', >> + parser.ID_BOOL: 'TType.BOOL', >> + parser.ID_BYTE: 'TType.BYTE', >> + parser.ID_I16: 'TType.I16', >> + parser.ID_I32: 'TType.I32', >> + parser.ID_I64: 'TType.I64', >> + parser.ID_DOUBLE: 'TType.DOUBLE', >> + parser.ID_MAP: 'TType.MAP', >> + parser.ID_SET: 'TType.SET', >> + parser.ID_LIST: 'TType.LIST', >> + # TType.STRUCT and TType.I32 for enums >> + } >> + >> + >> +def gen_value(template, ob): >> + buf = cStringIO.StringIO() >> + template.generate(buf, ob) >> + return buf.getvalue() >> + >> + >> +if __name__ == '__main__': >> + import sys >> + program = parser.parse(open(sys.argv[1]).read()) >> + generate(program) >> >> Added: incubator/thrift/branches/py-compiler/compiler/py/src/parser.py >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/parser.py?rev=739520&view=auto >> ============================================================================== >> --- incubator/thrift/branches/py-compiler/compiler/py/src/parser.py (added) >> +++ incubator/thrift/branches/py-compiler/compiler/py/src/parser.py Sat Jan >> 31 07:40:26 2009 >> @@ -0,0 +1,497 @@ >> +# >> +# simple parser for Thrift. >> +# >> + >> +# Note: the scanner module is designed to allow this wildcard import >> +from scanner import * >> + >> + >> +def parse(contents): >> + >> + scanner = Scanner(contents) >> + program = Program() >> + >> + while True: >> + >> + t = scanner.get() >> + if t is None: >> + return program >> + >> + ### delta: we don't enforce HeaderList followed by DefinitionList >> + ### delta: deprecated namespaces are not parsed >> + >> + if t == ID_INCLUDE: >> + inc = scanner.value_of(TYPE_LIT) >> + program.add_include(inc) >> + elif t == ID_NAMESPACE: >> + lang = scanner.value_of(TYPE_ID) >> + ns = scanner.value_of(TYPE_ID) >> + program.add_namespace(lang, ns) >> + elif t == ID_CPP_INCLUDE: >> + inc = scanner.value_of(TYPE_LIT) >> + program.add_cpp_include(inc) >> + elif t == ID_PHP_NAMESPACE: >> + ns = scanner.value_of(TYPE_ID) >> + program.set_php_namespace(ns) >> + elif t == ID_XSD_NAMESPACE: >> + ns = scanner.value_of(TYPE_LIT) >> + program.set_xsd_namespace(ns) >> + elif t == ID_CONST: >> + doc = scanner.doc >> + ft = parse_field_type(scanner, True) >> + ident = scanner.value_of(TYPE_ID) >> + scanner.eat_expected(SYM_EQ) >> + value = parse_const_value(scanner) >> + scanner.eat_commasemi() >> + program.add_const(ident, ft, value, doc) >> + elif t == ID_TYPEDEF: >> + doc = scanner.doc >> + ft = parse_field_type(scanner, False) >> + ident = scanner.value_of(TYPE_ID) >> + program.add_typedef(ident, ft, doc) >> + elif t == ID_ENUM: >> + enum_doc = scanner.doc >> + enum_ident = scanner.value_of(TYPE_ID) >> + scanner.eat_expected(SYM_LBRACE) >> + values = [ ] >> + while True: >> + t = scanner.get(eof_allowed=False) >> + if t == SYM_RBRACE: >> + break >> + if t.ttype != TYPE_ID: >> + raise ExpectedType(TYPE_ID, t.ttype, scanner.lineno) >> + doc = scanner.doc >> + ident = t.tvalue >> + t = scanner.get(eof_allowed=False) >> + if t == SYM_EQ: >> + value = scanner.value_of(TYPE_INT) >> + else: >> + scanner.pushback(t) >> + value = None >> + scanner.eat_commasemi() >> + values.append(EnumValue(ident, value, doc)) >> + program.add_enum(enum_ident, values, enum_doc) >> + elif t == ID_SENUM: >> + doc = scanner.doc >> + ident = scanner.value_of(TYPE_ID) >> + scanner.eat_expected(SYM_LBRACE) >> + values = [ ] >> + while True: >> + t = scanner.get(eof_allowed=False) >> + if t == SYM_RBRACE: >> + break >> + if t.ttype != TYPE_LIT: >> + raise ExpectedType(TYPE_LIT, t.ttype, scanner.lineno) >> + scanner.eat_commasemi() >> + values.append(t.tvalue) >> + program.add_senum(ident, values, doc) >> + elif t == ID_STRUCT: >> + doc = scanner.doc >> + ident = scanner.value_of(TYPE_ID) >> + t = scanner.get(eof_allowed=False) >> + if t == ID_XSD_ALL: >> + xsd_all = True >> + else: >> + xsd_all = False >> + scanner.pushback(t) >> + fields = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE) >> + annotations = parse_annotations(scanner) >> + program.add_struct(ident, fields, annotations, doc) >> + elif t == ID_EXCEPTION: >> + doc = scanner.doc >> + ident = scanner.value_of(TYPE_ID) >> + fields = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE) >> + program.add_exception(ident, fields, doc) >> + elif t == ID_SERVICE: >> + svc_doc = scanner.doc >> + svc_ident = scanner.value_of(TYPE_ID) >> + t = scanner.get(eof_allowed=False) >> + if t == ID_EXTENDS: >> + extends = t.tvalue >> + t = scanner.get(eof_allowed=False) >> + else: >> + extends = None >> + if t != SYM_LBRACE: >> + raise ExpectedError(SYM_LBRACE, t, scanner.lineno) >> + functions = [ ] >> + while True: >> + t = scanner.get(eof_allowed=False) >> + doc = scanner.doc >> + if t == SYM_RBRACE: >> + break >> + if t == ID_ASYNC: >> + async = True >> + t = scanner.get(eof_allowed=False) >> + else: >> + async = False >> + if t == ID_VOID: >> + ft = FieldType(ident=ID_VOID) >> + else: >> + scanner.pushback(t) >> + ft = parse_field_type(scanner, True) >> + ident = scanner.value_of(TYPE_ID) >> + params = parse_field_list(scanner, SYM_LPAREN, SYM_RPAREN) >> + t = scanner.get(eof_allowed=False) >> + if t == ID_THROWS: >> + throws = parse_field_list(scanner, SYM_LPAREN, SYM_RPAREN) >> + else: >> + throws = None >> + scanner.pushback(t) >> + scanner.eat_commasemi() >> + functions.append(Function(ident, async, ft, params, throws, doc)) >> + program.add_service(svc_ident, extends, functions, svc_doc) >> + else: >> + raise IncorrectSyntax(scanner.lineno) >> + >> + >> +def parse_field_type(scanner, ident_allowed): >> + ident = scanner.get_type(TYPE_ID) >> + if ident in BASE_TYPES: >> + return FieldType(ident=ident) >> + >> + cpp_type = None >> + >> + if ident == ID_MAP: >> + t = scanner.get(eof_allowed=False) >> + if t == ID_CPP_TYPE: >> + cpp_type = scanner.value_of(TYPE_LITERAL) >> + t = scanner.get() >> + if t != SYM_LT: >> + raise ExpectedError(SYM_LT, t, scanner.lineno) >> + map_from = parse_field_type(scanner, True) >> + scanner.eat_expected(SYM_COMMA) >> + map_to = parse_field_type(scanner, True) >> + scanner.eat_expected(SYM_GT) >> + return FieldType(cpp_type=cpp_type, map_from=map_from, map_to=map_to, >> + annotations=parse_annotations(scanner)) >> + >> + if ident == ID_SET: >> + t = scanner.get(eof_allowed=False) >> + if t == ID_CPP_TYPE: >> + cpp_type = scanner.value_of(TYPE_LITERAL) >> + t = scanner.get() >> + if t != SYM_LT: >> + raise ExpectedError(SYM_LT, t, scanner.lineno) >> + set_of = parse_field_type(scanner, True) >> + scanner.eat_expected(SYM_GT) >> + return FieldType(cpp_type=cpp_type, set_of=set_of, >> + annotations=parse_annotations(scanner)) >> + >> + if ident == ID_LIST: >> + scanner.eat_expected(SYM_LT) >> + list_of = parse_field_type(scanner, True) >> + scanner.eat_expected(SYM_GT) >> + t = scanner.get() >> + if t == ID_CPP_TYPE: >> + cpp_type = scanner.value_of(TYPE_LITERAL) >> + elif t is not None: >> + scanner.pushback(t) >> + return FieldType(cpp_type=cpp_type, list_of=list_of, >> + annotations=parse_annotations(scanner)) >> + >> + # random identifiers are allowed for FieldType, but not DefinitionType >> + if ident_allowed: >> + return FieldType(ident=ident) >> + >> + raise IncorrectSyntax(scanner.lineno) >> + >> + >> +def parse_const_value(scanner): >> + value = scanner.get(eof_allowed=False) >> + if value.ttype in [TYPE_INT, TYPE_HEX, TYPE_DUB, TYPE_LIT, TYPE_ID]: >> + return ConstValue(ConstValue.CTYPE_BASE, value) >> + >> + if value == SYM_LBRKT: >> + values = [ ] >> + while True: >> + t = scanner.get(eof_allowed=False) >> + if t == SYM_RBRKT: >> + return ConstValue(ConstValue.CTYPE_LIST, values) >> + scanner.pushback(t) >> + scanner.eat_commasemi() >> + values.append(parse_const_value(scanner)) >> + >> + if value == SYM_LBRACE: >> + values = [ ] >> + while True: >> + t = scanner.get(eof_allowed=False) >> + if t == SYM_RBRACE: >> + return ConstValue(ConstValue.CTYPE_MAP, values) >> + scanner.pushback(t) >> + key = parse_const_value(scanner) >> + scanner.eat_expected(SYM_COLON) >> + value = parse_const_value(scanner) >> + scanner.eat_commasemi() >> + values.append(KeyValuePair(key, value)) >> + >> + raise IncorrectSyntax(scanner.lineno) >> + >> + >> +def parse_field_list(scanner, start, end): >> + scanner.eat_expected(start) >> + >> + fields = [ ] >> + while True: >> + t = scanner.get(eof_allowed=False) >> + if t == end: >> + return fields >> + doc = scanner.doc >> + if t.ttype == TYPE_INT: >> + field_id = t.tvalue >> + scanner.eat_expected(SYM_COLON) >> + t = scanner.get(eof_allowed=False) >> + else: >> + field_id = None >> + if t == ID_REQUIRED or t == ID_OPTIONAL: >> + ### delta: we don't warn when this occurs in an arglist >> + requiredness = t >> + else: >> + requiredness = None >> + scanner.pushback(t) >> + ft = parse_field_type(scanner, True) >> + ident = scanner.value_of(TYPE_ID) >> + t = scanner.get() >> + if t == SYM_EQ: >> + value = parse_const_value(scanner) >> + t = scanner.get() >> + else: >> + value = None >> + if t == ID_XSD_OPTIONAL: >> + xsd_optional = True >> + t = scanner.get() >> + else: >> + xsd_optional = False >> + if t == ID_XSD_NILLABLE: >> + xsd_nillable = True >> + t = scanner.get() >> + else: >> + xsd_nillable = False >> + if t == ID_XSD_ATTRS: >> + xsd_attrs = parse_field_list(scanner, SYM_LBRACE, SYM_RBRACE) >> + else: >> + xsd_attrs = None >> + if t is not None: >> + scanner.pushback(t) >> + scanner.eat_commasemi() >> + fields.append(Field(ident, ft, doc, field_id, requiredness, value, >> + xsd_optional, xsd_nillable, xsd_attrs)) >> + >> + >> +def parse_annotations(scanner): >> + t = scanner.get() >> + if t is None: >> + return None >> + if t != SYM_LPAREN: >> + scanner.pushback(t) >> + return None >> + annotations = [ ] >> + while True: >> + ident = scanner.value_of(TYPE_ID) >> + scanner.eat_expected(SYM_EQ) >> + value = scanner.value_of(TYPE_LIT) >> + annotations.append(KeyValuePair(ident, value)) >> + >> + scanner.eat_commasemi() >> + t = scanner.get() >> + if t == SYM_RPAREN: >> + return annotations >> + scanner.pushback(t) >> + >> + >> +class Program(object): >> + def __init__(self): >> + self.includes = [ ] >> + self.namespaces = [ ] >> + self.cpp_includes = [ ] >> + self.php_namespace = None >> + self.xsd_namespace = None >> + self.consts = [ ] >> + self.typedefs = [ ] >> + self.enums = [ ] >> + self.structs = [ ] >> + self.exceptions = [ ] >> + self.services = [ ] >> + >> + def add_include(self, include): >> + self.includes.append(include) >> + >> + def add_namespace(self, lang, namespace): >> + self.namespaces.append(Namespace(lang, namespace)) >> + >> + def add_cpp_include(self, include): >> + self.cpp_includes.append(include) >> + >> + def set_php_namespace(self, namespace): >> + self.php_namespace = namespace >> + >> + def set_xsd_namespace(self, namespace): >> + self.xsd_namespace = namespace >> + >> + def add_const(self, ident, field_type, value, doc): >> + self.consts.append(ConstDef(ident, field_type, value, doc)) >> + >> + def add_typedef(self, ident, field_type, doc): >> + self.typedefs.append(Typedef(ident, field_type, doc)) >> + >> + def add_enum(self, ident, value, doc): >> + self.enums.append(Enum(ident, value, doc)) >> + >> + def add_senum(self, ident, values, doc): >> + self.typedefs.append(Typedef(ident, FieldType(values=values), doc)) >> + >> + def add_struct(self, ident, fields, annotations, doc): >> + self.structs.append(Struct(ident, fields, annotations, doc)) >> + >> + def add_exception(self, ident, fields, doc): >> + self.exceptions.append(Exception(ident, fields, doc)) >> + >> + def add_service(self, ident, extends, functions, doc): >> + self.services.append(Service(ident, extends, functions, doc)) >> + >> + >> +class Service(object): >> + def __init__(self, ident, extends, functions, doc): >> + self.ident = ident >> + self.extends = extends >> + self.functions = functions >> + self.doc = doc >> + >> + >> +class Function(object): >> + def __init__(self, ident, async, field_type, params, throws, doc): >> + self.ident = ident >> + self.async = async >> + self.field_type = field_type >> + self.params = params >> + self.throws = throws >> + self.doc = doc >> + >> + >> +class Enum(object): >> + def __init__(self, ident, values, doc): >> + self.ident = ident >> + self.values = values >> + self.doc = doc >> + >> + for i in range(1, len(values)): >> + if values[i].value is None: >> + ### keep as integer? >> + values[i].value = str(int(values[i - 1].value) + 1) >> + >> + >> +class EnumValue(object): >> + def __init__(self, ident, value, doc): >> + self.ident = ident >> + self.value = value >> + self.doc = doc >> + >> + >> +class Field(object): >> + def __init__(self, ident, field_type, doc, field_id, requiredness, value, >> + xsd_optional, xsd_nillable, xsd_attrs): >> + assert value is None or isinstance(value, ConstValue) >> + >> + self.ident = ident >> + self.field_type = field_type >> + self.doc = doc >> + self.field_id = field_id >> + self.requiredness = requiredness >> + self.value = value >> + self.xsd_optional = xsd_optional >> + self.xsd_nillable = xsd_nillable >> + self.xsd_attrs = xsd_attrs >> + >> + >> +class FieldType(object): >> + def __init__(self, ident=None, cpp_type=None, map_from=None, map_to=None, >> + set_of=None, list_of=None, annotations=None, values=None): >> + if map_from is not None: >> + self.ident = ID_MAP >> + elif set_of is not None: >> + self.ident = ID_SET >> + elif list_of is not None: >> + self.ident = ID_LIST >> + elif values is not None: >> + self.ident = ID_STRING >> + else: >> + assert ident is not None >> + self.ident = ident >> + self.cpp_type = cpp_type >> + self.map_from = map_from >> + self.map_to = map_to >> + self.set_of = set_of >> + self.list_of = list_of >> + self.annotations = annotations >> + self.values = values >> + >> + >> +class KeyValuePair(object): >> + def __init__(self, key, value): >> + self.key = key >> + self.value = value >> + >> + >> +class ConstDef(object): >> + def __init__(self, ident, field_type, value, doc): >> + assert isinstance(value, ConstValue) >> + >> + self.ident = ident >> + self.field_type = field_type >> + self.value = value >> + self.doc = doc >> + >> + >> +class ConstValue(object): >> + CTYPE_BASE = 'base' >> + CTYPE_LIST = 'list' >> + CTYPE_MAP = 'map' >> + >> + def __init__(self, ctype, value): >> + self.ctype = ctype >> + self.value = value >> + >> + >> +class Typedef(object): >> + def __init__(self, ident, field_type, doc): >> + self.ident = ident >> + self.field_type = field_type >> + self.doc = doc >> + >> + >> +class Struct(object): >> + def __init__(self, ident, fields, annotations, doc): >> + self.ident = ident >> + self.fields = fields >> + self.annotations = annotations >> + self.doc = doc >> + >> + >> +class Exception(object): >> + def __init__(self, ident, fields, doc): >> + self.ident = ident >> + self.fields = fields >> + self.doc = doc >> + >> + >> +class Namespace(object): >> + def __init__(self, lang, namespace): >> + self.lang = lang >> + self.namespace = namespace >> + >> + >> +BASE_TYPES = [ >> + ID_STRING, >> + ID_BINARY, >> + ID_SLIST, >> + ID_BOOL, >> + ID_BYTE, >> + ID_I16, >> + ID_I32, >> + ID_I64, >> + ID_DOUBLE, >> + ] >> + >> + >> +if __name__ == '__main__': >> + import sys >> + parse(open(sys.argv[1]).read()) >> >> Added: incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py?rev=739520&view=auto >> ============================================================================== >> --- incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py (added) >> +++ incubator/thrift/branches/py-compiler/compiler/py/src/scanner.py Sat Jan >> 31 07:40:26 2009 >> @@ -0,0 +1,271 @@ >> +# >> +# simple scanner for Thrift. emits tokens. >> +# >> + >> +__all__ = ['Scanner', 'SimpleScanner', 'Token', 'TYPE_INT', >> + 'ExpectedError', 'ExpectedType', 'UnexpectedEOF', >> + 'UnknownToken', 'IncorrectSyntax', >> + ] >> + >> +import re >> + >> +re_int = re.compile('[+-]?[0-9]+$') # special handling >> +re_hex = re.compile('0x[0-9A-Fa-f]+') >> +re_dub = re.compile(r'[+-]?[0-9]*(\.[0-9]+)?([eE][+-]?[0-9]+)?') >> + >> +re_white = re.compile('[ \t\r\n]+') >> +re_silly = re.compile(r'/\*+\*/') >> +re_multi = re.compile(r'/\*[^*]/*([^*/]|[^*]/|\*[^/])*\*+/') >> +re_comment = re.compile('//[^\n]*') >> +re_unix = re.compile('#[^\n]*') >> + >> +re_doc = re.compile(r'/\*\*([^*/]|[^*]/|\*[^/])*\*+/') >> + >> +re_ident = re.compile('[a-zA-Z_][\.a-zA-Z_0-9]*') >> +re_symbol = re.compile(r'[:;,{}()=<>\[\]]') >> +re_dliteral = re.compile('"[^"]*"') >> +re_sliteral = re.compile("'[^']*'") >> +re_st_ident = re.compile('[a-zA-Z-][.a-zA-Z_0-9-]*') >> + >> +skip_re = [re_white, re_silly, re_multi, re_comment, re_unix] >> + >> +types = [ >> + ('HEX', re_hex), # keep before re_dub >> + ('DUB', re_dub), >> + ('DOC', re_doc), >> + ('ID', re_ident), >> + ('SYM', re_symbol), >> + ('LIT', re_dliteral), >> + ('LIT', re_sliteral), >> + ('STID', re_st_ident), >> + ] >> + >> +for key, pattern in types: >> + globals()['TYPE_' + key] = key >> + __all__.append('TYPE_' + key) >> +TYPE_INT = 'INT' >> + >> + >> +class SimpleScanner(object): >> + >> + def __init__(self, contents): >> + self.contents = contents >> + self.lineno = 1 >> + >> + def get(self): >> + """Get the next token. >> + >> + Consumes and returns the next token. Note that leading whitespace is >> + skipped. >> + >> + Returns None if there are no more tokens. >> + """ >> + self._skip() >> + >> + if not self.contents: >> + return None >> + >> + for ttype, pattern in types: >> + m = pattern.match(self.contents) >> + if m: >> + if m.end() == 0: >> + continue >> + tvalue = m.group() >> + if pattern is re_dub and re_int.match(tvalue): >> + ttype = TYPE_INT >> + elif ttype == TYPE_LIT: >> + # strip quotes >> + tvalue = tvalue[1:-1] >> + ### fold TYPE_HEX into TYPE_INT? convert INT/DUB away from string? >> + token = Token(ttype, tvalue) >> + self._chomp(m.end()) >> + return token >> + >> + raise UnknownToken(self.lineno) >> + >> + def _skip(self): >> + "Skip over leading whitespace." >> + >> + while True: >> + for pattern in skip_re: >> + m = pattern.match(self.contents) >> + if m: >> + self._chomp(m.end()) >> + break >> + else: >> + # nothing matched. all done. >> + return >> + >> + def _chomp(self, amt): >> + "Chomp AMT bytes off the front of the contents. Count newlines." >> + self.lineno += self.contents[:amt].count('\n') >> + self.contents = self.contents[amt:] >> + >> + >> +class Scanner(SimpleScanner): >> + def __init__(self, contents): >> + SimpleScanner.__init__(self, contents) >> + >> + self.doc = None >> + self.pending = None >> + >> + def get(self, eof_allowed=True): >> + if self.pending is not None: >> + token = self.pending >> + self.pending = None >> + return token >> + >> + self.doc = None >> + while True: >> + t = SimpleScanner.get(self) >> + if t is None: >> + if eof_allowed: >> + return None >> + raise UnexpectedEOF(self.lineno) >> + if t.ttype != TYPE_DOC: >> + #print 'TOKEN:', t >> + return t >> + self.doc = t >> + >> + def get_type(self, ttype): >> + "Get the next token, ensuring it is of the given type." >> + t = self.get(eof_allowed=False) >> + if t.ttype != ttype: >> + raise ExpectedType(ttype, t.ttype, self.lineno) >> + return t >> + >> + def value_of(self, ttype): >> + "Get the next token's value, ensuring it is of the given type." >> + return self.get_type(ttype).tvalue >> + >> + def pushback(self, token): >> + "Push a token back into the scanner; it was unused." >> + assert token is not None >> + assert self.pending is None >> + self.pending = token >> + >> + def eat_commasemi(self): >> + "Eat a comma or a semicolon, if present." >> + t = self.get() >> + if t != SYM_COMMA and t != SYM_SEMI: >> + self.pushback(t) >> + >> + def eat_expected(self, token): >> + "Eat the expected token, or raise a ExpectedError." >> + t = self.get() >> + if t != token: >> + raise ExpectedError(token, t, self.lineno) >> + >> + >> +class Token(object): >> + def __init__(self, ttype, tvalue=None): >> + self.ttype = ttype >> + self.tvalue = tvalue >> + >> + def __str__(self): >> + if self.tvalue is None: >> + return 'T(%s)' % self.ttype >> + return 'T(%s, "%s")' % (self.ttype, self.tvalue) >> + >> + def __eq__(self, other): >> + return self.ttype == other.ttype and self.tvalue == other.tvalue >> + >> + def __ne__(self, other): >> + return self.ttype != other.ttype or self.tvalue != other.tvalue >> + >> + def __hash__(self): >> + return hash((self.ttype, self.tvalue)) >> + >> + >> +for ident in ['namespace', >> + 'cpp_namespace', >> + 'cpp_include', >> + 'cpp_type', >> + 'java_package', >> + 'cocoa_prefix', >> + 'csharp_namespace', >> + 'php_namespace', >> + 'py_module', >> + 'perl_package', >> + 'ruby_namespace', >> + 'smalltalk_category', >> + 'smalltalk_prefix', >> + 'xsd_all', >> + 'xsd_optional', >> + 'xsd_nillable', >> + 'xsd_namespace', >> + 'xsd_attrs', >> + 'include', >> + 'void', >> + 'bool', >> + 'byte', >> + 'i16', >> + 'i32', >> + 'i64', >> + 'double', >> + 'string', >> + 'binary', >> + 'slist', >> + 'senum', >> + 'map', >> + 'list', >> + 'set', >> + 'async', >> + 'typedef', >> + 'struct', >> + 'exception', >> + 'extends', >> + 'throws', >> + 'service', >> + 'enum', >> + 'const', >> + 'required', >> + 'optional', >> + ]: >> + name = 'ID_' + ident.upper() >> + globals()[name] = Token(TYPE_ID, ident) >> + __all__.append(name) >> + >> + >> +for name, sym in [('COLON', ':'), >> + ('SEMI', ';'), >> + ('COMMA', ','), >> + ('LBRACE', '{'), >> + ('RBRACE', '}'), >> + ('LPAREN', '('), >> + ('RPAREN', ')'), >> + ('LBRKT', '['), >> + ('RBRKT', ']'), >> + ('EQ', '='), >> + ('LT', '<'), >> + ('GT', '>'), >> + ]: >> + globals()['SYM_' + name] = Token(TYPE_SYM, sym) >> + __all__.append('SYM_' + name) >> + >> + >> +class ExpectedError(Exception): >> + "Expected token was not present." >> + >> +class ExpectedType(Exception): >> + "Expected token type was not present." >> + >> +class UnexpectedEOF(Exception): >> + "EOF reached unexpectedly." >> + >> +class UnknownToken(Exception): >> + "Unknown token encountered." >> + >> +class IncorrectSyntax(Exception): >> + "Incorrect syntax encountered." >> + >> + >> +if __name__ == '__main__': >> + import sys >> + >> + s = Scanner(open(sys.argv[1]).read()) >> + while True: >> + token = s.get() >> + if token is None: >> + break >> + print token >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt?rev=739520&view=auto >> ============================================================================== >> --- >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt >> (added) >> +++ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py.ezt >> Sat Jan 31 07:40:26 2009 >> @@ -0,0 +1,33 @@ >> +[if-any includes]Includes:[for includes] >> + [includes][end] >> + >> +[end][if-any namespaces]Namespaces:[for namespaces] >> + [namespaces.lang] [namespaces.namespace][end] >> + >> +[end][if-any cpp_includes]C++ Includes:[for cpp_includes] >> + [cpp_includes][end] >> + >> +[end][if-any php_namespace]PHP Namespace: [php_namespace] >> +[end][if-any xsd_namespace]XSD Namespace: [xsd_namespace] >> +[end][if-any consts]Constants:[for consts] >> + [consts.ident] [consts.field_type.serializer] = [consts.value.cvalue][end] >> + >> +[end][if-any typedefs]Typedefs:[for typedefs] >> + [typedefs.ident] => [typedefs.field_type.serializer][end] >> + >> +[end][if-any enums]Enums:[for enums] >> + [enums.ident] {[for enums.values] >> + [enums.values.ident] = [enums.values.value],[end] >> + }[end] >> + >> +[end][if-any structs]Structs:[for structs] >> + [structs.ident] {[for structs.fields] >> + [structs.fields.field_id]: [structs.fields.field_type.serializer] >> [structs.fields.ident],[end] >> + }[end] >> + >> +[end][if-any exceptions]Exceptions:[for exceptions] >> + [exceptions.ident][end] >> + >> +[end][if-any services]Services:[for services] >> + [services.ident][end] >> +[end] >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt?rev=739520&view=auto >> ============================================================================== >> --- >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt >> (added) >> +++ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_cvalue.ezt >> Sat Jan 31 07:40:26 2009 >> @@ -0,0 +1,3 @@ >> +[is ctype "map"]{[for value][value.key.cvalue]: >> [value.value.cvalue],[end]}[# >> +#][else][is ctype "list"][[][for value][value.cvalue],[end]][# >> +#][else][value.tvalue][end][end] >> \ No newline at end of file >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt?rev=739520&view=auto >> ============================================================================== >> --- >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt >> (added) >> +++ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_deser.ezt >> Sat Jan 31 07:40:26 2009 >> @@ -0,0 +1 @@ >> +deserializer >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt?rev=739520&view=auto >> ============================================================================== >> --- >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt >> (added) >> +++ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-plain/py_ser.ezt >> Sat Jan 31 07:40:26 2009 >> @@ -0,0 +1,6 @@ >> +[is ident.tvalue "map"]map<[map_from.serializer],[map_to.serializer]>[# >> +][else][is ident.tvalue "set"]set<[set_of.serializer]>[# >> +][else][is ident.tvalue "list"]list<[list_of.serializer]>[# >> +][else][if-any values]string {[for values][values],[end]}[# >> +][else][ident.tvalue][# >> +][end][end][end][end] >> \ No newline at end of file >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt?rev=739520&view=auto >> ============================================================================== >> --- >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt >> (added) >> +++ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py.ezt >> Sat Jan 31 07:40:26 2009 >> @@ -0,0 +1,100 @@ >> +# >> +# Autogenerated by Thrift >> +# >> +# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING >> +# >> + >> +from thrift.Thrift import * >> +[# ### probably need some massaging to really locate this module >> +][for includes]import [includes].ttypes >> +[end] >> +from thrift.transport import TTransport >> +from thrift.protocol import TBinaryProtocol >> +try: >> + from thrift.protocol import fastbinary >> +except: >> + fastbinary = None >> + >> + >> +[# ### need gen_newstyle flag >> +][for enums]class [enums.ident](object):[for enums.values] >> + [enums.values.ident] = [enums.values.value][end] >> +[end] >> + >> +[for consts][consts.ident] = [consts.value.cvalue] >> +[end] >> + >> +[# ### need gen_newstyle flag >> +][for structs]class [structs.ident](object): >> + >> + thrift_spec = ([# ### should sort fields. need None markers for gaps. >> +][if-any structs.fields][for structs.sorted_fields] >> + [if-any structs.sorted_fields]([structs.sorted_fields.field_id], >> [structs.sorted_fields.type_enum], [# >> +]'[structs.sorted_fields.ident]', [# >> +]None, [# ### should have spec_args here >> +][if-any >> structs.sorted_fields.value][structs.sorted_fields.value.cvalue][else]None[end], >> [# >> +]),[else]None,[end] # ### list-index[# >> structs.sorted_fields.list-index][end] >> + ) >> +[else] thrift_spec = None >> +[end] >> +[if-any structs.fields] def __init__(self,[# >> +][for structs.fields] [structs.fields.ident]=[# >> +][if-any >> structs.fields.value]thrift_spec[[][structs.fields.field_id]][[]4][# >> +][else]None[end],[end]): >> +[for structs.fields][if-any ""][# ### complex test here >> +] if [structs.fields.ident] is >> self.thrift_spec[[]structs.fields.field_id][[]4]: >> + [structs.fields.ident] = [structs.fields.value.cvalue] >> +[end] self.[structs.fields.ident] = [structs.fields.ident] >> +[end] >> +[end][# close: if-any structs.fields] >> + >> + def read(self, iprot): >> + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated [# >> +]and isinstance(iprot.trans, TTransport.CReadableTransport) [# >> +]and self.thrift_spec is not None and fastbinary is not None: >> + fastbinary.decode_binary(self, iprot.trans, [# >> +](self.__class__, self.thrift_spec)) >> + return >> + iprot.readStructBegin() >> + while True: >> + (fname, ftype, fid) = iprot.readFieldBegin() >> + if ftype == TType.STOP: >> + break[for structs.fields] >> + [if-index structs.fields first]if[else]elif[end] fid == [# >> +][structs.fields.field_id]: >> + if ftype == [structs.fields.type_enum]: >> + pass # deserialize >> + else: >> + iprot.skip(ftype)[end] >> + else: >> + iprot.skip(ftype) >> + iprot.readFieldEnd() >> + iprot.readStructEnd() >> + >> + def write(self, oprot): >> + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated [# >> +]and self.thrift_spec is not None and fastbinary is not None: >> + oprot.trans.write(fastbinary.encode_binary(self, [# >> +](self.__class__, self.thrift_spec))) >> + return >> + oprot.writeStructBegin('[structs.ident]')[for structs.fields] >> + if self.[structs.fields.ident] != None: >> + oprot.writeFieldBegin('[structs.fields.ident]', [# >> +][structs.fields.type_enum], [structs.fields.field_id]) >> + # serialize >> + oprot.writeFieldEnd()[end] >> + oprot.writeFieldStop() >> + oprot.writeStructEnd() >> + >> + def __repr__(self): >> + L = [[]'%s=%r' % (key, value) >> + for key, value in self.__dict__.iteritems()] >> + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) >> + >> + def __eq__(self, other): >> + return isinstance(other, self.__class__) and self.__dict__ == >> other.__dict__ >> + >> + def __ne__(self, other): >> + return not (self == other) >> + >> +[end][# for structs] >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt?rev=739520&view=auto >> ============================================================================== >> --- >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt >> (added) >> +++ >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_cvalue.ezt >> Sat Jan 31 07:40:26 2009 >> @@ -0,0 +1,10 @@ >> +[is ctype "map"]{ >> + [for value][value.key.cvalue]: [value.value.cvalue], >> +[end]}[# >> +#][else][is ctype "set"]set([[] >> + [for value][value.cvalue], >> +[end]])[# >> +#][else][is ctype "list"][[] >> + [for value][value.cvalue], >> +[end]][# >> +#][else][value.tvalue][end][end][end] >> \ No newline at end of file >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_deser.ezt?rev=739520&view=auto >> ============================================================================== >> (empty) >> >> Added: >> incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt >> URL: >> http://svn.apache.org/viewvc/incubator/thrift/branches/py-compiler/compiler/py/src/templates-py/py_ser.ezt?rev=739520&view=auto >> ============================================================================== >> (empty) >> >> >
