[
https://issues.apache.org/jira/browse/THRIFT-5228?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
qinluo updated THRIFT-5228:
---------------------------
Description:
thrift generate python code overwrite `ttypes.py`, does not include all things
defined in different IDL files with same namespace, and `a self-import` would
also occur. But generate for go or java would be as expected, i.e., in
different files.
one can use the following snippets to re-produce the wired things
```
$ thrift --version
Thrift version 0.11.0
```
IDL definition
`crop.thrift`
```
namespace py test_thrift.common.media
namespace go test_thrift.common.media
struct CropRequest {
1: optional list<string> ratio;
}
```
`media.thrift`
```
include "crop.thrift"
namespace py test_thrift.common.media
namespace go test_thrift.common.media
enum MediaType
{
UNKNOWN = 0;
IMAGE = 1;
VIDEO = 2;
}
struct Identifier {
1: optional MediaType media_type;
2: optional string token;
}
struct Request {
3: optional Identifier identifier;
20: optional crop.CropRequest crop_request;
}
```
generate code for python/go
```
$ thrift -r -gen py media.thrift
$ thrift -r -gen go media.thrift
```
```
├── gen-go
│ └── test_thrift
│ └── common
│ └── media
│ ├── GoUnusedProtection__.go
│ ├── crop-consts.go
│ ├── crop.go
│ ├── media-consts.go
│ └── media.go
├── gen-py
│ ├── __init__.py
│ └── test_thrift
│ ├── __init__.py
│ └── common
│ ├── __init__.py
│ └── media
│ ├── __init__.py
│ ├── constants.py
│ └── ttypes.py
```
things in `gen-py/test_thrift/common/media/ttypes.py` does not include the
definitions in `crop.thrift`, and there is a `self import`: (`import
test_thrift.common.media.ttypes` )
looks like
<details><pre><code>
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException,
TApplicationException
from thrift.protocol.TProtocol import TProtocolException
from thrift.TRecursive import fix_spec
import sys
import test_thrift.common.media.ttypes
from thrift.transport import TTransport
all_structs = []
class MediaType(object):
UNKNOWN = 0
IMAGE = 1
VIDEO = 2
_VALUES_TO_NAMES =
{ 0: "UNKNOWN", 1: "IMAGE", 2: "VIDEO", }
_NAMES_TO_VALUES =
{ "UNKNOWN": 0, "IMAGE": 1, "VIDEO": 2, }
class Identifier(object):
"""
Attributes:
- media_type
- token
"""
def __init__(self, media_type=None, token=None,):
self.media_type = media_type
self.token = token
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans,
TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.I32:
self.media_type = iprot.readI32()
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.STRING:
self.token = iprot.readString().decode('utf-8') if sys.version_info[0] == 2
else iprot.readString()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('Identifier')
if self.media_type is not None:
oprot.writeFieldBegin('media_type', TType.I32, 1)
oprot.writeI32(self.media_type)
oprot.writeFieldEnd()
if self.token is not None:
oprot.writeFieldBegin('token', TType.STRING, 2)
oprot.writeString(self.token.encode('utf-8') if sys.version_info[0] == 2 else
self.token)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
class Request(object):
"""
Attributes:
- identifier
- crop_request
"""
def __init__(self, identifier=None, crop_request=None,):
self.identifier = identifier
self.crop_request = crop_request
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans,
TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 3:
if ftype == TType.STRUCT:
self.identifier = Identifier()
self.identifier.read(iprot)
else:
iprot.skip(ftype)
elif fid == 20:
if ftype == TType.STRUCT:
self.crop_request = test_thrift.common.media.ttypes.CropRequest()
self.crop_request.read(iprot)
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('Request')
if self.identifier is not None:
oprot.writeFieldBegin('identifier', TType.STRUCT, 3)
self.identifier.write(oprot)
oprot.writeFieldEnd()
if self.crop_request is not None:
oprot.writeFieldBegin('crop_request', TType.STRUCT, 20)
self.crop_request.write(oprot)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(Identifier)
Identifier.thrift_spec = (
None, # 0
(1, TType.I32, 'media_type', None, None, ), # 1
(2, TType.STRING, 'token', 'UTF8', None, ), # 2
)
all_structs.append(Request)
Request.thrift_spec = (
None, # 0
None, # 1
None, # 2
(3, TType.STRUCT, 'identifier', [Identifier, None], None, ), # 3
None, # 4
None, # 5
None, # 6
None, # 7
None, # 8
None, # 9
None, # 10
None, # 11
None, # 12
None, # 13
None, # 14
None, # 15
None, # 16
None, # 17
None, # 18
None, # 19
(20, TType.STRUCT, 'crop_request',
[test_thrift.common.media.ttypes.CropRequest, None], None, ), # 20
)
fix_spec(all_structs)
del all_structs
</code></pre></details>
```
was:
thrift generate python code overwrite `ttypes.py`, does not include all things
defined in different IDL files with same namespace, and `a self-import` would
also occur. But generate for go or java would be as expected, i.e., in
different files.
one can use the following snippets to re-produce the wired things
<pre>
$ thrift --version
Thrift version 0.11.0
</pre>
IDL definition
`crop.thrift`
```
namespace py test_thrift.common.media
namespace go test_thrift.common.media
struct CropRequest {
1: optional list<string> ratio;
}
```
`media.thrift`
```
include "crop.thrift"
namespace py test_thrift.common.media
namespace go test_thrift.common.media
enum MediaType {
UNKNOWN = 0;
IMAGE = 1;
VIDEO = 2;
}
struct Identifier {
1: optional MediaType media_type;
2: optional string token;
}
struct Request {
3: optional Identifier identifier;
20: optional crop.CropRequest crop_request;
}
```
generate code for python/go
```
$ thrift -r -gen py media.thrift
$ thrift -r -gen go media.thrift
```
```
├── gen-go
│ └── test_thrift
│ └── common
│ └── media
│ ├── GoUnusedProtection__.go
│ ├── crop-consts.go
│ ├── crop.go
│ ├── media-consts.go
│ └── media.go
├── gen-py
│ ├── __init__.py
│ └── test_thrift
│ ├── __init__.py
│ └── common
│ ├── __init__.py
│ └── media
│ ├── __init__.py
│ ├── constants.py
│ └── ttypes.py
```
things in `gen-py/test_thrift/common/media/ttypes.py` does not include the
definitions in `crop.thrift`, and there is a `self import`: (`import
test_thrift.common.media.ttypes` )
looks like
<details><pre><code>
#
# Autogenerated by Thrift Compiler (0.11.0)
#
# DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
#
# options string: py
#
from thrift.Thrift import TType, TMessageType, TFrozenDict, TException,
TApplicationException
from thrift.protocol.TProtocol import TProtocolException
from thrift.TRecursive import fix_spec
import sys
import test_thrift.common.media.ttypes
from thrift.transport import TTransport
all_structs = []
class MediaType(object):
UNKNOWN = 0
IMAGE = 1
VIDEO = 2
_VALUES_TO_NAMES = {
0: "UNKNOWN",
1: "IMAGE",
2: "VIDEO",
}
_NAMES_TO_VALUES = {
"UNKNOWN": 0,
"IMAGE": 1,
"VIDEO": 2,
}
class Identifier(object):
"""
Attributes:
- media_type
- token
"""
def __init__(self, media_type=None, token=None,):
self.media_type = media_type
self.token = token
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans,
TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 1:
if ftype == TType.I32:
self.media_type = iprot.readI32()
else:
iprot.skip(ftype)
elif fid == 2:
if ftype == TType.STRING:
self.token = iprot.readString().decode('utf-8') if sys.version_info[0] == 2
else iprot.readString()
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('Identifier')
if self.media_type is not None:
oprot.writeFieldBegin('media_type', TType.I32, 1)
oprot.writeI32(self.media_type)
oprot.writeFieldEnd()
if self.token is not None:
oprot.writeFieldBegin('token', TType.STRING, 2)
oprot.writeString(self.token.encode('utf-8') if sys.version_info[0] == 2 else
self.token)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
class Request(object):
"""
Attributes:
- identifier
- crop_request
"""
def __init__(self, identifier=None, crop_request=None,):
self.identifier = identifier
self.crop_request = crop_request
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans,
TTransport.CReadableTransport) and self.thrift_spec is not None:
iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
return
iprot.readStructBegin()
while True:
(fname, ftype, fid) = iprot.readFieldBegin()
if ftype == TType.STOP:
break
if fid == 3:
if ftype == TType.STRUCT:
self.identifier = Identifier()
self.identifier.read(iprot)
else:
iprot.skip(ftype)
elif fid == 20:
if ftype == TType.STRUCT:
self.crop_request = test_thrift.common.media.ttypes.CropRequest()
self.crop_request.read(iprot)
else:
iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
iprot.readStructEnd()
def write(self, oprot):
if oprot._fast_encode is not None and self.thrift_spec is not None:
oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec]))
return
oprot.writeStructBegin('Request')
if self.identifier is not None:
oprot.writeFieldBegin('identifier', TType.STRUCT, 3)
self.identifier.write(oprot)
oprot.writeFieldEnd()
if self.crop_request is not None:
oprot.writeFieldBegin('crop_request', TType.STRUCT, 20)
self.crop_request.write(oprot)
oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
def validate(self):
return
def __repr__(self):
L = ['%s=%r' % (key, value)
for key, value in self.__dict__.items()]
return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
def __ne__(self, other):
return not (self == other)
all_structs.append(Identifier)
Identifier.thrift_spec = (
None, # 0
(1, TType.I32, 'media_type', None, None, ), # 1
(2, TType.STRING, 'token', 'UTF8', None, ), # 2
)
all_structs.append(Request)
Request.thrift_spec = (
None, # 0
None, # 1
None, # 2
(3, TType.STRUCT, 'identifier', [Identifier, None], None, ), # 3
None, # 4
None, # 5
None, # 6
None, # 7
None, # 8
None, # 9
None, # 10
None, # 11
None, # 12
None, # 13
None, # 14
None, # 15
None, # 16
None, # 17
None, # 18
None, # 19
(20, TType.STRUCT, 'crop_request',
[test_thrift.common.media.ttypes.CropRequest, None], None, ), # 20
)
fix_spec(all_structs)
del all_structs
</code></pre></details>
```
> gen python with recursive overwrite ttypes.py and self-import
> -------------------------------------------------------------
>
> Key: THRIFT-5228
> URL: https://issues.apache.org/jira/browse/THRIFT-5228
> Project: Thrift
> Issue Type: Bug
> Affects Versions: 0.11.0
> Reporter: qinluo
> Priority: Major
>
> thrift generate python code overwrite `ttypes.py`, does not include all
> things defined in different IDL files with same namespace, and `a
> self-import` would also occur. But generate for go or java would be as
> expected, i.e., in different files.
> one can use the following snippets to re-produce the wired things
> ```
> $ thrift --version
> Thrift version 0.11.0
> ```
> IDL definition
> `crop.thrift`
> ```
> namespace py test_thrift.common.media
> namespace go test_thrift.common.media
> struct CropRequest {
> 1: optional list<string> ratio;
> }
> ```
>
> `media.thrift`
> ```
> include "crop.thrift"
> namespace py test_thrift.common.media
> namespace go test_thrift.common.media
> enum MediaType
> {
> UNKNOWN = 0;
> IMAGE = 1;
> VIDEO = 2;
> }
> struct Identifier {
> 1: optional MediaType media_type;
> 2: optional string token;
> }
> struct Request {
> 3: optional Identifier identifier;
> 20: optional crop.CropRequest crop_request;
> }
> ```
> generate code for python/go
> ```
> $ thrift -r -gen py media.thrift
> $ thrift -r -gen go media.thrift
> ```
> ```
> ├── gen-go
> │ └── test_thrift
> │ └── common
> │ └── media
> │ ├── GoUnusedProtection__.go
> │ ├── crop-consts.go
> │ ├── crop.go
> │ ├── media-consts.go
> │ └── media.go
> ├── gen-py
> │ ├── __init__.py
> │ └── test_thrift
> │ ├── __init__.py
> │ └── common
> │ ├── __init__.py
> │ └── media
> │ ├── __init__.py
> │ ├── constants.py
> │ └── ttypes.py
> ```
> things in `gen-py/test_thrift/common/media/ttypes.py` does not include the
> definitions in `crop.thrift`, and there is a `self import`: (`import
> test_thrift.common.media.ttypes` )
> looks like
> <details><pre><code>
> #
> # Autogenerated by Thrift Compiler (0.11.0)
> #
> # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
> #
> # options string: py
> #
> from thrift.Thrift import TType, TMessageType, TFrozenDict, TException,
> TApplicationException
> from thrift.protocol.TProtocol import TProtocolException
> from thrift.TRecursive import fix_spec
> import sys
> import test_thrift.common.media.ttypes
> from thrift.transport import TTransport
> all_structs = []
> class MediaType(object):
> UNKNOWN = 0
> IMAGE = 1
> VIDEO = 2
> _VALUES_TO_NAMES =
> { 0: "UNKNOWN", 1: "IMAGE", 2: "VIDEO", }
> _NAMES_TO_VALUES =
> { "UNKNOWN": 0, "IMAGE": 1, "VIDEO": 2, }
> class Identifier(object):
> """
> Attributes:
> - media_type
> - token
> """
> def __init__(self, media_type=None, token=None,):
> self.media_type = media_type
> self.token = token
> def read(self, iprot):
> if iprot._fast_decode is not None and isinstance(iprot.trans,
> TTransport.CReadableTransport) and self.thrift_spec is not None:
> iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
> return
> iprot.readStructBegin()
> while True:
> (fname, ftype, fid) = iprot.readFieldBegin()
> if ftype == TType.STOP:
> break
> if fid == 1:
> if ftype == TType.I32:
> self.media_type = iprot.readI32()
> else:
> iprot.skip(ftype)
> elif fid == 2:
> if ftype == TType.STRING:
> self.token = iprot.readString().decode('utf-8') if sys.version_info[0] == 2
> else iprot.readString()
> else:
> iprot.skip(ftype)
> else:
> iprot.skip(ftype)
> iprot.readFieldEnd()
> iprot.readStructEnd()
> def write(self, oprot):
> if oprot._fast_encode is not None and self.thrift_spec is not None:
> oprot.trans.write(oprot._fast_encode(self, [self.__class__,
> self.thrift_spec]))
> return
> oprot.writeStructBegin('Identifier')
> if self.media_type is not None:
> oprot.writeFieldBegin('media_type', TType.I32, 1)
> oprot.writeI32(self.media_type)
> oprot.writeFieldEnd()
> if self.token is not None:
> oprot.writeFieldBegin('token', TType.STRING, 2)
> oprot.writeString(self.token.encode('utf-8') if sys.version_info[0] == 2
> else self.token)
> oprot.writeFieldEnd()
> oprot.writeFieldStop()
> oprot.writeStructEnd()
> def validate(self):
> return
> def __repr__(self):
> L = ['%s=%r' % (key, value)
> for key, value in self.__dict__.items()]
> return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
> def __eq__(self, other):
> return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
> def __ne__(self, other):
> return not (self == other)
> class Request(object):
> """
> Attributes:
> - identifier
> - crop_request
> """
> def __init__(self, identifier=None, crop_request=None,):
> self.identifier = identifier
> self.crop_request = crop_request
> def read(self, iprot):
> if iprot._fast_decode is not None and isinstance(iprot.trans,
> TTransport.CReadableTransport) and self.thrift_spec is not None:
> iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec])
> return
> iprot.readStructBegin()
> while True:
> (fname, ftype, fid) = iprot.readFieldBegin()
> if ftype == TType.STOP:
> break
> if fid == 3:
> if ftype == TType.STRUCT:
> self.identifier = Identifier()
> self.identifier.read(iprot)
> else:
> iprot.skip(ftype)
> elif fid == 20:
> if ftype == TType.STRUCT:
> self.crop_request = test_thrift.common.media.ttypes.CropRequest()
> self.crop_request.read(iprot)
> else:
> iprot.skip(ftype)
> else:
> iprot.skip(ftype)
> iprot.readFieldEnd()
> iprot.readStructEnd()
> def write(self, oprot):
> if oprot._fast_encode is not None and self.thrift_spec is not None:
> oprot.trans.write(oprot._fast_encode(self, [self.__class__,
> self.thrift_spec]))
> return
> oprot.writeStructBegin('Request')
> if self.identifier is not None:
> oprot.writeFieldBegin('identifier', TType.STRUCT, 3)
> self.identifier.write(oprot)
> oprot.writeFieldEnd()
> if self.crop_request is not None:
> oprot.writeFieldBegin('crop_request', TType.STRUCT, 20)
> self.crop_request.write(oprot)
> oprot.writeFieldEnd()
> oprot.writeFieldStop()
> oprot.writeStructEnd()
> def validate(self):
> return
> def __repr__(self):
> L = ['%s=%r' % (key, value)
> for key, value in self.__dict__.items()]
> return '%s(%s)' % (self.__class__.__name__, ', '.join(L))
> def __eq__(self, other):
> return isinstance(other, self.__class__) and self.__dict__ == other.__dict__
> def __ne__(self, other):
> return not (self == other)
> all_structs.append(Identifier)
> Identifier.thrift_spec = (
> None, # 0
> (1, TType.I32, 'media_type', None, None, ), # 1
> (2, TType.STRING, 'token', 'UTF8', None, ), # 2
> )
> all_structs.append(Request)
> Request.thrift_spec = (
> None, # 0
> None, # 1
> None, # 2
> (3, TType.STRUCT, 'identifier', [Identifier, None], None, ), # 3
> None, # 4
> None, # 5
> None, # 6
> None, # 7
> None, # 8
> None, # 9
> None, # 10
> None, # 11
> None, # 12
> None, # 13
> None, # 14
> None, # 15
> None, # 16
> None, # 17
> None, # 18
> None, # 19
> (20, TType.STRUCT, 'crop_request',
> [test_thrift.common.media.ttypes.CropRequest, None], None, ), # 20
> )
> fix_spec(all_structs)
> del all_structs
> </code></pre></details>
> ```
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
