Repository: hive Updated Branches: refs/heads/master 49f8f84d8 -> 926c1e8e3
http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index b85731e..f8b8628 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -14703,6 +14703,288 @@ class CompactionRequest: def __ne__(self, other): return not (self == other) +class OptionalCompactionInfoStruct: + """ + Attributes: + - ci + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'ci', (CompactionInfoStruct, CompactionInfoStruct.thrift_spec), None, ), # 1 + ) + + def __init__(self, ci=None,): + self.ci = ci + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.ci = CompactionInfoStruct() + self.ci.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('OptionalCompactionInfoStruct') + if self.ci is not None: + oprot.writeFieldBegin('ci', TType.STRUCT, 1) + self.ci.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.ci) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class CompactionInfoStruct: + """ + Attributes: + - id + - dbname + - tablename + - partitionname + - type + - runas + - properties + - toomanyaborts + - state + - workerId + - start + - highestWriteId + """ + + thrift_spec = ( + None, # 0 + (1, TType.I64, 'id', None, None, ), # 1 + (2, TType.STRING, 'dbname', None, None, ), # 2 + (3, TType.STRING, 'tablename', None, None, ), # 3 + (4, TType.STRING, 'partitionname', None, None, ), # 4 + (5, TType.I32, 'type', None, None, ), # 5 + (6, TType.STRING, 'runas', None, None, ), # 6 + (7, TType.STRING, 'properties', None, None, ), # 7 + (8, TType.BOOL, 'toomanyaborts', None, None, ), # 8 + (9, TType.STRING, 'state', None, None, ), # 9 + (10, TType.STRING, 'workerId', None, None, ), # 10 + (11, TType.I64, 'start', None, None, ), # 11 + (12, TType.I64, 'highestWriteId', None, None, ), # 12 + ) + + def __init__(self, id=None, dbname=None, tablename=None, partitionname=None, type=None, runas=None, properties=None, toomanyaborts=None, state=None, workerId=None, start=None, highestWriteId=None,): + self.id = id + self.dbname = dbname + self.tablename = tablename + self.partitionname = partitionname + self.type = type + self.runas = runas + self.properties = properties + self.toomanyaborts = toomanyaborts + self.state = state + self.workerId = workerId + self.start = start + self.highestWriteId = highestWriteId + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I64: + self.id = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.dbname = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.tablename = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.partitionname = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.I32: + self.type = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.runas = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.STRING: + self.properties = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.BOOL: + self.toomanyaborts = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 9: + if ftype == TType.STRING: + self.state = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 10: + if ftype == TType.STRING: + self.workerId = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 11: + if ftype == TType.I64: + self.start = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 12: + if ftype == TType.I64: + self.highestWriteId = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('CompactionInfoStruct') + if self.id is not None: + oprot.writeFieldBegin('id', TType.I64, 1) + oprot.writeI64(self.id) + oprot.writeFieldEnd() + if self.dbname is not None: + oprot.writeFieldBegin('dbname', TType.STRING, 2) + oprot.writeString(self.dbname) + oprot.writeFieldEnd() + if self.tablename is not None: + oprot.writeFieldBegin('tablename', TType.STRING, 3) + oprot.writeString(self.tablename) + oprot.writeFieldEnd() + if self.partitionname is not None: + oprot.writeFieldBegin('partitionname', TType.STRING, 4) + oprot.writeString(self.partitionname) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin('type', TType.I32, 5) + oprot.writeI32(self.type) + oprot.writeFieldEnd() + if self.runas is not None: + oprot.writeFieldBegin('runas', TType.STRING, 6) + oprot.writeString(self.runas) + oprot.writeFieldEnd() + if self.properties is not None: + oprot.writeFieldBegin('properties', TType.STRING, 7) + oprot.writeString(self.properties) + oprot.writeFieldEnd() + if self.toomanyaborts is not None: + oprot.writeFieldBegin('toomanyaborts', TType.BOOL, 8) + oprot.writeBool(self.toomanyaborts) + oprot.writeFieldEnd() + if self.state is not None: + oprot.writeFieldBegin('state', TType.STRING, 9) + oprot.writeString(self.state) + oprot.writeFieldEnd() + if self.workerId is not None: + oprot.writeFieldBegin('workerId', TType.STRING, 10) + oprot.writeString(self.workerId) + oprot.writeFieldEnd() + if self.start is not None: + oprot.writeFieldBegin('start', TType.I64, 11) + oprot.writeI64(self.start) + oprot.writeFieldEnd() + if self.highestWriteId is not None: + oprot.writeFieldBegin('highestWriteId', TType.I64, 12) + oprot.writeI64(self.highestWriteId) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + if self.id is None: + raise TProtocol.TProtocolException(message='Required field id is unset!') + if self.dbname is None: + raise TProtocol.TProtocolException(message='Required field dbname is unset!') + if self.tablename is None: + raise TProtocol.TProtocolException(message='Required field tablename is unset!') + if self.type is None: + raise TProtocol.TProtocolException(message='Required field type is unset!') + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.id) + value = (value * 31) ^ hash(self.dbname) + value = (value * 31) ^ hash(self.tablename) + value = (value * 31) ^ hash(self.partitionname) + value = (value * 31) ^ hash(self.type) + value = (value * 31) ^ hash(self.runas) + value = (value * 31) ^ hash(self.properties) + value = (value * 31) ^ hash(self.toomanyaborts) + value = (value * 31) ^ hash(self.state) + value = (value * 31) ^ hash(self.workerId) + value = (value * 31) ^ hash(self.start) + value = (value * 31) ^ hash(self.highestWriteId) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + class CompactionResponse: """ Attributes: @@ -15619,10 +15901,10 @@ class NotificationEventRequest: elif fid == 3: if ftype == TType.LIST: self.eventTypeSkipList = [] - (_etype647, _size644) = iprot.readListBegin() - for _i648 in xrange(_size644): - _elem649 = iprot.readString() - self.eventTypeSkipList.append(_elem649) + (_etype654, _size651) = iprot.readListBegin() + for _i655 in xrange(_size651): + _elem656 = iprot.readString() + self.eventTypeSkipList.append(_elem656) iprot.readListEnd() else: iprot.skip(ftype) @@ -15647,8 +15929,8 @@ class NotificationEventRequest: if self.eventTypeSkipList is not None: oprot.writeFieldBegin('eventTypeSkipList', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.eventTypeSkipList)) - for iter650 in self.eventTypeSkipList: - oprot.writeString(iter650) + for iter657 in self.eventTypeSkipList: + oprot.writeString(iter657) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -15868,11 +16150,11 @@ class NotificationEventResponse: if fid == 1: if ftype == TType.LIST: self.events = [] - (_etype654, _size651) = iprot.readListBegin() - for _i655 in xrange(_size651): - _elem656 = NotificationEvent() - _elem656.read(iprot) - self.events.append(_elem656) + (_etype661, _size658) = iprot.readListBegin() + for _i662 in xrange(_size658): + _elem663 = NotificationEvent() + _elem663.read(iprot) + self.events.append(_elem663) iprot.readListEnd() else: iprot.skip(ftype) @@ -15889,8 +16171,8 @@ class NotificationEventResponse: if self.events is not None: oprot.writeFieldBegin('events', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.events)) - for iter657 in self.events: - iter657.write(oprot) + for iter664 in self.events: + iter664.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -16213,30 +16495,30 @@ class InsertEventRequestData: elif fid == 2: if ftype == TType.LIST: self.filesAdded = [] - (_etype661, _size658) = iprot.readListBegin() - for _i662 in xrange(_size658): - _elem663 = iprot.readString() - self.filesAdded.append(_elem663) + (_etype668, _size665) = iprot.readListBegin() + for _i669 in xrange(_size665): + _elem670 = iprot.readString() + self.filesAdded.append(_elem670) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.filesAddedChecksum = [] - (_etype667, _size664) = iprot.readListBegin() - for _i668 in xrange(_size664): - _elem669 = iprot.readString() - self.filesAddedChecksum.append(_elem669) + (_etype674, _size671) = iprot.readListBegin() + for _i675 in xrange(_size671): + _elem676 = iprot.readString() + self.filesAddedChecksum.append(_elem676) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.subDirectoryList = [] - (_etype673, _size670) = iprot.readListBegin() - for _i674 in xrange(_size670): - _elem675 = iprot.readString() - self.subDirectoryList.append(_elem675) + (_etype680, _size677) = iprot.readListBegin() + for _i681 in xrange(_size677): + _elem682 = iprot.readString() + self.subDirectoryList.append(_elem682) iprot.readListEnd() else: iprot.skip(ftype) @@ -16257,22 +16539,22 @@ class InsertEventRequestData: if self.filesAdded is not None: oprot.writeFieldBegin('filesAdded', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.filesAdded)) - for iter676 in self.filesAdded: - oprot.writeString(iter676) + for iter683 in self.filesAdded: + oprot.writeString(iter683) oprot.writeListEnd() oprot.writeFieldEnd() if self.filesAddedChecksum is not None: oprot.writeFieldBegin('filesAddedChecksum', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.filesAddedChecksum)) - for iter677 in self.filesAddedChecksum: - oprot.writeString(iter677) + for iter684 in self.filesAddedChecksum: + oprot.writeString(iter684) oprot.writeListEnd() oprot.writeFieldEnd() if self.subDirectoryList is not None: oprot.writeFieldBegin('subDirectoryList', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.subDirectoryList)) - for iter678 in self.subDirectoryList: - oprot.writeString(iter678) + for iter685 in self.subDirectoryList: + oprot.writeString(iter685) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -16431,10 +16713,10 @@ class FireEventRequest: elif fid == 5: if ftype == TType.LIST: self.partitionVals = [] - (_etype682, _size679) = iprot.readListBegin() - for _i683 in xrange(_size679): - _elem684 = iprot.readString() - self.partitionVals.append(_elem684) + (_etype689, _size686) = iprot.readListBegin() + for _i690 in xrange(_size686): + _elem691 = iprot.readString() + self.partitionVals.append(_elem691) iprot.readListEnd() else: iprot.skip(ftype) @@ -16472,8 +16754,8 @@ class FireEventRequest: if self.partitionVals is not None: oprot.writeFieldBegin('partitionVals', TType.LIST, 5) oprot.writeListBegin(TType.STRING, len(self.partitionVals)) - for iter685 in self.partitionVals: - oprot.writeString(iter685) + for iter692 in self.partitionVals: + oprot.writeString(iter692) oprot.writeListEnd() oprot.writeFieldEnd() if self.catName is not None: @@ -16625,10 +16907,10 @@ class WriteNotificationLogRequest: elif fid == 6: if ftype == TType.LIST: self.partitionVals = [] - (_etype689, _size686) = iprot.readListBegin() - for _i690 in xrange(_size686): - _elem691 = iprot.readString() - self.partitionVals.append(_elem691) + (_etype696, _size693) = iprot.readListBegin() + for _i697 in xrange(_size693): + _elem698 = iprot.readString() + self.partitionVals.append(_elem698) iprot.readListEnd() else: iprot.skip(ftype) @@ -16665,8 +16947,8 @@ class WriteNotificationLogRequest: if self.partitionVals is not None: oprot.writeFieldBegin('partitionVals', TType.LIST, 6) oprot.writeListBegin(TType.STRING, len(self.partitionVals)) - for iter692 in self.partitionVals: - oprot.writeString(iter692) + for iter699 in self.partitionVals: + oprot.writeString(iter699) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -16860,12 +17142,12 @@ class GetFileMetadataByExprResult: if fid == 1: if ftype == TType.MAP: self.metadata = {} - (_ktype694, _vtype695, _size693 ) = iprot.readMapBegin() - for _i697 in xrange(_size693): - _key698 = iprot.readI64() - _val699 = MetadataPpdResult() - _val699.read(iprot) - self.metadata[_key698] = _val699 + (_ktype701, _vtype702, _size700 ) = iprot.readMapBegin() + for _i704 in xrange(_size700): + _key705 = iprot.readI64() + _val706 = MetadataPpdResult() + _val706.read(iprot) + self.metadata[_key705] = _val706 iprot.readMapEnd() else: iprot.skip(ftype) @@ -16887,9 +17169,9 @@ class GetFileMetadataByExprResult: if self.metadata is not None: oprot.writeFieldBegin('metadata', TType.MAP, 1) oprot.writeMapBegin(TType.I64, TType.STRUCT, len(self.metadata)) - for kiter700,viter701 in self.metadata.items(): - oprot.writeI64(kiter700) - viter701.write(oprot) + for kiter707,viter708 in self.metadata.items(): + oprot.writeI64(kiter707) + viter708.write(oprot) oprot.writeMapEnd() oprot.writeFieldEnd() if self.isSupported is not None: @@ -16959,10 +17241,10 @@ class GetFileMetadataByExprRequest: if fid == 1: if ftype == TType.LIST: self.fileIds = [] - (_etype705, _size702) = iprot.readListBegin() - for _i706 in xrange(_size702): - _elem707 = iprot.readI64() - self.fileIds.append(_elem707) + (_etype712, _size709) = iprot.readListBegin() + for _i713 in xrange(_size709): + _elem714 = iprot.readI64() + self.fileIds.append(_elem714) iprot.readListEnd() else: iprot.skip(ftype) @@ -16994,8 +17276,8 @@ class GetFileMetadataByExprRequest: if self.fileIds is not None: oprot.writeFieldBegin('fileIds', TType.LIST, 1) oprot.writeListBegin(TType.I64, len(self.fileIds)) - for iter708 in self.fileIds: - oprot.writeI64(iter708) + for iter715 in self.fileIds: + oprot.writeI64(iter715) oprot.writeListEnd() oprot.writeFieldEnd() if self.expr is not None: @@ -17069,11 +17351,11 @@ class GetFileMetadataResult: if fid == 1: if ftype == TType.MAP: self.metadata = {} - (_ktype710, _vtype711, _size709 ) = iprot.readMapBegin() - for _i713 in xrange(_size709): - _key714 = iprot.readI64() - _val715 = iprot.readString() - self.metadata[_key714] = _val715 + (_ktype717, _vtype718, _size716 ) = iprot.readMapBegin() + for _i720 in xrange(_size716): + _key721 = iprot.readI64() + _val722 = iprot.readString() + self.metadata[_key721] = _val722 iprot.readMapEnd() else: iprot.skip(ftype) @@ -17095,9 +17377,9 @@ class GetFileMetadataResult: if self.metadata is not None: oprot.writeFieldBegin('metadata', TType.MAP, 1) oprot.writeMapBegin(TType.I64, TType.STRING, len(self.metadata)) - for kiter716,viter717 in self.metadata.items(): - oprot.writeI64(kiter716) - oprot.writeString(viter717) + for kiter723,viter724 in self.metadata.items(): + oprot.writeI64(kiter723) + oprot.writeString(viter724) oprot.writeMapEnd() oprot.writeFieldEnd() if self.isSupported is not None: @@ -17158,10 +17440,10 @@ class GetFileMetadataRequest: if fid == 1: if ftype == TType.LIST: self.fileIds = [] - (_etype721, _size718) = iprot.readListBegin() - for _i722 in xrange(_size718): - _elem723 = iprot.readI64() - self.fileIds.append(_elem723) + (_etype728, _size725) = iprot.readListBegin() + for _i729 in xrange(_size725): + _elem730 = iprot.readI64() + self.fileIds.append(_elem730) iprot.readListEnd() else: iprot.skip(ftype) @@ -17178,8 +17460,8 @@ class GetFileMetadataRequest: if self.fileIds is not None: oprot.writeFieldBegin('fileIds', TType.LIST, 1) oprot.writeListBegin(TType.I64, len(self.fileIds)) - for iter724 in self.fileIds: - oprot.writeI64(iter724) + for iter731 in self.fileIds: + oprot.writeI64(iter731) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -17285,20 +17567,20 @@ class PutFileMetadataRequest: if fid == 1: if ftype == TType.LIST: self.fileIds = [] - (_etype728, _size725) = iprot.readListBegin() - for _i729 in xrange(_size725): - _elem730 = iprot.readI64() - self.fileIds.append(_elem730) + (_etype735, _size732) = iprot.readListBegin() + for _i736 in xrange(_size732): + _elem737 = iprot.readI64() + self.fileIds.append(_elem737) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.metadata = [] - (_etype734, _size731) = iprot.readListBegin() - for _i735 in xrange(_size731): - _elem736 = iprot.readString() - self.metadata.append(_elem736) + (_etype741, _size738) = iprot.readListBegin() + for _i742 in xrange(_size738): + _elem743 = iprot.readString() + self.metadata.append(_elem743) iprot.readListEnd() else: iprot.skip(ftype) @@ -17320,15 +17602,15 @@ class PutFileMetadataRequest: if self.fileIds is not None: oprot.writeFieldBegin('fileIds', TType.LIST, 1) oprot.writeListBegin(TType.I64, len(self.fileIds)) - for iter737 in self.fileIds: - oprot.writeI64(iter737) + for iter744 in self.fileIds: + oprot.writeI64(iter744) oprot.writeListEnd() oprot.writeFieldEnd() if self.metadata is not None: oprot.writeFieldBegin('metadata', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.metadata)) - for iter738 in self.metadata: - oprot.writeString(iter738) + for iter745 in self.metadata: + oprot.writeString(iter745) oprot.writeListEnd() oprot.writeFieldEnd() if self.type is not None: @@ -17436,10 +17718,10 @@ class ClearFileMetadataRequest: if fid == 1: if ftype == TType.LIST: self.fileIds = [] - (_etype742, _size739) = iprot.readListBegin() - for _i743 in xrange(_size739): - _elem744 = iprot.readI64() - self.fileIds.append(_elem744) + (_etype749, _size746) = iprot.readListBegin() + for _i750 in xrange(_size746): + _elem751 = iprot.readI64() + self.fileIds.append(_elem751) iprot.readListEnd() else: iprot.skip(ftype) @@ -17456,8 +17738,8 @@ class ClearFileMetadataRequest: if self.fileIds is not None: oprot.writeFieldBegin('fileIds', TType.LIST, 1) oprot.writeListBegin(TType.I64, len(self.fileIds)) - for iter745 in self.fileIds: - oprot.writeI64(iter745) + for iter752 in self.fileIds: + oprot.writeI64(iter752) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -17686,11 +17968,11 @@ class GetAllFunctionsResponse: if fid == 1: if ftype == TType.LIST: self.functions = [] - (_etype749, _size746) = iprot.readListBegin() - for _i750 in xrange(_size746): - _elem751 = Function() - _elem751.read(iprot) - self.functions.append(_elem751) + (_etype756, _size753) = iprot.readListBegin() + for _i757 in xrange(_size753): + _elem758 = Function() + _elem758.read(iprot) + self.functions.append(_elem758) iprot.readListEnd() else: iprot.skip(ftype) @@ -17707,8 +17989,8 @@ class GetAllFunctionsResponse: if self.functions is not None: oprot.writeFieldBegin('functions', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.functions)) - for iter752 in self.functions: - iter752.write(oprot) + for iter759 in self.functions: + iter759.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -17760,10 +18042,10 @@ class ClientCapabilities: if fid == 1: if ftype == TType.LIST: self.values = [] - (_etype756, _size753) = iprot.readListBegin() - for _i757 in xrange(_size753): - _elem758 = iprot.readI32() - self.values.append(_elem758) + (_etype763, _size760) = iprot.readListBegin() + for _i764 in xrange(_size760): + _elem765 = iprot.readI32() + self.values.append(_elem765) iprot.readListEnd() else: iprot.skip(ftype) @@ -17780,8 +18062,8 @@ class ClientCapabilities: if self.values is not None: oprot.writeFieldBegin('values', TType.LIST, 1) oprot.writeListBegin(TType.I32, len(self.values)) - for iter759 in self.values: - oprot.writeI32(iter759) + for iter766 in self.values: + oprot.writeI32(iter766) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -18053,10 +18335,10 @@ class GetTablesRequest: elif fid == 2: if ftype == TType.LIST: self.tblNames = [] - (_etype763, _size760) = iprot.readListBegin() - for _i764 in xrange(_size760): - _elem765 = iprot.readString() - self.tblNames.append(_elem765) + (_etype770, _size767) = iprot.readListBegin() + for _i771 in xrange(_size767): + _elem772 = iprot.readString() + self.tblNames.append(_elem772) iprot.readListEnd() else: iprot.skip(ftype) @@ -18088,8 +18370,8 @@ class GetTablesRequest: if self.tblNames is not None: oprot.writeFieldBegin('tblNames', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.tblNames)) - for iter766 in self.tblNames: - oprot.writeString(iter766) + for iter773 in self.tblNames: + oprot.writeString(iter773) oprot.writeListEnd() oprot.writeFieldEnd() if self.capabilities is not None: @@ -18154,11 +18436,11 @@ class GetTablesResult: if fid == 1: if ftype == TType.LIST: self.tables = [] - (_etype770, _size767) = iprot.readListBegin() - for _i771 in xrange(_size767): - _elem772 = Table() - _elem772.read(iprot) - self.tables.append(_elem772) + (_etype777, _size774) = iprot.readListBegin() + for _i778 in xrange(_size774): + _elem779 = Table() + _elem779.read(iprot) + self.tables.append(_elem779) iprot.readListEnd() else: iprot.skip(ftype) @@ -18175,8 +18457,8 @@ class GetTablesResult: if self.tables is not None: oprot.writeFieldBegin('tables', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.tables)) - for iter773 in self.tables: - iter773.write(oprot) + for iter780 in self.tables: + iter780.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -19475,44 +19757,44 @@ class WMFullResourcePlan: elif fid == 2: if ftype == TType.LIST: self.pools = [] - (_etype777, _size774) = iprot.readListBegin() - for _i778 in xrange(_size774): - _elem779 = WMPool() - _elem779.read(iprot) - self.pools.append(_elem779) + (_etype784, _size781) = iprot.readListBegin() + for _i785 in xrange(_size781): + _elem786 = WMPool() + _elem786.read(iprot) + self.pools.append(_elem786) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.mappings = [] - (_etype783, _size780) = iprot.readListBegin() - for _i784 in xrange(_size780): - _elem785 = WMMapping() - _elem785.read(iprot) - self.mappings.append(_elem785) + (_etype790, _size787) = iprot.readListBegin() + for _i791 in xrange(_size787): + _elem792 = WMMapping() + _elem792.read(iprot) + self.mappings.append(_elem792) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.triggers = [] - (_etype789, _size786) = iprot.readListBegin() - for _i790 in xrange(_size786): - _elem791 = WMTrigger() - _elem791.read(iprot) - self.triggers.append(_elem791) + (_etype796, _size793) = iprot.readListBegin() + for _i797 in xrange(_size793): + _elem798 = WMTrigger() + _elem798.read(iprot) + self.triggers.append(_elem798) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 5: if ftype == TType.LIST: self.poolTriggers = [] - (_etype795, _size792) = iprot.readListBegin() - for _i796 in xrange(_size792): - _elem797 = WMPoolTrigger() - _elem797.read(iprot) - self.poolTriggers.append(_elem797) + (_etype802, _size799) = iprot.readListBegin() + for _i803 in xrange(_size799): + _elem804 = WMPoolTrigger() + _elem804.read(iprot) + self.poolTriggers.append(_elem804) iprot.readListEnd() else: iprot.skip(ftype) @@ -19533,29 +19815,29 @@ class WMFullResourcePlan: if self.pools is not None: oprot.writeFieldBegin('pools', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.pools)) - for iter798 in self.pools: - iter798.write(oprot) + for iter805 in self.pools: + iter805.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.mappings is not None: oprot.writeFieldBegin('mappings', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.mappings)) - for iter799 in self.mappings: - iter799.write(oprot) + for iter806 in self.mappings: + iter806.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.triggers is not None: oprot.writeFieldBegin('triggers', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.triggers)) - for iter800 in self.triggers: - iter800.write(oprot) + for iter807 in self.triggers: + iter807.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.poolTriggers is not None: oprot.writeFieldBegin('poolTriggers', TType.LIST, 5) oprot.writeListBegin(TType.STRUCT, len(self.poolTriggers)) - for iter801 in self.poolTriggers: - iter801.write(oprot) + for iter808 in self.poolTriggers: + iter808.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -20080,11 +20362,11 @@ class WMGetAllResourcePlanResponse: if fid == 1: if ftype == TType.LIST: self.resourcePlans = [] - (_etype805, _size802) = iprot.readListBegin() - for _i806 in xrange(_size802): - _elem807 = WMResourcePlan() - _elem807.read(iprot) - self.resourcePlans.append(_elem807) + (_etype812, _size809) = iprot.readListBegin() + for _i813 in xrange(_size809): + _elem814 = WMResourcePlan() + _elem814.read(iprot) + self.resourcePlans.append(_elem814) iprot.readListEnd() else: iprot.skip(ftype) @@ -20101,8 +20383,8 @@ class WMGetAllResourcePlanResponse: if self.resourcePlans is not None: oprot.writeFieldBegin('resourcePlans', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.resourcePlans)) - for iter808 in self.resourcePlans: - iter808.write(oprot) + for iter815 in self.resourcePlans: + iter815.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -20432,20 +20714,20 @@ class WMValidateResourcePlanResponse: if fid == 1: if ftype == TType.LIST: self.errors = [] - (_etype812, _size809) = iprot.readListBegin() - for _i813 in xrange(_size809): - _elem814 = iprot.readString() - self.errors.append(_elem814) + (_etype819, _size816) = iprot.readListBegin() + for _i820 in xrange(_size816): + _elem821 = iprot.readString() + self.errors.append(_elem821) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.warnings = [] - (_etype818, _size815) = iprot.readListBegin() - for _i819 in xrange(_size815): - _elem820 = iprot.readString() - self.warnings.append(_elem820) + (_etype825, _size822) = iprot.readListBegin() + for _i826 in xrange(_size822): + _elem827 = iprot.readString() + self.warnings.append(_elem827) iprot.readListEnd() else: iprot.skip(ftype) @@ -20462,15 +20744,15 @@ class WMValidateResourcePlanResponse: if self.errors is not None: oprot.writeFieldBegin('errors', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.errors)) - for iter821 in self.errors: - oprot.writeString(iter821) + for iter828 in self.errors: + oprot.writeString(iter828) oprot.writeListEnd() oprot.writeFieldEnd() if self.warnings is not None: oprot.writeFieldBegin('warnings', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.warnings)) - for iter822 in self.warnings: - oprot.writeString(iter822) + for iter829 in self.warnings: + oprot.writeString(iter829) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -21086,11 +21368,11 @@ class WMGetTriggersForResourePlanResponse: if fid == 1: if ftype == TType.LIST: self.triggers = [] - (_etype826, _size823) = iprot.readListBegin() - for _i827 in xrange(_size823): - _elem828 = WMTrigger() - _elem828.read(iprot) - self.triggers.append(_elem828) + (_etype833, _size830) = iprot.readListBegin() + for _i834 in xrange(_size830): + _elem835 = WMTrigger() + _elem835.read(iprot) + self.triggers.append(_elem835) iprot.readListEnd() else: iprot.skip(ftype) @@ -21107,8 +21389,8 @@ class WMGetTriggersForResourePlanResponse: if self.triggers is not None: oprot.writeFieldBegin('triggers', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.triggers)) - for iter829 in self.triggers: - iter829.write(oprot) + for iter836 in self.triggers: + iter836.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -22318,11 +22600,11 @@ class SchemaVersion: elif fid == 4: if ftype == TType.LIST: self.cols = [] - (_etype833, _size830) = iprot.readListBegin() - for _i834 in xrange(_size830): - _elem835 = FieldSchema() - _elem835.read(iprot) - self.cols.append(_elem835) + (_etype840, _size837) = iprot.readListBegin() + for _i841 in xrange(_size837): + _elem842 = FieldSchema() + _elem842.read(iprot) + self.cols.append(_elem842) iprot.readListEnd() else: iprot.skip(ftype) @@ -22382,8 +22664,8 @@ class SchemaVersion: if self.cols is not None: oprot.writeFieldBegin('cols', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.cols)) - for iter836 in self.cols: - iter836.write(oprot) + for iter843 in self.cols: + iter843.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.state is not None: @@ -22638,11 +22920,11 @@ class FindSchemasByColsResp: if fid == 1: if ftype == TType.LIST: self.schemaVersions = [] - (_etype840, _size837) = iprot.readListBegin() - for _i841 in xrange(_size837): - _elem842 = SchemaVersionDescriptor() - _elem842.read(iprot) - self.schemaVersions.append(_elem842) + (_etype847, _size844) = iprot.readListBegin() + for _i848 in xrange(_size844): + _elem849 = SchemaVersionDescriptor() + _elem849.read(iprot) + self.schemaVersions.append(_elem849) iprot.readListEnd() else: iprot.skip(ftype) @@ -22659,8 +22941,8 @@ class FindSchemasByColsResp: if self.schemaVersions is not None: oprot.writeFieldBegin('schemaVersions', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.schemaVersions)) - for iter843 in self.schemaVersions: - iter843.write(oprot) + for iter850 in self.schemaVersions: + iter850.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -23145,11 +23427,11 @@ class AlterPartitionsRequest: elif fid == 4: if ftype == TType.LIST: self.partitions = [] - (_etype847, _size844) = iprot.readListBegin() - for _i848 in xrange(_size844): - _elem849 = Partition() - _elem849.read(iprot) - self.partitions.append(_elem849) + (_etype854, _size851) = iprot.readListBegin() + for _i855 in xrange(_size851): + _elem856 = Partition() + _elem856.read(iprot) + self.partitions.append(_elem856) iprot.readListEnd() else: iprot.skip(ftype) @@ -23194,8 +23476,8 @@ class AlterPartitionsRequest: if self.partitions is not None: oprot.writeFieldBegin('partitions', TType.LIST, 4) oprot.writeListBegin(TType.STRUCT, len(self.partitions)) - for iter850 in self.partitions: - iter850.write(oprot) + for iter857 in self.partitions: + iter857.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.environmentContext is not None: @@ -23347,10 +23629,10 @@ class RenamePartitionRequest: elif fid == 4: if ftype == TType.LIST: self.partVals = [] - (_etype854, _size851) = iprot.readListBegin() - for _i855 in xrange(_size851): - _elem856 = iprot.readString() - self.partVals.append(_elem856) + (_etype861, _size858) = iprot.readListBegin() + for _i862 in xrange(_size858): + _elem863 = iprot.readString() + self.partVals.append(_elem863) iprot.readListEnd() else: iprot.skip(ftype) @@ -23390,8 +23672,8 @@ class RenamePartitionRequest: if self.partVals is not None: oprot.writeFieldBegin('partVals', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.partVals)) - for iter857 in self.partVals: - oprot.writeString(iter857) + for iter864 in self.partVals: + oprot.writeString(iter864) oprot.writeListEnd() oprot.writeFieldEnd() if self.newPart is not None: @@ -23713,10 +23995,10 @@ class GetPartitionsProjectionSpec: if fid == 1: if ftype == TType.LIST: self.fieldList = [] - (_etype861, _size858) = iprot.readListBegin() - for _i862 in xrange(_size858): - _elem863 = iprot.readString() - self.fieldList.append(_elem863) + (_etype868, _size865) = iprot.readListBegin() + for _i869 in xrange(_size865): + _elem870 = iprot.readString() + self.fieldList.append(_elem870) iprot.readListEnd() else: iprot.skip(ftype) @@ -23743,8 +24025,8 @@ class GetPartitionsProjectionSpec: if self.fieldList is not None: oprot.writeFieldBegin('fieldList', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.fieldList)) - for iter864 in self.fieldList: - oprot.writeString(iter864) + for iter871 in self.fieldList: + oprot.writeString(iter871) oprot.writeListEnd() oprot.writeFieldEnd() if self.includeParamKeyPattern is not None: @@ -23820,10 +24102,10 @@ class GetPartitionsFilterSpec: elif fid == 8: if ftype == TType.LIST: self.filters = [] - (_etype868, _size865) = iprot.readListBegin() - for _i869 in xrange(_size865): - _elem870 = iprot.readString() - self.filters.append(_elem870) + (_etype875, _size872) = iprot.readListBegin() + for _i876 in xrange(_size872): + _elem877 = iprot.readString() + self.filters.append(_elem877) iprot.readListEnd() else: iprot.skip(ftype) @@ -23844,8 +24126,8 @@ class GetPartitionsFilterSpec: if self.filters is not None: oprot.writeFieldBegin('filters', TType.LIST, 8) oprot.writeListBegin(TType.STRING, len(self.filters)) - for iter871 in self.filters: - oprot.writeString(iter871) + for iter878 in self.filters: + oprot.writeString(iter878) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -23898,11 +24180,11 @@ class GetPartitionsResponse: if fid == 1: if ftype == TType.LIST: self.partitionSpec = [] - (_etype875, _size872) = iprot.readListBegin() - for _i876 in xrange(_size872): - _elem877 = PartitionSpec() - _elem877.read(iprot) - self.partitionSpec.append(_elem877) + (_etype882, _size879) = iprot.readListBegin() + for _i883 in xrange(_size879): + _elem884 = PartitionSpec() + _elem884.read(iprot) + self.partitionSpec.append(_elem884) iprot.readListEnd() else: iprot.skip(ftype) @@ -23919,8 +24201,8 @@ class GetPartitionsResponse: if self.partitionSpec is not None: oprot.writeFieldBegin('partitionSpec', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.partitionSpec)) - for iter878 in self.partitionSpec: - iter878.write(oprot) + for iter885 in self.partitionSpec: + iter885.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -24018,10 +24300,10 @@ class GetPartitionsRequest: elif fid == 6: if ftype == TType.LIST: self.groupNames = [] - (_etype882, _size879) = iprot.readListBegin() - for _i883 in xrange(_size879): - _elem884 = iprot.readString() - self.groupNames.append(_elem884) + (_etype889, _size886) = iprot.readListBegin() + for _i890 in xrange(_size886): + _elem891 = iprot.readString() + self.groupNames.append(_elem891) iprot.readListEnd() else: iprot.skip(ftype) @@ -24070,8 +24352,8 @@ class GetPartitionsRequest: if self.groupNames is not None: oprot.writeFieldBegin('groupNames', TType.LIST, 6) oprot.writeListBegin(TType.STRING, len(self.groupNames)) - for iter885 in self.groupNames: - oprot.writeString(iter885) + for iter892 in self.groupNames: + oprot.writeString(iter892) oprot.writeListEnd() oprot.writeFieldEnd() if self.projectionSpec is not None: http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index 6215479..dfc5d7b 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -3263,6 +3263,67 @@ class CompactionRequest ::Thrift::Struct.generate_accessors self end +class OptionalCompactionInfoStruct + include ::Thrift::Struct, ::Thrift::Struct_Union + CI = 1 + + FIELDS = { + CI => {:type => ::Thrift::Types::STRUCT, :name => 'ci', :class => ::CompactionInfoStruct, :optional => true} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class CompactionInfoStruct + include ::Thrift::Struct, ::Thrift::Struct_Union + ID = 1 + DBNAME = 2 + TABLENAME = 3 + PARTITIONNAME = 4 + TYPE = 5 + RUNAS = 6 + PROPERTIES = 7 + TOOMANYABORTS = 8 + STATE = 9 + WORKERID = 10 + START = 11 + HIGHESTWRITEID = 12 + + FIELDS = { + ID => {:type => ::Thrift::Types::I64, :name => 'id'}, + DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'}, + TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'}, + PARTITIONNAME => {:type => ::Thrift::Types::STRING, :name => 'partitionname', :optional => true}, + TYPE => {:type => ::Thrift::Types::I32, :name => 'type', :enum_class => ::CompactionType}, + RUNAS => {:type => ::Thrift::Types::STRING, :name => 'runas', :optional => true}, + PROPERTIES => {:type => ::Thrift::Types::STRING, :name => 'properties', :optional => true}, + TOOMANYABORTS => {:type => ::Thrift::Types::BOOL, :name => 'toomanyaborts', :optional => true}, + STATE => {:type => ::Thrift::Types::STRING, :name => 'state', :optional => true}, + WORKERID => {:type => ::Thrift::Types::STRING, :name => 'workerId', :optional => true}, + START => {:type => ::Thrift::Types::I64, :name => 'start', :optional => true}, + HIGHESTWRITEID => {:type => ::Thrift::Types::I64, :name => 'highestWriteId', :optional => true} + } + + def struct_fields; FIELDS; end + + def validate + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field id is unset!') unless @id + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field dbname is unset!') unless @dbname + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field tablename is unset!') unless @tablename + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field type is unset!') unless @type + unless @type.nil? || ::CompactionType::VALID_VALUES.include?(@type) + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field type!') + end + end + + ::Thrift::Struct.generate_accessors self +end + class CompactionResponse include ::Thrift::Struct, ::Thrift::Struct_Union ID = 1 http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb index a5f976b..35e1693 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/thrift_hive_metastore.rb @@ -2782,6 +2782,110 @@ module ThriftHiveMetastore return end + def find_next_compact(workerId) + send_find_next_compact(workerId) + return recv_find_next_compact() + end + + def send_find_next_compact(workerId) + send_message('find_next_compact', Find_next_compact_args, :workerId => workerId) + end + + def recv_find_next_compact() + result = receive_message(Find_next_compact_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'find_next_compact failed: unknown result') + end + + def update_compactor_state(cr, txn_id) + send_update_compactor_state(cr, txn_id) + recv_update_compactor_state() + end + + def send_update_compactor_state(cr, txn_id) + send_message('update_compactor_state', Update_compactor_state_args, :cr => cr, :txn_id => txn_id) + end + + def recv_update_compactor_state() + result = receive_message(Update_compactor_state_result) + return + end + + def find_columns_with_stats(cr) + send_find_columns_with_stats(cr) + return recv_find_columns_with_stats() + end + + def send_find_columns_with_stats(cr) + send_message('find_columns_with_stats', Find_columns_with_stats_args, :cr => cr) + end + + def recv_find_columns_with_stats() + result = receive_message(Find_columns_with_stats_result) + return result.success unless result.success.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'find_columns_with_stats failed: unknown result') + end + + def mark_cleaned(cr) + send_mark_cleaned(cr) + recv_mark_cleaned() + end + + def send_mark_cleaned(cr) + send_message('mark_cleaned', Mark_cleaned_args, :cr => cr) + end + + def recv_mark_cleaned() + result = receive_message(Mark_cleaned_result) + raise result.o1 unless result.o1.nil? + return + end + + def mark_compacted(cr) + send_mark_compacted(cr) + recv_mark_compacted() + end + + def send_mark_compacted(cr) + send_message('mark_compacted', Mark_compacted_args, :cr => cr) + end + + def recv_mark_compacted() + result = receive_message(Mark_compacted_result) + raise result.o1 unless result.o1.nil? + return + end + + def mark_failed(cr) + send_mark_failed(cr) + recv_mark_failed() + end + + def send_mark_failed(cr) + send_message('mark_failed', Mark_failed_args, :cr => cr) + end + + def recv_mark_failed() + result = receive_message(Mark_failed_result) + raise result.o1 unless result.o1.nil? + return + end + + def set_hadoop_jobid(jobId, cq_id) + send_set_hadoop_jobid(jobId, cq_id) + recv_set_hadoop_jobid() + end + + def send_set_hadoop_jobid(jobId, cq_id) + send_message('set_hadoop_jobid', Set_hadoop_jobid_args, :jobId => jobId, :cq_id => cq_id) + end + + def recv_set_hadoop_jobid() + result = receive_message(Set_hadoop_jobid_result) + return + end + def get_next_notification(rqst) send_get_next_notification(rqst) return recv_get_next_notification() @@ -5705,6 +5809,71 @@ module ThriftHiveMetastore write_result(result, oprot, 'add_dynamic_partitions', seqid) end + def process_find_next_compact(seqid, iprot, oprot) + args = read_args(iprot, Find_next_compact_args) + result = Find_next_compact_result.new() + begin + result.success = @handler.find_next_compact(args.workerId) + rescue ::MetaException => o1 + result.o1 = o1 + end + write_result(result, oprot, 'find_next_compact', seqid) + end + + def process_update_compactor_state(seqid, iprot, oprot) + args = read_args(iprot, Update_compactor_state_args) + result = Update_compactor_state_result.new() + @handler.update_compactor_state(args.cr, args.txn_id) + write_result(result, oprot, 'update_compactor_state', seqid) + end + + def process_find_columns_with_stats(seqid, iprot, oprot) + args = read_args(iprot, Find_columns_with_stats_args) + result = Find_columns_with_stats_result.new() + result.success = @handler.find_columns_with_stats(args.cr) + write_result(result, oprot, 'find_columns_with_stats', seqid) + end + + def process_mark_cleaned(seqid, iprot, oprot) + args = read_args(iprot, Mark_cleaned_args) + result = Mark_cleaned_result.new() + begin + @handler.mark_cleaned(args.cr) + rescue ::MetaException => o1 + result.o1 = o1 + end + write_result(result, oprot, 'mark_cleaned', seqid) + end + + def process_mark_compacted(seqid, iprot, oprot) + args = read_args(iprot, Mark_compacted_args) + result = Mark_compacted_result.new() + begin + @handler.mark_compacted(args.cr) + rescue ::MetaException => o1 + result.o1 = o1 + end + write_result(result, oprot, 'mark_compacted', seqid) + end + + def process_mark_failed(seqid, iprot, oprot) + args = read_args(iprot, Mark_failed_args) + result = Mark_failed_result.new() + begin + @handler.mark_failed(args.cr) + rescue ::MetaException => o1 + result.o1 = o1 + end + write_result(result, oprot, 'mark_failed', seqid) + end + + def process_set_hadoop_jobid(seqid, iprot, oprot) + args = read_args(iprot, Set_hadoop_jobid_args) + result = Set_hadoop_jobid_result.new() + @handler.set_hadoop_jobid(args.jobId, args.cq_id) + write_result(result, oprot, 'set_hadoop_jobid', seqid) + end + def process_get_next_notification(seqid, iprot, oprot) args = read_args(iprot, Get_next_notification_args) result = Get_next_notification_result.new() @@ -12523,6 +12692,234 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Find_next_compact_args + include ::Thrift::Struct, ::Thrift::Struct_Union + WORKERID = 1 + + FIELDS = { + WORKERID => {:type => ::Thrift::Types::STRING, :name => 'workerId'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Find_next_compact_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::STRUCT, :name => 'success', :class => ::OptionalCompactionInfoStruct}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Update_compactor_state_args + include ::Thrift::Struct, ::Thrift::Struct_Union + CR = 1 + TXN_ID = 2 + + FIELDS = { + CR => {:type => ::Thrift::Types::STRUCT, :name => 'cr', :class => ::CompactionInfoStruct}, + TXN_ID => {:type => ::Thrift::Types::I64, :name => 'txn_id'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Update_compactor_state_result + include ::Thrift::Struct, ::Thrift::Struct_Union + + FIELDS = { + + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Find_columns_with_stats_args + include ::Thrift::Struct, ::Thrift::Struct_Union + CR = 1 + + FIELDS = { + CR => {:type => ::Thrift::Types::STRUCT, :name => 'cr', :class => ::CompactionInfoStruct} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Find_columns_with_stats_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRING}} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Mark_cleaned_args + include ::Thrift::Struct, ::Thrift::Struct_Union + CR = 1 + + FIELDS = { + CR => {:type => ::Thrift::Types::STRUCT, :name => 'cr', :class => ::CompactionInfoStruct} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Mark_cleaned_result + include ::Thrift::Struct, ::Thrift::Struct_Union + O1 = 1 + + FIELDS = { + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Mark_compacted_args + include ::Thrift::Struct, ::Thrift::Struct_Union + CR = 1 + + FIELDS = { + CR => {:type => ::Thrift::Types::STRUCT, :name => 'cr', :class => ::CompactionInfoStruct} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Mark_compacted_result + include ::Thrift::Struct, ::Thrift::Struct_Union + O1 = 1 + + FIELDS = { + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Mark_failed_args + include ::Thrift::Struct, ::Thrift::Struct_Union + CR = 1 + + FIELDS = { + CR => {:type => ::Thrift::Types::STRUCT, :name => 'cr', :class => ::CompactionInfoStruct} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Mark_failed_result + include ::Thrift::Struct, ::Thrift::Struct_Union + O1 = 1 + + FIELDS = { + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Set_hadoop_jobid_args + include ::Thrift::Struct, ::Thrift::Struct_Union + JOBID = 1 + CQ_ID = 2 + + FIELDS = { + JOBID => {:type => ::Thrift::Types::STRING, :name => 'jobId'}, + CQ_ID => {:type => ::Thrift::Types::I64, :name => 'cq_id'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Set_hadoop_jobid_result + include ::Thrift::Struct, ::Thrift::Struct_Union + + FIELDS = { + + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Get_next_notification_args include ::Thrift::Struct, ::Thrift::Struct_Union RQST = 1 http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 9eb1193..748b56b 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -3701,4 +3701,39 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { throws TException { return client.get_partitions_with_specs(request); } + + @Override + public OptionalCompactionInfoStruct findNextCompact(String workerId) throws MetaException, TException { + return client.find_next_compact(workerId); + } + + @Override + public void updateCompactorState(CompactionInfoStruct cr, long txnId) throws TException { + client.update_compactor_state(cr, txnId); + } + + @Override + public List<String> findColumnsWithStats(CompactionInfoStruct cr) throws TException { + return client.find_columns_with_stats(cr); + } + + @Override + public void markCleaned(CompactionInfoStruct cr) throws MetaException, TException { + client.mark_cleaned(cr); + } + + @Override + public void markCompacted(CompactionInfoStruct cr) throws MetaException, TException { + client.mark_compacted(cr); + } + + @Override + public void markFailed(CompactionInfoStruct cr) throws MetaException, TException { + client.mark_failed(cr); + } + + @Override + public void setHadoopJobid(String jobId, long cqId) throws MetaException, TException { + client.set_hadoop_jobid(jobId, cqId); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index fa19440..c96bf0d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -129,6 +129,8 @@ import org.apache.hadoop.hive.metastore.api.WMResourcePlan; import org.apache.hadoop.hive.metastore.api.WMTrigger; import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; import org.apache.hadoop.hive.metastore.api.WriteNotificationLogRequest; +import org.apache.hadoop.hive.metastore.api.CompactionInfoStruct; +import org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.thrift.TException; @@ -3805,4 +3807,61 @@ public interface IMetaStoreClient { */ GetPartitionsResponse getPartitionsWithSpecs(GetPartitionsRequest request) throws TException; + /** + * Get the next compaction job to do. + * @param workerId id of the worker requesting. + * @return next compaction job encapsulated in a {@link CompactionInfoStruct}. + * @throws MetaException + * @throws TException + */ + OptionalCompactionInfoStruct findNextCompact(String workerId) throws MetaException, TException; + + /** + * Set the compaction highest write id. + * @param cr compaction job being done. + * @param txnId transaction id. + * @throws TException + */ + void updateCompactorState(CompactionInfoStruct cr, long txnId) throws TException; + + /** + * Get columns. + * @param cr compaction job. + * @return + * @throws TException + */ + List<String> findColumnsWithStats(CompactionInfoStruct cr) throws TException; + + /** + * Mark a finished compaction as cleaned. + * @param cr compaction job. + * @throws MetaException + * @throws TException + */ + void markCleaned(CompactionInfoStruct cr) throws MetaException, TException; + + /** + * Mark a finished compaction as compacted. + * @param cr compaction job. + * @throws MetaException + * @throws TException + */ + void markCompacted(CompactionInfoStruct cr) throws MetaException, TException; + + /** + * Mark a finished compaction as failed. + * @param cr compaction job. + * @throws MetaException + * @throws TException + */ + void markFailed(CompactionInfoStruct cr) throws MetaException, TException; + + /** + * Set the hadoop id for a compaction. + * @param jobId mapreduce job id that will do the compaction. + * @param cqId compaction id. + * @throws MetaException + * @throws TException + */ + void setHadoopJobid(String jobId, long cqId) throws MetaException, TException; } http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index fb0b2fe..eb95e12 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -1037,6 +1037,10 @@ public class MetastoreConf { "Batch size for partition and other object retrieval from the underlying DB in JDO.\n" + "The JDO implementation such as DataNucleus may run into issues when the generated queries are\n" + "too large. Use this parameter to break the query into multiple batches. -1 means no batching."), + HIVE_METASTORE_RUNWORKER_IN("hive.metastore.runworker.in", + "hive.metastore.runworker.in", "metastore", new StringSetValidator("metastore", "hs2"), + "Chooses where the compactor worker threads should run, Only possible values" + + " are \"metastore\" and \"hs2\""), // Hive values we have copied and use as is // These two are used to indicate that we are running tests http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 9e4f3c2..7d09a5c 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -1076,6 +1076,25 @@ struct CompactionRequest { 6: optional map<string, string> properties } +struct OptionalCompactionInfoStruct { + 1: optional CompactionInfoStruct ci, +} + +struct CompactionInfoStruct { + 1: required i64 id, + 2: required string dbname, + 3: required string tablename, + 4: optional string partitionname, + 5: required CompactionType type, + 6: optional string runas, + 7: optional string properties + 8: optional bool toomanyaborts + 9: optional string state + 10: optional string workerId + 11: optional i64 start + 12: optional i64 highestWriteId +} + struct CompactionResponse { 1: required i64 id, 2: required string state, @@ -2319,6 +2338,13 @@ service ThriftHiveMetastore extends fb303.FacebookService CompactionResponse compact2(1:CompactionRequest rqst) ShowCompactResponse show_compact(1:ShowCompactRequest rqst) void add_dynamic_partitions(1:AddDynamicPartitions rqst) throws (1:NoSuchTxnException o1, 2:TxnAbortedException o2) + OptionalCompactionInfoStruct find_next_compact(1: string workerId) throws(1:MetaException o1) + void update_compactor_state(1: CompactionInfoStruct cr, 2: i64 txn_id) + list<string> find_columns_with_stats(1: CompactionInfoStruct cr) + void mark_cleaned(1:CompactionInfoStruct cr) throws(1:MetaException o1) + void mark_compacted(1: CompactionInfoStruct cr) throws(1:MetaException o1) + void mark_failed(1: CompactionInfoStruct cr) throws(1:MetaException o1) + void set_hadoop_jobid(1: string jobId, 2: i64 cq_id) // Notification logging calls NotificationEventResponse get_next_notification(1:NotificationEventRequest rqst) http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 0a1b96d..a9398ae 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -153,6 +153,7 @@ import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.security.MetastoreDelegationTokenManager; import org.apache.hadoop.hive.metastore.security.TUGIContainingTransport; +import org.apache.hadoop.hive.metastore.txn.CompactionInfo; import org.apache.hadoop.hive.metastore.txn.TxnStore; import org.apache.hadoop.hive.metastore.txn.TxnUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; @@ -7433,6 +7434,43 @@ public class HiveMetaStore extends ThriftHiveMetastore { } @Override + public void set_hadoop_jobid(String jobId, long cqId) { + getTxnHandler().setHadoopJobId(jobId, cqId); + } + + @Override + public OptionalCompactionInfoStruct find_next_compact(String workerId) throws MetaException{ + return CompactionInfo.compactionInfoToOptionalStruct( + getTxnHandler().findNextToCompact(workerId)); + } + + @Override + public void mark_cleaned(CompactionInfoStruct cr) throws MetaException { + getTxnHandler().markCleaned(CompactionInfo.compactionStructToInfo(cr)); + } + + @Override + public void mark_compacted(CompactionInfoStruct cr) throws MetaException { + getTxnHandler().markCompacted(CompactionInfo.compactionStructToInfo(cr)); + } + + @Override + public void mark_failed(CompactionInfoStruct cr) throws MetaException { + getTxnHandler().markFailed(CompactionInfo.compactionStructToInfo(cr)); + } + + @Override + public List<String> find_columns_with_stats(CompactionInfoStruct cr) throws MetaException { + return getTxnHandler().findColumnsWithStats(CompactionInfo.compactionStructToInfo(cr)); + } + + @Override + public void update_compactor_state(CompactionInfoStruct cr, long highWaterMark) throws MetaException { + getTxnHandler().updateCompactorState( + CompactionInfo.compactionStructToInfo(cr), highWaterMark); + } + + @Override public AllocateTableWriteIdsResponse allocate_table_write_ids( AllocateTableWriteIdsRequest rqst) throws TException { AllocateTableWriteIdsResponse response = getTxnHandler().allocateTableWriteIds(rqst); @@ -9546,7 +9584,9 @@ public class HiveMetaStore extends ThriftHiveMetastore { startCondition.await(); } startCompactorInitiator(conf); - startCompactorWorkers(conf); + if (MetastoreConf.getVar(conf, MetastoreConf.ConfVars.HIVE_METASTORE_RUNWORKER_IN).equals("metastore")) { + startCompactorWorkers(conf); + } startCompactorCleaner(conf); startRemoteOnlyTasks(conf); startStatsUpdater(conf); @@ -9613,7 +9653,7 @@ public class HiveMetaStore extends ThriftHiveMetastore { private static int nextThreadId = 1000000; private static void initializeAndStartThread(MetaStoreThread thread, Configuration conf) throws - MetaException { + Exception { LOG.info("Starting metastore thread of type " + thread.getClass().getName()); thread.setConf(conf); thread.setThreadId(nextThreadId++); http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java index 6ef2e35..ea61552 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreThread.java @@ -19,7 +19,6 @@ package org.apache.hadoop.hive.metastore; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.api.MetaException; import java.util.concurrent.atomic.AtomicBoolean; @@ -47,7 +46,7 @@ public interface MetaStoreThread extends Configurable { * least once. */ // TODO: move these test parameters to more specific places... there's no need to have them here - void init(AtomicBoolean stop, AtomicBoolean looped) throws MetaException; + void init(AtomicBoolean stop, AtomicBoolean looped) throws Exception; /** * Run the thread in the background. This must not be called until http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java index b34b7d7..ea70503 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/CompactionInfo.java @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.metastore.txn; import org.apache.hadoop.hive.common.ValidCompactorWriteIdList; +import org.apache.hadoop.hive.metastore.api.CompactionInfoStruct; import org.apache.hadoop.hive.metastore.api.CompactionType; +import org.apache.hadoop.hive.metastore.api.OptionalCompactionInfoStruct; import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; import java.sql.PreparedStatement; @@ -29,14 +31,22 @@ import java.sql.SQLException; * Information on a possible or running compaction. */ public class CompactionInfo implements Comparable<CompactionInfo> { + + /** + * Modifying this variables or adding new ones should be done in sync + * with the static methods {@code compactionStructToInfo()} and + * {@code compactionInfoToStruct()}. This class is going to be deserialized + * and serialized so missing this may result in the value of the field + * being resetted. This will be fixed at HIVE-21056. + */ public long id; public String dbname; public String tableName; public String partName; - char state; + public char state; public CompactionType type; - String workerId; - long start; + public String workerId; + public long start; public String runAs; public String properties; public boolean tooManyAborts = false; @@ -167,4 +177,61 @@ public class CompactionInfo implements Comparable<CompactionInfo> { pStmt.setBytes(13, ci.metaInfo); pStmt.setString(14, ci.hadoopJobId); } + + public static CompactionInfo compactionStructToInfo(CompactionInfoStruct cr) { + if (cr == null) { + return null; + } + CompactionInfo ci = new CompactionInfo(cr.getDbname(), cr.getTablename(), cr.getPartitionname(), cr.getType()); + ci.id = cr.getId(); + ci.runAs = cr.getRunas(); + ci.properties = cr.getProperties(); + if (cr.isSetToomanyaborts()) { + ci.tooManyAborts = cr.isToomanyaborts(); + } + if (cr.isSetState() && cr.getState().length() != 1) { + throw new IllegalStateException("State should only be one character but it was set to " + cr.getState()); + } else if (cr.isSetState()) { + ci.state = cr.getState().charAt(0); + } + ci.workerId = cr.getWorkerId(); + if (cr.isSetStart()) { + ci.start = cr.getStart(); + } + if (cr.isSetHighestWriteId()) { + ci.highestWriteId = cr.getHighestWriteId(); + } + return ci; + } + + public static CompactionInfoStruct compactionInfoToStruct(CompactionInfo ci) { + if (ci == null) { + return null; + } + CompactionInfoStruct cr = new CompactionInfoStruct(ci.id, ci.dbname, ci.tableName, ci.type); + cr.setPartitionname(ci.partName); + cr.setRunas(ci.runAs); + cr.setProperties(ci.properties); + cr.setToomanyaborts(ci.tooManyAborts); + cr.setStart(ci.start); + cr.setState(Character.toString(ci.state)); + cr.setWorkerId(ci.workerId); + cr.setHighestWriteId(ci.highestWriteId); + return cr; + } + + public static OptionalCompactionInfoStruct compactionInfoToOptionalStruct(CompactionInfo ci) { + CompactionInfoStruct cis = compactionInfoToStruct(ci); + OptionalCompactionInfoStruct ocis = new OptionalCompactionInfoStruct(); + if (cis != null) { + ocis.setCi(cis); + } + return ocis; + } + public static CompactionInfo optionalCompactionInfoStructToInfo(OptionalCompactionInfoStruct ocis) { + if (ocis.isSetCi()) { + return compactionStructToInfo(ocis.getCi()); + } + return null; + } } http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java index 3f611d6..ea4e394 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java @@ -65,8 +65,6 @@ import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.metastore.ColumnType; import org.apache.hadoop.hive.metastore.HiveMetaStore; import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.RawStore; -import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 9fe9a65..a3f5e2c 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -3571,4 +3571,39 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos throws TException { throw new UnsupportedOperationException(); } + + @Override + public OptionalCompactionInfoStruct findNextCompact(String workerId) throws MetaException, TException { + return client.find_next_compact(workerId); + } + + @Override + public void updateCompactorState(CompactionInfoStruct cr, long txnId) throws TException { + client.update_compactor_state(cr, txnId); + } + + @Override + public List<String> findColumnsWithStats(CompactionInfoStruct cr) throws TException { + return client.find_columns_with_stats(cr); + } + + @Override + public void markCleaned(CompactionInfoStruct cr) throws MetaException, TException { + client.mark_cleaned(cr); + } + + @Override + public void markCompacted(CompactionInfoStruct cr) throws MetaException, TException { + client.mark_compacted(cr); + } + + @Override + public void markFailed(CompactionInfoStruct cr) throws MetaException, TException { + client.mark_failed(cr); + } + + @Override + public void setHadoopJobid(String jobId, long cqId) throws MetaException, TException { + client.set_hadoop_jobid(jobId, cqId); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/926c1e8e/streaming/src/test/org/apache/hive/streaming/TestStreaming.java ---------------------------------------------------------------------- diff --git a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java index 2170178..c6d7e7f 100644 --- a/streaming/src/test/org/apache/hive/streaming/TestStreaming.java +++ b/streaming/src/test/org/apache/hive/streaming/TestStreaming.java @@ -63,7 +63,6 @@ import org.apache.hadoop.hive.metastore.IMetaStoreClient; import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.metastore.api.LockType; -import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.ShowLocksRequest; @@ -783,7 +782,7 @@ public class TestStreaming { /** * this is a clone from TestHiveStreamingConnection.TxnStatement2.... */ - public static void runWorker(HiveConf hiveConf) throws MetaException { + public static void runWorker(HiveConf hiveConf) throws Exception { AtomicBoolean stop = new AtomicBoolean(true); Worker t = new Worker(); t.setThreadId((int) t.getId());
