Modified: hive/branches/tez/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original) +++ hive/branches/tez/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Fri Sep 5 19:15:44 2014 @@ -2889,6 +2889,406 @@ class Partition: def __ne__(self, other): return not (self == other) +class PartitionWithoutSD: + """ + Attributes: + - values + - createTime + - lastAccessTime + - relativePath + - parameters + - privileges + """ + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'values', (TType.STRING,None), None, ), # 1 + (2, TType.I32, 'createTime', None, None, ), # 2 + (3, TType.I32, 'lastAccessTime', None, None, ), # 3 + (4, TType.STRING, 'relativePath', None, None, ), # 4 + (5, TType.MAP, 'parameters', (TType.STRING,None,TType.STRING,None), None, ), # 5 + (6, TType.STRUCT, 'privileges', (PrincipalPrivilegeSet, PrincipalPrivilegeSet.thrift_spec), None, ), # 6 + ) + + def __init__(self, values=None, createTime=None, lastAccessTime=None, relativePath=None, parameters=None, privileges=None,): + self.values = values + self.createTime = createTime + self.lastAccessTime = lastAccessTime + self.relativePath = relativePath + self.parameters = parameters + self.privileges = privileges + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.values = [] + (_etype203, _size200) = iprot.readListBegin() + for _i204 in xrange(_size200): + _elem205 = iprot.readString(); + self.values.append(_elem205) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.createTime = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.lastAccessTime = iprot.readI32(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRING: + self.relativePath = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.MAP: + self.parameters = {} + (_ktype207, _vtype208, _size206 ) = iprot.readMapBegin() + for _i210 in xrange(_size206): + _key211 = iprot.readString(); + _val212 = iprot.readString(); + self.parameters[_key211] = _val212 + iprot.readMapEnd() + else: + iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRUCT: + self.privileges = PrincipalPrivilegeSet() + self.privileges.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('PartitionWithoutSD') + if self.values is not None: + oprot.writeFieldBegin('values', TType.LIST, 1) + oprot.writeListBegin(TType.STRING, len(self.values)) + for iter213 in self.values: + oprot.writeString(iter213) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.createTime is not None: + oprot.writeFieldBegin('createTime', TType.I32, 2) + oprot.writeI32(self.createTime) + oprot.writeFieldEnd() + if self.lastAccessTime is not None: + oprot.writeFieldBegin('lastAccessTime', TType.I32, 3) + oprot.writeI32(self.lastAccessTime) + oprot.writeFieldEnd() + if self.relativePath is not None: + oprot.writeFieldBegin('relativePath', TType.STRING, 4) + oprot.writeString(self.relativePath) + oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin('parameters', TType.MAP, 5) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) + for kiter214,viter215 in self.parameters.items(): + oprot.writeString(kiter214) + oprot.writeString(viter215) + oprot.writeMapEnd() + oprot.writeFieldEnd() + if self.privileges is not None: + oprot.writeFieldBegin('privileges', TType.STRUCT, 6) + self.privileges.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class PartitionSpecWithSharedSD: + """ + Attributes: + - partitions + - sd + """ + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'partitions', (TType.STRUCT,(PartitionWithoutSD, PartitionWithoutSD.thrift_spec)), None, ), # 1 + (2, TType.STRUCT, 'sd', (StorageDescriptor, StorageDescriptor.thrift_spec), None, ), # 2 + ) + + def __init__(self, partitions=None, sd=None,): + self.partitions = partitions + self.sd = sd + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype219, _size216) = iprot.readListBegin() + for _i220 in xrange(_size216): + _elem221 = PartitionWithoutSD() + _elem221.read(iprot) + self.partitions.append(_elem221) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.sd = StorageDescriptor() + self.sd.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('PartitionSpecWithSharedSD') + if self.partitions is not None: + oprot.writeFieldBegin('partitions', TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter222 in self.partitions: + iter222.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.sd is not None: + oprot.writeFieldBegin('sd', TType.STRUCT, 2) + self.sd.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class PartitionListComposingSpec: + """ + Attributes: + - partitions + """ + + thrift_spec = ( + None, # 0 + (1, TType.LIST, 'partitions', (TType.STRUCT,(Partition, Partition.thrift_spec)), None, ), # 1 + ) + + def __init__(self, partitions=None,): + self.partitions = partitions + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.LIST: + self.partitions = [] + (_etype226, _size223) = iprot.readListBegin() + for _i227 in xrange(_size223): + _elem228 = Partition() + _elem228.read(iprot) + self.partitions.append(_elem228) + iprot.readListEnd() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('PartitionListComposingSpec') + if self.partitions is not None: + oprot.writeFieldBegin('partitions', TType.LIST, 1) + oprot.writeListBegin(TType.STRUCT, len(self.partitions)) + for iter229 in self.partitions: + iter229.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class PartitionSpec: + """ + Attributes: + - dbName + - tableName + - rootPath + - sharedSDPartitionSpec + - partitionList + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'dbName', None, None, ), # 1 + (2, TType.STRING, 'tableName', None, None, ), # 2 + (3, TType.STRING, 'rootPath', None, None, ), # 3 + (4, TType.STRUCT, 'sharedSDPartitionSpec', (PartitionSpecWithSharedSD, PartitionSpecWithSharedSD.thrift_spec), None, ), # 4 + (5, TType.STRUCT, 'partitionList', (PartitionListComposingSpec, PartitionListComposingSpec.thrift_spec), None, ), # 5 + ) + + def __init__(self, dbName=None, tableName=None, rootPath=None, sharedSDPartitionSpec=None, partitionList=None,): + self.dbName = dbName + self.tableName = tableName + self.rootPath = rootPath + self.sharedSDPartitionSpec = sharedSDPartitionSpec + self.partitionList = partitionList + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.dbName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tableName = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.rootPath = iprot.readString(); + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.sharedSDPartitionSpec = PartitionSpecWithSharedSD() + self.sharedSDPartitionSpec.read(iprot) + else: + iprot.skip(ftype) + elif fid == 5: + if ftype == TType.STRUCT: + self.partitionList = PartitionListComposingSpec() + self.partitionList.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('PartitionSpec') + if self.dbName is not None: + oprot.writeFieldBegin('dbName', TType.STRING, 1) + oprot.writeString(self.dbName) + oprot.writeFieldEnd() + if self.tableName is not None: + oprot.writeFieldBegin('tableName', TType.STRING, 2) + oprot.writeString(self.tableName) + oprot.writeFieldEnd() + if self.rootPath is not None: + oprot.writeFieldBegin('rootPath', TType.STRING, 3) + oprot.writeString(self.rootPath) + oprot.writeFieldEnd() + if self.sharedSDPartitionSpec is not None: + oprot.writeFieldBegin('sharedSDPartitionSpec', TType.STRUCT, 4) + self.sharedSDPartitionSpec.write(oprot) + oprot.writeFieldEnd() + if self.partitionList is not None: + oprot.writeFieldBegin('partitionList', TType.STRUCT, 5) + self.partitionList.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + class Index: """ Attributes: @@ -2983,11 +3383,11 @@ class Index: elif fid == 9: if ftype == TType.MAP: self.parameters = {} - (_ktype201, _vtype202, _size200 ) = iprot.readMapBegin() - for _i204 in xrange(_size200): - _key205 = iprot.readString(); - _val206 = iprot.readString(); - self.parameters[_key205] = _val206 + (_ktype231, _vtype232, _size230 ) = iprot.readMapBegin() + for _i234 in xrange(_size230): + _key235 = iprot.readString(); + _val236 = iprot.readString(); + self.parameters[_key235] = _val236 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3041,9 +3441,9 @@ class Index: if self.parameters is not None: oprot.writeFieldBegin('parameters', TType.MAP, 9) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.parameters)) - for kiter207,viter208 in self.parameters.items(): - oprot.writeString(kiter207) - oprot.writeString(viter208) + for kiter237,viter238 in self.parameters.items(): + oprot.writeString(kiter237) + oprot.writeString(viter238) oprot.writeMapEnd() oprot.writeFieldEnd() if self.deferredRebuild is not None: @@ -4097,11 +4497,11 @@ class ColumnStatistics: elif fid == 2: if ftype == TType.LIST: self.statsObj = [] - (_etype212, _size209) = iprot.readListBegin() - for _i213 in xrange(_size209): - _elem214 = ColumnStatisticsObj() - _elem214.read(iprot) - self.statsObj.append(_elem214) + (_etype242, _size239) = iprot.readListBegin() + for _i243 in xrange(_size239): + _elem244 = ColumnStatisticsObj() + _elem244.read(iprot) + self.statsObj.append(_elem244) iprot.readListEnd() else: iprot.skip(ftype) @@ -4122,8 +4522,8 @@ class ColumnStatistics: if self.statsObj is not None: oprot.writeFieldBegin('statsObj', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.statsObj)) - for iter215 in self.statsObj: - iter215.write(oprot) + for iter245 in self.statsObj: + iter245.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4177,11 +4577,11 @@ class AggrStats: if fid == 1: if ftype == TType.LIST: self.colStats = [] - (_etype219, _size216) = iprot.readListBegin() - for _i220 in xrange(_size216): - _elem221 = ColumnStatisticsObj() - _elem221.read(iprot) - self.colStats.append(_elem221) + (_etype249, _size246) = iprot.readListBegin() + for _i250 in xrange(_size246): + _elem251 = ColumnStatisticsObj() + _elem251.read(iprot) + self.colStats.append(_elem251) iprot.readListEnd() else: iprot.skip(ftype) @@ -4203,8 +4603,8 @@ class AggrStats: if self.colStats is not None: oprot.writeFieldBegin('colStats', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.colStats)) - for iter222 in self.colStats: - iter222.write(oprot) + for iter252 in self.colStats: + iter252.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.partsFound is not None: @@ -4259,11 +4659,11 @@ class SetPartitionsStatsRequest: if fid == 1: if ftype == TType.LIST: self.colStats = [] - (_etype226, _size223) = iprot.readListBegin() - for _i227 in xrange(_size223): - _elem228 = ColumnStatistics() - _elem228.read(iprot) - self.colStats.append(_elem228) + (_etype256, _size253) = iprot.readListBegin() + for _i257 in xrange(_size253): + _elem258 = ColumnStatistics() + _elem258.read(iprot) + self.colStats.append(_elem258) iprot.readListEnd() else: iprot.skip(ftype) @@ -4280,8 +4680,8 @@ class SetPartitionsStatsRequest: if self.colStats is not None: oprot.writeFieldBegin('colStats', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.colStats)) - for iter229 in self.colStats: - iter229.write(oprot) + for iter259 in self.colStats: + iter259.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4333,22 +4733,22 @@ class Schema: if fid == 1: if ftype == TType.LIST: self.fieldSchemas = [] - (_etype233, _size230) = iprot.readListBegin() - for _i234 in xrange(_size230): - _elem235 = FieldSchema() - _elem235.read(iprot) - self.fieldSchemas.append(_elem235) + (_etype263, _size260) = iprot.readListBegin() + for _i264 in xrange(_size260): + _elem265 = FieldSchema() + _elem265.read(iprot) + self.fieldSchemas.append(_elem265) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.MAP: self.properties = {} - (_ktype237, _vtype238, _size236 ) = iprot.readMapBegin() - for _i240 in xrange(_size236): - _key241 = iprot.readString(); - _val242 = iprot.readString(); - self.properties[_key241] = _val242 + (_ktype267, _vtype268, _size266 ) = iprot.readMapBegin() + for _i270 in xrange(_size266): + _key271 = iprot.readString(); + _val272 = iprot.readString(); + self.properties[_key271] = _val272 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4365,16 +4765,16 @@ class Schema: if self.fieldSchemas is not None: oprot.writeFieldBegin('fieldSchemas', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.fieldSchemas)) - for iter243 in self.fieldSchemas: - iter243.write(oprot) + for iter273 in self.fieldSchemas: + iter273.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.properties is not None: oprot.writeFieldBegin('properties', TType.MAP, 2) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) - for kiter244,viter245 in self.properties.items(): - oprot.writeString(kiter244) - oprot.writeString(viter245) + for kiter274,viter275 in self.properties.items(): + oprot.writeString(kiter274) + oprot.writeString(viter275) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4421,11 +4821,11 @@ class EnvironmentContext: if fid == 1: if ftype == TType.MAP: self.properties = {} - (_ktype247, _vtype248, _size246 ) = iprot.readMapBegin() - for _i250 in xrange(_size246): - _key251 = iprot.readString(); - _val252 = iprot.readString(); - self.properties[_key251] = _val252 + (_ktype277, _vtype278, _size276 ) = iprot.readMapBegin() + for _i280 in xrange(_size276): + _key281 = iprot.readString(); + _val282 = iprot.readString(); + self.properties[_key281] = _val282 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4442,9 +4842,9 @@ class EnvironmentContext: if self.properties is not None: oprot.writeFieldBegin('properties', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) - for kiter253,viter254 in self.properties.items(): - oprot.writeString(kiter253) - oprot.writeString(viter254) + for kiter283,viter284 in self.properties.items(): + oprot.writeString(kiter283) + oprot.writeString(viter284) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4494,11 +4894,11 @@ class PartitionsByExprResult: if fid == 1: if ftype == TType.LIST: self.partitions = [] - (_etype258, _size255) = iprot.readListBegin() - for _i259 in xrange(_size255): - _elem260 = Partition() - _elem260.read(iprot) - self.partitions.append(_elem260) + (_etype288, _size285) = iprot.readListBegin() + for _i289 in xrange(_size285): + _elem290 = Partition() + _elem290.read(iprot) + self.partitions.append(_elem290) iprot.readListEnd() else: iprot.skip(ftype) @@ -4520,8 +4920,8 @@ class PartitionsByExprResult: if self.partitions is not None: oprot.writeFieldBegin('partitions', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.partitions)) - for iter261 in self.partitions: - iter261.write(oprot) + for iter291 in self.partitions: + iter291.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.hasUnknownPartitions is not None: @@ -4690,11 +5090,11 @@ class TableStatsResult: if fid == 1: if ftype == TType.LIST: self.tableStats = [] - (_etype265, _size262) = iprot.readListBegin() - for _i266 in xrange(_size262): - _elem267 = ColumnStatisticsObj() - _elem267.read(iprot) - self.tableStats.append(_elem267) + (_etype295, _size292) = iprot.readListBegin() + for _i296 in xrange(_size292): + _elem297 = ColumnStatisticsObj() + _elem297.read(iprot) + self.tableStats.append(_elem297) iprot.readListEnd() else: iprot.skip(ftype) @@ -4711,8 +5111,8 @@ class TableStatsResult: if self.tableStats is not None: oprot.writeFieldBegin('tableStats', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.tableStats)) - for iter268 in self.tableStats: - iter268.write(oprot) + for iter298 in self.tableStats: + iter298.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4761,17 +5161,17 @@ class PartitionsStatsResult: if fid == 1: if ftype == TType.MAP: self.partStats = {} - (_ktype270, _vtype271, _size269 ) = iprot.readMapBegin() - for _i273 in xrange(_size269): - _key274 = iprot.readString(); - _val275 = [] - (_etype279, _size276) = iprot.readListBegin() - for _i280 in xrange(_size276): - _elem281 = ColumnStatisticsObj() - _elem281.read(iprot) - _val275.append(_elem281) + (_ktype300, _vtype301, _size299 ) = iprot.readMapBegin() + for _i303 in xrange(_size299): + _key304 = iprot.readString(); + _val305 = [] + (_etype309, _size306) = iprot.readListBegin() + for _i310 in xrange(_size306): + _elem311 = ColumnStatisticsObj() + _elem311.read(iprot) + _val305.append(_elem311) iprot.readListEnd() - self.partStats[_key274] = _val275 + self.partStats[_key304] = _val305 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4788,11 +5188,11 @@ class PartitionsStatsResult: if self.partStats is not None: oprot.writeFieldBegin('partStats', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.LIST, len(self.partStats)) - for kiter282,viter283 in self.partStats.items(): - oprot.writeString(kiter282) - oprot.writeListBegin(TType.STRUCT, len(viter283)) - for iter284 in viter283: - iter284.write(oprot) + for kiter312,viter313 in self.partStats.items(): + oprot.writeString(kiter312) + oprot.writeListBegin(TType.STRUCT, len(viter313)) + for iter314 in viter313: + iter314.write(oprot) oprot.writeListEnd() oprot.writeMapEnd() oprot.writeFieldEnd() @@ -4858,10 +5258,10 @@ class TableStatsRequest: elif fid == 3: if ftype == TType.LIST: self.colNames = [] - (_etype288, _size285) = iprot.readListBegin() - for _i289 in xrange(_size285): - _elem290 = iprot.readString(); - self.colNames.append(_elem290) + (_etype318, _size315) = iprot.readListBegin() + for _i319 in xrange(_size315): + _elem320 = iprot.readString(); + self.colNames.append(_elem320) iprot.readListEnd() else: iprot.skip(ftype) @@ -4886,8 +5286,8 @@ class TableStatsRequest: if self.colNames is not None: oprot.writeFieldBegin('colNames', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.colNames)) - for iter291 in self.colNames: - oprot.writeString(iter291) + for iter321 in self.colNames: + oprot.writeString(iter321) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -4959,20 +5359,20 @@ class PartitionsStatsRequest: elif fid == 3: if ftype == TType.LIST: self.colNames = [] - (_etype295, _size292) = iprot.readListBegin() - for _i296 in xrange(_size292): - _elem297 = iprot.readString(); - self.colNames.append(_elem297) + (_etype325, _size322) = iprot.readListBegin() + for _i326 in xrange(_size322): + _elem327 = iprot.readString(); + self.colNames.append(_elem327) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 4: if ftype == TType.LIST: self.partNames = [] - (_etype301, _size298) = iprot.readListBegin() - for _i302 in xrange(_size298): - _elem303 = iprot.readString(); - self.partNames.append(_elem303) + (_etype331, _size328) = iprot.readListBegin() + for _i332 in xrange(_size328): + _elem333 = iprot.readString(); + self.partNames.append(_elem333) iprot.readListEnd() else: iprot.skip(ftype) @@ -4997,15 +5397,15 @@ class PartitionsStatsRequest: if self.colNames is not None: oprot.writeFieldBegin('colNames', TType.LIST, 3) oprot.writeListBegin(TType.STRING, len(self.colNames)) - for iter304 in self.colNames: - oprot.writeString(iter304) + for iter334 in self.colNames: + oprot.writeString(iter334) oprot.writeListEnd() oprot.writeFieldEnd() if self.partNames is not None: oprot.writeFieldBegin('partNames', TType.LIST, 4) oprot.writeListBegin(TType.STRING, len(self.partNames)) - for iter305 in self.partNames: - oprot.writeString(iter305) + for iter335 in self.partNames: + oprot.writeString(iter335) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -5060,11 +5460,11 @@ class AddPartitionsResult: if fid == 1: if ftype == TType.LIST: self.partitions = [] - (_etype309, _size306) = iprot.readListBegin() - for _i310 in xrange(_size306): - _elem311 = Partition() - _elem311.read(iprot) - self.partitions.append(_elem311) + (_etype339, _size336) = iprot.readListBegin() + for _i340 in xrange(_size336): + _elem341 = Partition() + _elem341.read(iprot) + self.partitions.append(_elem341) iprot.readListEnd() else: iprot.skip(ftype) @@ -5081,8 +5481,8 @@ class AddPartitionsResult: if self.partitions is not None: oprot.writeFieldBegin('partitions', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.partitions)) - for iter312 in self.partitions: - iter312.write(oprot) + for iter342 in self.partitions: + iter342.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -5151,11 +5551,11 @@ class AddPartitionsRequest: elif fid == 3: if ftype == TType.LIST: self.parts = [] - (_etype316, _size313) = iprot.readListBegin() - for _i317 in xrange(_size313): - _elem318 = Partition() - _elem318.read(iprot) - self.parts.append(_elem318) + (_etype346, _size343) = iprot.readListBegin() + for _i347 in xrange(_size343): + _elem348 = Partition() + _elem348.read(iprot) + self.parts.append(_elem348) iprot.readListEnd() else: iprot.skip(ftype) @@ -5190,8 +5590,8 @@ class AddPartitionsRequest: if self.parts is not None: oprot.writeFieldBegin('parts', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.parts)) - for iter319 in self.parts: - iter319.write(oprot) + for iter349 in self.parts: + iter349.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.ifNotExists is not None: @@ -5254,11 +5654,11 @@ class DropPartitionsResult: if fid == 1: if ftype == TType.LIST: self.partitions = [] - (_etype323, _size320) = iprot.readListBegin() - for _i324 in xrange(_size320): - _elem325 = Partition() - _elem325.read(iprot) - self.partitions.append(_elem325) + (_etype353, _size350) = iprot.readListBegin() + for _i354 in xrange(_size350): + _elem355 = Partition() + _elem355.read(iprot) + self.partitions.append(_elem355) iprot.readListEnd() else: iprot.skip(ftype) @@ -5275,8 +5675,8 @@ class DropPartitionsResult: if self.partitions is not None: oprot.writeFieldBegin('partitions', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.partitions)) - for iter326 in self.partitions: - iter326.write(oprot) + for iter356 in self.partitions: + iter356.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -5400,21 +5800,21 @@ class RequestPartsSpec: if fid == 1: if ftype == TType.LIST: self.names = [] - (_etype330, _size327) = iprot.readListBegin() - for _i331 in xrange(_size327): - _elem332 = iprot.readString(); - self.names.append(_elem332) + (_etype360, _size357) = iprot.readListBegin() + for _i361 in xrange(_size357): + _elem362 = iprot.readString(); + self.names.append(_elem362) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.exprs = [] - (_etype336, _size333) = iprot.readListBegin() - for _i337 in xrange(_size333): - _elem338 = DropPartitionsExpr() - _elem338.read(iprot) - self.exprs.append(_elem338) + (_etype366, _size363) = iprot.readListBegin() + for _i367 in xrange(_size363): + _elem368 = DropPartitionsExpr() + _elem368.read(iprot) + self.exprs.append(_elem368) iprot.readListEnd() else: iprot.skip(ftype) @@ -5431,15 +5831,15 @@ class RequestPartsSpec: if self.names is not None: oprot.writeFieldBegin('names', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.names)) - for iter339 in self.names: - oprot.writeString(iter339) + for iter369 in self.names: + oprot.writeString(iter369) oprot.writeListEnd() oprot.writeFieldEnd() if self.exprs is not None: oprot.writeFieldBegin('exprs', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.exprs)) - for iter340 in self.exprs: - iter340.write(oprot) + for iter370 in self.exprs: + iter370.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -5766,11 +6166,11 @@ class Function: elif fid == 8: if ftype == TType.LIST: self.resourceUris = [] - (_etype344, _size341) = iprot.readListBegin() - for _i345 in xrange(_size341): - _elem346 = ResourceUri() - _elem346.read(iprot) - self.resourceUris.append(_elem346) + (_etype374, _size371) = iprot.readListBegin() + for _i375 in xrange(_size371): + _elem376 = ResourceUri() + _elem376.read(iprot) + self.resourceUris.append(_elem376) iprot.readListEnd() else: iprot.skip(ftype) @@ -5815,8 +6215,8 @@ class Function: if self.resourceUris is not None: oprot.writeFieldBegin('resourceUris', TType.LIST, 8) oprot.writeListBegin(TType.STRUCT, len(self.resourceUris)) - for iter347 in self.resourceUris: - iter347.write(oprot) + for iter377 in self.resourceUris: + iter377.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -5975,11 +6375,11 @@ class GetOpenTxnsInfoResponse: elif fid == 2: if ftype == TType.LIST: self.open_txns = [] - (_etype351, _size348) = iprot.readListBegin() - for _i352 in xrange(_size348): - _elem353 = TxnInfo() - _elem353.read(iprot) - self.open_txns.append(_elem353) + (_etype381, _size378) = iprot.readListBegin() + for _i382 in xrange(_size378): + _elem383 = TxnInfo() + _elem383.read(iprot) + self.open_txns.append(_elem383) iprot.readListEnd() else: iprot.skip(ftype) @@ -6000,8 +6400,8 @@ class GetOpenTxnsInfoResponse: if self.open_txns is not None: oprot.writeFieldBegin('open_txns', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.open_txns)) - for iter354 in self.open_txns: - iter354.write(oprot) + for iter384 in self.open_txns: + iter384.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -6060,10 +6460,10 @@ class GetOpenTxnsResponse: elif fid == 2: if ftype == TType.SET: self.open_txns = set() - (_etype358, _size355) = iprot.readSetBegin() - for _i359 in xrange(_size355): - _elem360 = iprot.readI64(); - self.open_txns.add(_elem360) + (_etype388, _size385) = iprot.readSetBegin() + for _i389 in xrange(_size385): + _elem390 = iprot.readI64(); + self.open_txns.add(_elem390) iprot.readSetEnd() else: iprot.skip(ftype) @@ -6084,8 +6484,8 @@ class GetOpenTxnsResponse: if self.open_txns is not None: oprot.writeFieldBegin('open_txns', TType.SET, 2) oprot.writeSetBegin(TType.I64, len(self.open_txns)) - for iter361 in self.open_txns: - oprot.writeI64(iter361) + for iter391 in self.open_txns: + oprot.writeI64(iter391) oprot.writeSetEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -6226,10 +6626,10 @@ class OpenTxnsResponse: if fid == 1: if ftype == TType.LIST: self.txn_ids = [] - (_etype365, _size362) = iprot.readListBegin() - for _i366 in xrange(_size362): - _elem367 = iprot.readI64(); - self.txn_ids.append(_elem367) + (_etype395, _size392) = iprot.readListBegin() + for _i396 in xrange(_size392): + _elem397 = iprot.readI64(); + self.txn_ids.append(_elem397) iprot.readListEnd() else: iprot.skip(ftype) @@ -6246,8 +6646,8 @@ class OpenTxnsResponse: if self.txn_ids is not None: oprot.writeFieldBegin('txn_ids', TType.LIST, 1) oprot.writeListBegin(TType.I64, len(self.txn_ids)) - for iter368 in self.txn_ids: - oprot.writeI64(iter368) + for iter398 in self.txn_ids: + oprot.writeI64(iter398) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -6543,11 +6943,11 @@ class LockRequest: if fid == 1: if ftype == TType.LIST: self.component = [] - (_etype372, _size369) = iprot.readListBegin() - for _i373 in xrange(_size369): - _elem374 = LockComponent() - _elem374.read(iprot) - self.component.append(_elem374) + (_etype402, _size399) = iprot.readListBegin() + for _i403 in xrange(_size399): + _elem404 = LockComponent() + _elem404.read(iprot) + self.component.append(_elem404) iprot.readListEnd() else: iprot.skip(ftype) @@ -6579,8 +6979,8 @@ class LockRequest: if self.component is not None: oprot.writeFieldBegin('component', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.component)) - for iter375 in self.component: - iter375.write(oprot) + for iter405 in self.component: + iter405.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.txnid is not None: @@ -7081,11 +7481,11 @@ class ShowLocksResponse: if fid == 1: if ftype == TType.LIST: self.locks = [] - (_etype379, _size376) = iprot.readListBegin() - for _i380 in xrange(_size376): - _elem381 = ShowLocksResponseElement() - _elem381.read(iprot) - self.locks.append(_elem381) + (_etype409, _size406) = iprot.readListBegin() + for _i410 in xrange(_size406): + _elem411 = ShowLocksResponseElement() + _elem411.read(iprot) + self.locks.append(_elem411) iprot.readListEnd() else: iprot.skip(ftype) @@ -7102,8 +7502,8 @@ class ShowLocksResponse: if self.locks is not None: oprot.writeFieldBegin('locks', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.locks)) - for iter382 in self.locks: - iter382.write(oprot) + for iter412 in self.locks: + iter412.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -7301,20 +7701,20 @@ class HeartbeatTxnRangeResponse: if fid == 1: if ftype == TType.SET: self.aborted = set() - (_etype386, _size383) = iprot.readSetBegin() - for _i387 in xrange(_size383): - _elem388 = iprot.readI64(); - self.aborted.add(_elem388) + (_etype416, _size413) = iprot.readSetBegin() + for _i417 in xrange(_size413): + _elem418 = iprot.readI64(); + self.aborted.add(_elem418) iprot.readSetEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.SET: self.nosuch = set() - (_etype392, _size389) = iprot.readSetBegin() - for _i393 in xrange(_size389): - _elem394 = iprot.readI64(); - self.nosuch.add(_elem394) + (_etype422, _size419) = iprot.readSetBegin() + for _i423 in xrange(_size419): + _elem424 = iprot.readI64(); + self.nosuch.add(_elem424) iprot.readSetEnd() else: iprot.skip(ftype) @@ -7331,15 +7731,15 @@ class HeartbeatTxnRangeResponse: if self.aborted is not None: oprot.writeFieldBegin('aborted', TType.SET, 1) oprot.writeSetBegin(TType.I64, len(self.aborted)) - for iter395 in self.aborted: - oprot.writeI64(iter395) + for iter425 in self.aborted: + oprot.writeI64(iter425) oprot.writeSetEnd() oprot.writeFieldEnd() if self.nosuch is not None: oprot.writeFieldBegin('nosuch', TType.SET, 2) oprot.writeSetBegin(TType.I64, len(self.nosuch)) - for iter396 in self.nosuch: - oprot.writeI64(iter396) + for iter426 in self.nosuch: + oprot.writeI64(iter426) oprot.writeSetEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -7706,11 +8106,11 @@ class ShowCompactResponse: if fid == 1: if ftype == TType.LIST: self.compacts = [] - (_etype400, _size397) = iprot.readListBegin() - for _i401 in xrange(_size397): - _elem402 = ShowCompactResponseElement() - _elem402.read(iprot) - self.compacts.append(_elem402) + (_etype430, _size427) = iprot.readListBegin() + for _i431 in xrange(_size427): + _elem432 = ShowCompactResponseElement() + _elem432.read(iprot) + self.compacts.append(_elem432) iprot.readListEnd() else: iprot.skip(ftype) @@ -7727,8 +8127,8 @@ class ShowCompactResponse: if self.compacts is not None: oprot.writeFieldBegin('compacts', TType.LIST, 1) oprot.writeListBegin(TType.STRUCT, len(self.compacts)) - for iter403 in self.compacts: - iter403.write(oprot) + for iter433 in self.compacts: + iter433.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop()
Modified: hive/branches/tez/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original) +++ hive/branches/tez/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Fri Sep 5 19:15:44 2014 @@ -687,6 +687,90 @@ class Partition ::Thrift::Struct.generate_accessors self end +class PartitionWithoutSD + include ::Thrift::Struct, ::Thrift::Struct_Union + VALUES = 1 + CREATETIME = 2 + LASTACCESSTIME = 3 + RELATIVEPATH = 4 + PARAMETERS = 5 + PRIVILEGES = 6 + + FIELDS = { + VALUES => {:type => ::Thrift::Types::LIST, :name => 'values', :element => {:type => ::Thrift::Types::STRING}}, + CREATETIME => {:type => ::Thrift::Types::I32, :name => 'createTime'}, + LASTACCESSTIME => {:type => ::Thrift::Types::I32, :name => 'lastAccessTime'}, + RELATIVEPATH => {:type => ::Thrift::Types::STRING, :name => 'relativePath'}, + PARAMETERS => {:type => ::Thrift::Types::MAP, :name => 'parameters', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, + PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class PartitionSpecWithSharedSD + include ::Thrift::Struct, ::Thrift::Struct_Union + PARTITIONS = 1 + SD = 2 + + FIELDS = { + PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::PartitionWithoutSD}}, + SD => {:type => ::Thrift::Types::STRUCT, :name => 'sd', :class => ::StorageDescriptor} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class PartitionListComposingSpec + include ::Thrift::Struct, ::Thrift::Struct_Union + PARTITIONS = 1 + + FIELDS = { + PARTITIONS => {:type => ::Thrift::Types::LIST, :name => 'partitions', :element => {:type => ::Thrift::Types::STRUCT, :class => ::Partition}} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class PartitionSpec + include ::Thrift::Struct, ::Thrift::Struct_Union + DBNAME = 1 + TABLENAME = 2 + ROOTPATH = 3 + SHAREDSDPARTITIONSPEC = 4 + PARTITIONLIST = 5 + + FIELDS = { + DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbName'}, + TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'}, + ROOTPATH => {:type => ::Thrift::Types::STRING, :name => 'rootPath'}, + SHAREDSDPARTITIONSPEC => {:type => ::Thrift::Types::STRUCT, :name => 'sharedSDPartitionSpec', :class => ::PartitionSpecWithSharedSD, :optional => true}, + PARTITIONLIST => {:type => ::Thrift::Types::STRUCT, :name => 'partitionList', :class => ::PartitionListComposingSpec, :optional => true} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + class Index include ::Thrift::Struct, ::Thrift::Struct_Union INDEXNAME = 1 Modified: hive/branches/tez/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb (original) +++ hive/branches/tez/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb Fri Sep 5 19:15:44 2014 @@ -485,6 +485,24 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'add_partitions failed: unknown result') end + def add_partitions_pspec(new_parts) + send_add_partitions_pspec(new_parts) + return recv_add_partitions_pspec() + end + + def send_add_partitions_pspec(new_parts) + send_message('add_partitions_pspec', Add_partitions_pspec_args, :new_parts => new_parts) + end + + def recv_add_partitions_pspec() + result = receive_message(Add_partitions_pspec_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise result.o3 unless result.o3.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'add_partitions_pspec failed: unknown result') + end + def append_partition(db_name, tbl_name, part_vals) send_append_partition(db_name, tbl_name, part_vals) return recv_append_partition() @@ -764,6 +782,23 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_partitions_with_auth failed: unknown result') end + def get_partitions_pspec(db_name, tbl_name, max_parts) + send_get_partitions_pspec(db_name, tbl_name, max_parts) + return recv_get_partitions_pspec() + end + + def send_get_partitions_pspec(db_name, tbl_name, max_parts) + send_message('get_partitions_pspec', Get_partitions_pspec_args, :db_name => db_name, :tbl_name => tbl_name, :max_parts => max_parts) + end + + def recv_get_partitions_pspec() + result = receive_message(Get_partitions_pspec_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_partitions_pspec failed: unknown result') + end + def get_partition_names(db_name, tbl_name, max_parts) send_get_partition_names(db_name, tbl_name, max_parts) return recv_get_partition_names() @@ -848,6 +883,23 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_partitions_by_filter failed: unknown result') end + def get_part_specs_by_filter(db_name, tbl_name, filter, max_parts) + send_get_part_specs_by_filter(db_name, tbl_name, filter, max_parts) + return recv_get_part_specs_by_filter() + end + + def send_get_part_specs_by_filter(db_name, tbl_name, filter, max_parts) + send_message('get_part_specs_by_filter', Get_part_specs_by_filter_args, :db_name => db_name, :tbl_name => tbl_name, :filter => filter, :max_parts => max_parts) + end + + def recv_get_part_specs_by_filter() + result = receive_message(Get_part_specs_by_filter_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_part_specs_by_filter failed: unknown result') + end + def get_partitions_by_expr(req) send_get_partitions_by_expr(req) return recv_get_partitions_by_expr() @@ -2292,6 +2344,21 @@ module ThriftHiveMetastore write_result(result, oprot, 'add_partitions', seqid) end + def process_add_partitions_pspec(seqid, iprot, oprot) + args = read_args(iprot, Add_partitions_pspec_args) + result = Add_partitions_pspec_result.new() + begin + result.success = @handler.add_partitions_pspec(args.new_parts) + rescue ::InvalidObjectException => o1 + result.o1 = o1 + rescue ::AlreadyExistsException => o2 + result.o2 = o2 + rescue ::MetaException => o3 + result.o3 = o3 + end + write_result(result, oprot, 'add_partitions_pspec', seqid) + end + def process_append_partition(seqid, iprot, oprot) args = read_args(iprot, Append_partition_args) result = Append_partition_result.new() @@ -2514,6 +2581,19 @@ module ThriftHiveMetastore write_result(result, oprot, 'get_partitions_with_auth', seqid) end + def process_get_partitions_pspec(seqid, iprot, oprot) + args = read_args(iprot, Get_partitions_pspec_args) + result = Get_partitions_pspec_result.new() + begin + result.success = @handler.get_partitions_pspec(args.db_name, args.tbl_name, args.max_parts) + rescue ::NoSuchObjectException => o1 + result.o1 = o1 + rescue ::MetaException => o2 + result.o2 = o2 + end + write_result(result, oprot, 'get_partitions_pspec', seqid) + end + def process_get_partition_names(seqid, iprot, oprot) args = read_args(iprot, Get_partition_names_args) result = Get_partition_names_result.new() @@ -2577,6 +2657,19 @@ module ThriftHiveMetastore write_result(result, oprot, 'get_partitions_by_filter', seqid) end + def process_get_part_specs_by_filter(seqid, iprot, oprot) + args = read_args(iprot, Get_part_specs_by_filter_args) + result = Get_part_specs_by_filter_result.new() + begin + result.success = @handler.get_part_specs_by_filter(args.db_name, args.tbl_name, args.filter, args.max_parts) + rescue ::MetaException => o1 + result.o1 = o1 + rescue ::NoSuchObjectException => o2 + result.o2 = o2 + end + write_result(result, oprot, 'get_part_specs_by_filter', seqid) + end + def process_get_partitions_by_expr(seqid, iprot, oprot) args = read_args(iprot, Get_partitions_by_expr_args) result = Get_partitions_by_expr_result.new() @@ -4422,6 +4515,44 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Add_partitions_pspec_args + include ::Thrift::Struct, ::Thrift::Struct_Union + NEW_PARTS = 1 + + FIELDS = { + NEW_PARTS => {:type => ::Thrift::Types::LIST, :name => 'new_parts', :element => {:type => ::Thrift::Types::STRUCT, :class => ::PartitionSpec}} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Add_partitions_pspec_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + O2 = 2 + O3 = 3 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success'}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::InvalidObjectException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::AlreadyExistsException}, + O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => ::MetaException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Append_partition_args include ::Thrift::Struct, ::Thrift::Struct_Union DB_NAME = 1 @@ -5096,6 +5227,46 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Get_partitions_pspec_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DB_NAME = 1 + TBL_NAME = 2 + MAX_PARTS = 3 + + FIELDS = { + DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'}, + TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}, + MAX_PARTS => {:type => ::Thrift::Types::I32, :name => 'max_parts', :default => -1} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Get_partitions_pspec_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + O2 = 2 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => ::PartitionSpec}}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::NoSuchObjectException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::MetaException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Get_partition_names_args include ::Thrift::Struct, ::Thrift::Struct_Union DB_NAME = 1 @@ -5306,6 +5477,48 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Get_part_specs_by_filter_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DB_NAME = 1 + TBL_NAME = 2 + FILTER = 3 + MAX_PARTS = 4 + + FIELDS = { + DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'}, + TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}, + FILTER => {:type => ::Thrift::Types::STRING, :name => 'filter'}, + MAX_PARTS => {:type => ::Thrift::Types::I32, :name => 'max_parts', :default => -1} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Get_part_specs_by_filter_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + O2 = 2 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::LIST, :name => 'success', :element => {:type => ::Thrift::Types::STRUCT, :class => ::PartitionSpec}}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::NoSuchObjectException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Get_partitions_by_expr_args include ::Thrift::Struct, ::Thrift::Struct_Union REQ = 1 Modified: hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original) +++ hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Fri Sep 5 19:15:44 2014 @@ -48,6 +48,9 @@ import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; import java.util.regex.Pattern; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableListMultimap; +import com.google.common.collect.Multimaps; import org.apache.commons.cli.OptionBuilder; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -112,6 +115,10 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; +import org.apache.hadoop.hive.metastore.api.PartitionListComposingSpec; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; +import org.apache.hadoop.hive.metastore.api.PartitionSpecWithSharedSD; +import org.apache.hadoop.hive.metastore.api.PartitionWithoutSD; import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest; import org.apache.hadoop.hive.metastore.api.PartitionsByExprResult; import org.apache.hadoop.hive.metastore.api.PartitionsStatsRequest; @@ -129,6 +136,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.ShowLocksRequest; import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.SkewedInfo; +import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TableStatsRequest; import org.apache.hadoop.hive.metastore.api.TableStatsResult; @@ -171,6 +179,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.model.MRoleMap; import org.apache.hadoop.hive.metastore.model.MTableColumnPrivilege; import org.apache.hadoop.hive.metastore.model.MTablePrivilege; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.txn.TxnHandler; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; @@ -1852,6 +1861,52 @@ public class HiveMetaStore extends Thrif } } + private static class PartValEqWrapperLite { + List<String> values; + String location; + + public PartValEqWrapperLite(Partition partition) { + this.values = partition.isSetValues()? partition.getValues() : null; + this.location = partition.getSd().getLocation(); + } + + @Override + public int hashCode() { + return values == null ? 0 : values.hashCode(); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || !(obj instanceof PartValEqWrapperLite)) { + return false; + } + + List<String> lhsValues = this.values; + List<String> rhsValues = ((PartValEqWrapperLite)obj).values; + + if (lhsValues == null || rhsValues == null) + return lhsValues == rhsValues; + + if (lhsValues.size() != rhsValues.size()) + return false; + + for (int i=0; i<lhsValues.size(); ++i) { + String lhsValue = lhsValues.get(i); + String rhsValue = rhsValues.get(i); + + if ((lhsValue == null && rhsValue != null) + || (lhsValue != null && !lhsValue.equals(rhsValue))) { + return false; + } + } + + return true; + } + } + private List<Partition> add_partitions_core( RawStore ms, String dbName, String tblName, List<Partition> parts, boolean ifNotExists) throws MetaException, InvalidObjectException, AlreadyExistsException, TException { @@ -1979,6 +2034,85 @@ public class HiveMetaStore extends Thrif return ret; } + @Override + public int add_partitions_pspec(final List<PartitionSpec> partSpecs) + throws TException { + logInfo("add_partitions_pspec"); + + if (partSpecs.isEmpty()) { + return 0; + } + + String dbName = partSpecs.get(0).getDbName(); + String tableName = partSpecs.get(0).getTableName(); + + return add_partitions_pspec_core(getMS(), dbName, tableName, partSpecs, false); + } + + private int add_partitions_pspec_core( + RawStore ms, String dbName, String tblName, List<PartitionSpec> partSpecs, boolean ifNotExists) + throws TException { + boolean success = false; + // Ensures that the list doesn't have dups, and keeps track of directories we have created. + Map<PartValEqWrapperLite, Boolean> addedPartitions = new HashMap<PartValEqWrapperLite, Boolean>(); + PartitionSpecProxy partitionSpecProxy = PartitionSpecProxy.Factory.get(partSpecs); + PartitionSpecProxy.PartitionIterator partitionIterator = partitionSpecProxy.getPartitionIterator(); + Table tbl = null; + try { + ms.openTransaction(); + tbl = ms.getTable(dbName, tblName); + if (tbl == null) { + throw new InvalidObjectException("Unable to add partitions because " + + "database or table " + dbName + "." + tblName + " does not exist"); + } + + firePreEvent(new PreAddPartitionEvent(tbl, partitionSpecProxy, this)); + + int nPartitions = 0; + while(partitionIterator.hasNext()) { + + Partition part = partitionIterator.getCurrent(); + + if (!part.getTableName().equals(tblName) || !part.getDbName().equals(dbName)) { + throw new MetaException("Partition does not belong to target table " + + dbName + "." + tblName + ": " + part); + } + boolean shouldAdd = startAddPartition(ms, part, ifNotExists); + if (!shouldAdd) { + LOG.info("Not adding partition " + part + " as it already exists"); + continue; + } + boolean madeDir = createLocationForAddedPartition(tbl, part); + if (addedPartitions.put(new PartValEqWrapperLite(part), madeDir) != null) { + // Technically, for ifNotExists case, we could insert one and discard the other + // because the first one now "exists", but it seems better to report the problem + // upstream as such a command doesn't make sense. + throw new MetaException("Duplicate partitions in the list: " + part); + } + initializeAddedPartition(tbl, partitionIterator, madeDir); + + ++nPartitions; + partitionIterator.next(); + } + + success = ms.addPartitions(dbName, tblName, partitionSpecProxy, ifNotExists) + && ms.commitTransaction(); + + return nPartitions; + } finally { + if (!success) { + ms.rollbackTransaction(); + for (Entry<PartValEqWrapperLite, Boolean> e : addedPartitions.entrySet()) { + if (e.getValue()) { + wh.deleteDir(new Path(e.getKey().location), true); + // we just created this directory - it's not a case of pre-creation, so we nuke + } + } + } + fireMetaStoreAddPartitionEvent(tbl, partitionSpecProxy, null, true); + } + } + private boolean startAddPartition( RawStore ms, Partition part, boolean ifNotExists) throws MetaException, TException { MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), @@ -2039,9 +2173,14 @@ public class HiveMetaStore extends Thrif private void initializeAddedPartition( final Table tbl, final Partition part, boolean madeDir) throws MetaException { + initializeAddedPartition(tbl, new PartitionSpecProxy.SimplePartitionWrapperIterator(part), madeDir); + } + + private void initializeAddedPartition( + final Table tbl, final PartitionSpecProxy.PartitionIterator part, boolean madeDir) throws MetaException { if (HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVESTATSAUTOGATHER) && !MetaStoreUtils.isView(tbl)) { - MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir); + MetaStoreUtils.updatePartitionStatsFast(part, wh, madeDir, false); } // set create time @@ -2122,6 +2261,20 @@ public class HiveMetaStore extends Thrif } } + private void fireMetaStoreAddPartitionEvent(final Table tbl, + final PartitionSpecProxy partitionSpec, final EnvironmentContext envContext, boolean success) + throws MetaException { + if (tbl != null && partitionSpec != null) { + AddPartitionEvent addPartitionEvent = + new AddPartitionEvent(tbl, partitionSpec, success, this); + addPartitionEvent.setEnvironmentContext(envContext); + + for (MetaStoreEventListener listener : listeners) { + listener.onAddPartition(addPartitionEvent); + } + } + } + @Override public Partition add_partition(final Partition part) throws InvalidObjectException, AlreadyExistsException, MetaException { @@ -2582,6 +2735,161 @@ public class HiveMetaStore extends Thrif } @Override + public List<PartitionSpec> get_partitions_pspec(final String db_name, final String tbl_name, final int max_parts) + throws NoSuchObjectException, MetaException { + + String dbName = db_name.toLowerCase(); + String tableName = tbl_name.toLowerCase(); + + startTableFunction("get_partitions_pspec", dbName, tableName); + + List<PartitionSpec> partitionSpecs = null; + try { + Table table = get_table(dbName, tableName); + List<Partition> partitions = get_partitions(dbName, tableName, (short) max_parts); + + if (is_partition_spec_grouping_enabled(table)) { + partitionSpecs = get_partitionspecs_grouped_by_storage_descriptor(table, partitions); + } + else { + PartitionSpec pSpec = new PartitionSpec(); + pSpec.setPartitionList(new PartitionListComposingSpec(partitions)); + pSpec.setDbName(dbName); + pSpec.setTableName(tableName); + pSpec.setRootPath(table.getSd().getLocation()); + partitionSpecs = Arrays.asList(pSpec); + } + + return partitionSpecs; + } + finally { + endFunction("get_partitions_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tbl_name); + } + } + + private static class StorageDescriptorKey { + + private StorageDescriptor sd; + + StorageDescriptorKey(StorageDescriptor sd) { this.sd = sd; } + + StorageDescriptor getSd() { + return sd; + } + + private String hashCodeKey() { + return sd.getInputFormat() + "\t" + + sd.getOutputFormat() + "\t" + + sd.getSerdeInfo().getSerializationLib() + "\t" + + sd.getCols(); + } + + @Override + public int hashCode() { + return hashCodeKey().hashCode(); + } + + @Override + public boolean equals(Object rhs) { + if (rhs == this) + return true; + + if (!(rhs instanceof StorageDescriptorKey)) + return false; + + return (hashCodeKey().equals(((StorageDescriptorKey) rhs).hashCodeKey())); + } + } + + private List<PartitionSpec> get_partitionspecs_grouped_by_storage_descriptor(Table table, List<Partition> partitions) + throws NoSuchObjectException, MetaException { + + assert is_partition_spec_grouping_enabled(table); + + final String tablePath = table.getSd().getLocation(); + + ImmutableListMultimap<Boolean, Partition> partitionsWithinTableDirectory + = Multimaps.index(partitions, new com.google.common.base.Function<Partition, Boolean>() { + + @Override + public Boolean apply(Partition input) { + return input.getSd().getLocation().startsWith(tablePath); + } + }); + + List<PartitionSpec> partSpecs = new ArrayList<PartitionSpec>(); + + // Classify partitions within the table directory into groups, + // based on shared SD properties. + + Map<StorageDescriptorKey, List<PartitionWithoutSD>> sdToPartList + = new HashMap<StorageDescriptorKey, List<PartitionWithoutSD>>(); + + if (partitionsWithinTableDirectory.containsKey(true)) { + + ImmutableList<Partition> partsWithinTableDir = partitionsWithinTableDirectory.get(true); + for (Partition partition : partsWithinTableDir) { + + PartitionWithoutSD partitionWithoutSD + = new PartitionWithoutSD( partition.getValues(), + partition.getCreateTime(), + partition.getLastAccessTime(), + partition.getSd().getLocation().substring(tablePath.length()), partition.getParameters()); + + StorageDescriptorKey sdKey = new StorageDescriptorKey(partition.getSd()); + if (!sdToPartList.containsKey(sdKey)) { + sdToPartList.put(sdKey, new ArrayList<PartitionWithoutSD>()); + } + + sdToPartList.get(sdKey).add(partitionWithoutSD); + + } // for (partitionsWithinTableDirectory); + + for (Map.Entry<StorageDescriptorKey, List<PartitionWithoutSD>> entry : sdToPartList.entrySet()) { + partSpecs.add(getSharedSDPartSpec(table, entry.getKey(), entry.getValue())); + } + + } // Done grouping partitions within table-dir. + + // Lump all partitions outside the tablePath into one PartSpec. + if (partitionsWithinTableDirectory.containsKey(false)) { + List<Partition> partitionsOutsideTableDir = partitionsWithinTableDirectory.get(false); + if (!partitionsOutsideTableDir.isEmpty()) { + PartitionSpec partListSpec = new PartitionSpec(); + partListSpec.setDbName(table.getDbName()); + partListSpec.setTableName(table.getTableName()); + partListSpec.setPartitionList(new PartitionListComposingSpec(partitionsOutsideTableDir)); + partSpecs.add(partListSpec); + } + + } + return partSpecs; + } + + private PartitionSpec getSharedSDPartSpec(Table table, StorageDescriptorKey sdKey, List<PartitionWithoutSD> partitions) { + + StorageDescriptor sd = new StorageDescriptor(sdKey.getSd()); + sd.setLocation(table.getSd().getLocation()); // Use table-dir as root-dir. + PartitionSpecWithSharedSD sharedSDPartSpec = + new PartitionSpecWithSharedSD(partitions, sd); + + PartitionSpec ret = new PartitionSpec(); + ret.setRootPath(sd.getLocation()); + ret.setSharedSDPartitionSpec(sharedSDPartSpec); + ret.setDbName(table.getDbName()); + ret.setTableName(table.getTableName()); + + return ret; + } + + private static boolean is_partition_spec_grouping_enabled(Table table) { + + Map<String, String> parameters = table.getParameters(); + return parameters.containsKey("hive.hcatalog.partition.spec.grouping.enabled") + && parameters.get("hive.hcatalog.partition.spec.grouping.enabled").equalsIgnoreCase("true"); + } + + @Override public List<String> get_partition_names(final String db_name, final String tbl_name, final short max_parts) throws MetaException { startTableFunction("get_partition_names", db_name, tbl_name); @@ -3790,6 +4098,37 @@ public class HiveMetaStore extends Thrif } @Override + public List<PartitionSpec> get_part_specs_by_filter(final String dbName, + final String tblName, final String filter, final int maxParts) + throws MetaException, NoSuchObjectException, TException { + + startTableFunction("get_partitions_by_filter_pspec", dbName, tblName); + + List<PartitionSpec> partitionSpecs = null; + try { + Table table = get_table(dbName, tblName); + List<Partition> partitions = get_partitions_by_filter(dbName, tblName, filter, (short) maxParts); + + if (is_partition_spec_grouping_enabled(table)) { + partitionSpecs = get_partitionspecs_grouped_by_storage_descriptor(table, partitions); + } + else { + PartitionSpec pSpec = new PartitionSpec(); + pSpec.setPartitionList(new PartitionListComposingSpec(partitions)); + pSpec.setRootPath(table.getSd().getLocation()); + pSpec.setDbName(dbName); + pSpec.setTableName(tblName); + partitionSpecs = Arrays.asList(pSpec); + } + + return partitionSpecs; + } + finally { + endFunction("get_partitions_by_filter_pspec", partitionSpecs != null && !partitionSpecs.isEmpty(), null, tblName); + } + } + + @Override public PartitionsByExprResult get_partitions_by_expr( PartitionsByExprRequest req) throws TException { String dbName = req.getDbName(), tblName = req.getTblName(); Modified: hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original) +++ hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Fri Sep 5 19:15:44 2014 @@ -98,6 +98,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.PartitionEventType; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.PartitionsByExprRequest; import org.apache.hadoop.hive.metastore.api.PartitionsByExprResult; import org.apache.hadoop.hive.metastore.api.PartitionsStatsRequest; @@ -121,6 +122,8 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.api.UnlockRequest; +import org.apache.hadoop.hive.metastore.partition.spec.CompositePartitionSpecProxy; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.metastore.txn.TxnHandler; import org.apache.hadoop.hive.shims.HadoopShims; import org.apache.hadoop.hive.shims.ShimLoader; @@ -509,6 +512,11 @@ public class HiveMetaStoreClient impleme return needResults ? result.getPartitions() : null; } + @Override + public int add_partitions_pspec(PartitionSpecProxy partitionSpec) throws TException { + return client.add_partitions_pspec(partitionSpec.toPartitionSpec()); + } + /** * @param table_name * @param db_name @@ -911,6 +919,11 @@ public class HiveMetaStoreClient impleme } @Override + public PartitionSpecProxy listPartitionSpecs(String dbName, String tableName, int maxParts) throws TException { + return PartitionSpecProxy.Factory.get(client.get_partitions_pspec(dbName, tableName, maxParts)); + } + + @Override public List<Partition> listPartitions(String db_name, String tbl_name, List<String> part_vals, short max_parts) throws NoSuchObjectException, MetaException, TException { @@ -958,6 +971,14 @@ public class HiveMetaStoreClient impleme } @Override + public PartitionSpecProxy listPartitionSpecsByFilter(String db_name, String tbl_name, + String filter, int max_parts) throws MetaException, + NoSuchObjectException, TException { + return PartitionSpecProxy.Factory.get( + client.get_part_specs_by_filter(db_name, tbl_name, filter, max_parts)); + } + + @Override public boolean listPartitionsByExpr(String db_name, String tbl_name, byte[] expr, String default_partition_name, short max_parts, List<Partition> result) throws TException { Modified: hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original) +++ hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Fri Sep 5 19:15:44 2014 @@ -28,10 +28,12 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.NoSuchLockException; import org.apache.hadoop.hive.metastore.api.NoSuchTxnException; import org.apache.hadoop.hive.metastore.api.OpenTxnsResponse; +import org.apache.hadoop.hive.metastore.api.PartitionSpec; import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; import org.apache.hadoop.hive.metastore.api.ShowLocksResponse; import org.apache.hadoop.hive.metastore.api.TxnAbortedException; import org.apache.hadoop.hive.metastore.api.TxnOpenException; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.thrift.TException; import java.util.List; @@ -368,6 +370,9 @@ public interface IMetaStoreClient { int add_partitions(List<Partition> partitions) throws InvalidObjectException, AlreadyExistsException, MetaException, TException; + int add_partitions_pspec(PartitionSpecProxy partitionSpec) + throws InvalidObjectException, AlreadyExistsException, MetaException, TException; + /** * Add partitions to the table. * @@ -448,6 +453,8 @@ public interface IMetaStoreClient { List<Partition> listPartitions(String db_name, String tbl_name, short max_parts) throws NoSuchObjectException, MetaException, TException; + public PartitionSpecProxy listPartitionSpecs(String dbName, String tableName, int maxParts) + throws TException; List<Partition> listPartitions(String db_name, String tbl_name, List<String> part_vals, short max_parts) throws NoSuchObjectException, MetaException, TException; @@ -476,6 +483,9 @@ public interface IMetaStoreClient { String filter, short max_parts) throws MetaException, NoSuchObjectException, TException; + PartitionSpecProxy listPartitionSpecsByFilter(String db_name, String tbl_name, + String filter, int max_parts) throws MetaException, + NoSuchObjectException, TException; /** * Get list of partitions matching specified serialized expression Modified: hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java (original) +++ hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreUtils.java Fri Sep 5 19:15:44 2014 @@ -59,6 +59,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.Deserializer; import org.apache.hadoop.hive.serde2.SerDeException; @@ -288,6 +289,22 @@ public class MetaStoreUtils { */ public static boolean updatePartitionStatsFast(Partition part, Warehouse wh, boolean madeDir, boolean forceRecompute) throws MetaException { + return updatePartitionStatsFast(new PartitionSpecProxy.SimplePartitionWrapperIterator(part), + wh, madeDir, forceRecompute); + } + + /** + * Updates the numFiles and totalSize parameters for the passed Partition by querying + * the warehouse if the passed Partition does not already have values for these parameters. + * @param part + * @param wh + * @param madeDir if true, the directory was just created and can be assumed to be empty + * @param forceRecompute Recompute stats even if the passed Partition already has + * these parameters set + * @return true if the stats were updated, false otherwise + */ + public static boolean updatePartitionStatsFast(PartitionSpecProxy.PartitionIterator part, Warehouse wh, + boolean madeDir, boolean forceRecompute) throws MetaException { Map<String,String> params = part.getParameters(); boolean updated = false; if (forceRecompute || @@ -297,10 +314,10 @@ public class MetaStoreUtils { params = new HashMap<String,String>(); } if (!madeDir) { - // The partitition location already existed and may contain data. Lets try to + // The partition location already existed and may contain data. Lets try to // populate those statistics that don't require a full scan of the data. LOG.warn("Updating partition stats fast for: " + part.getTableName()); - FileStatus[] fileStatus = wh.getFileStatusesForSD(part.getSd()); + FileStatus[] fileStatus = wh.getFileStatusesForLocation(part.getLocation()); populateQuickStats(fileStatus, params); LOG.warn("Updated size to " + params.get(StatsSetupConst.TOTAL_SIZE)); if(!params.containsKey(StatsSetupConst.STATS_GENERATED_VIA_STATS_TASK)) { Modified: hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java URL: http://svn.apache.org/viewvc/hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1622763&r1=1622762&r2=1622763&view=diff ============================================================================== --- hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original) +++ hive/branches/tez/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Fri Sep 5 19:15:44 2014 @@ -39,6 +39,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.Lock; import java.util.concurrent.locks.ReentrantLock; +import java.util.regex.Pattern; import javax.jdo.JDODataStoreException; import javax.jdo.JDOHelper; @@ -63,6 +64,7 @@ import org.apache.hadoop.hive.common.cla import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -133,6 +135,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.metastore.parser.FilterParser; import org.apache.hadoop.hive.shims.ShimLoader; +import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; import org.apache.hadoop.util.StringUtils; import org.apache.thrift.TException; import org.datanucleus.store.rdbms.exceptions.MissingTableException; @@ -183,6 +186,8 @@ public class ObjectStore implements RawS private TXN_STATUS transactionStatus = TXN_STATUS.NO_STATE; private final AtomicBoolean isSchemaVerified = new AtomicBoolean(false); + private Pattern partitionValidationPattern; + public ObjectStore() { } @@ -227,6 +232,14 @@ public class ObjectStore implements RawS initialize(propsFromConf); + String partitionValidationRegex = + hiveConf.get(HiveConf.ConfVars.METASTORE_PARTITION_NAME_WHITELIST_PATTERN.name()); + if (partitionValidationRegex != null && partitionValidationRegex.equals("")) { + partitionValidationPattern = Pattern.compile(partitionValidationRegex); + } else { + partitionValidationPattern = null; + } + if (!isInitialized) { throw new RuntimeException( "Unable to create persistence manager. Check dss.log for details"); @@ -1295,6 +1308,76 @@ public class ObjectStore implements RawS return success; } + private boolean isValidPartition( + Partition part, boolean ifNotExists) throws MetaException { + MetaStoreUtils.validatePartitionNameCharacters(part.getValues(), + partitionValidationPattern); + boolean doesExist = doesPartitionExist( + part.getDbName(), part.getTableName(), part.getValues()); + if (doesExist && !ifNotExists) { + throw new MetaException("Partition already exists: " + part); + } + return !doesExist; + } + + + @Override + public boolean addPartitions(String dbName, String tblName, + PartitionSpecProxy partitionSpec, boolean ifNotExists) + throws InvalidObjectException, MetaException { + boolean success = false; + openTransaction(); + try { + List<MTablePrivilege> tabGrants = null; + List<MTableColumnPrivilege> tabColumnGrants = null; + MTable table = this.getMTable(dbName, tblName); + if ("TRUE".equalsIgnoreCase(table.getParameters().get("PARTITION_LEVEL_PRIVILEGE"))) { + tabGrants = this.listAllTableGrants(dbName, tblName); + tabColumnGrants = this.listTableAllColumnGrants(dbName, tblName); + } + + if (!partitionSpec.getTableName().equals(tblName) || !partitionSpec.getDbName().equals(dbName)) { + throw new MetaException("Partition does not belong to target table " + + dbName + "." + tblName + ": " + partitionSpec); + } + + PartitionSpecProxy.PartitionIterator iterator = partitionSpec.getPartitionIterator(); + + int now = (int)(System.currentTimeMillis()/1000); + + while (iterator.hasNext()) { + Partition part = iterator.next(); + + if (isValidPartition(part, ifNotExists)) { + MPartition mpart = convertToMPart(part, true); + pm.makePersistent(mpart); + if (tabGrants != null) { + for (MTablePrivilege tab : tabGrants) { + pm.makePersistent(new MPartitionPrivilege(tab.getPrincipalName(), + tab.getPrincipalType(), mpart, tab.getPrivilege(), now, + tab.getGrantor(), tab.getGrantorType(), tab.getGrantOption())); + } + } + + if (tabColumnGrants != null) { + for (MTableColumnPrivilege col : tabColumnGrants) { + pm.makePersistent(new MPartitionColumnPrivilege(col.getPrincipalName(), + col.getPrincipalType(), mpart, col.getColumnName(), col.getPrivilege(), + now, col.getGrantor(), col.getGrantorType(), col.getGrantOption())); + } + } + } + } + + success = commitTransaction(); + } finally { + if (!success) { + rollbackTransaction(); + } + } + return success; + } + @Override public boolean addPartition(Partition part) throws InvalidObjectException, MetaException {
