Modified: hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py (original) +++ hive/trunk/metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py Mon Jun 20 09:28:55 2011 @@ -56,6 +56,17 @@ class PrincipalType: "GROUP": 3, } +class PartitionEventType: + LOAD_DONE = 1 + + _VALUES_TO_NAMES = { + 1: "LOAD_DONE", + } + + _NAMES_TO_VALUES = { + "LOAD_DONE": 1, + } + class Version: """ @@ -2314,6 +2325,130 @@ class AlreadyExistsException(Exception): def __ne__(self, other): return not (self == other) +class InvalidPartitionException(Exception): + """ + Attributes: + - message + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'message', None, None, ), # 1 + ) + + def __init__(self, message=None,): + self.message = message + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('InvalidPartitionException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class UnknownPartitionException(Exception): + """ + Attributes: + - message + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'message', None, None, ), # 1 + ) + + def __init__(self, message=None,): + self.message = message + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.message = iprot.readString(); + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('UnknownPartitionException') + if self.message != None: + oprot.writeFieldBegin('message', TType.STRING, 1) + oprot.writeString(self.message) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + def validate(self): + return + + + def __str__(self): + return repr(self) + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + class InvalidObjectException(Exception): """ Attributes:
Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb (original) +++ hive/trunk/metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb Mon Jun 20 09:28:55 2011 @@ -25,6 +25,12 @@ module PrincipalType VALID_VALUES = Set.new([USER, ROLE, GROUP]).freeze end +module PartitionEventType + LOAD_DONE = 1 + VALUE_MAP = {1 => "LOAD_DONE"} + VALID_VALUES = Set.new([LOAD_DONE]).freeze +end + class Version include ::Thrift::Struct, ::Thrift::Struct_Union VERSION = 1 @@ -522,6 +528,48 @@ class AlreadyExistsException < ::Thrift: ::Thrift::Struct.generate_accessors self end +class InvalidPartitionException < ::Thrift::Exception + include ::Thrift::Struct, ::Thrift::Struct_Union + def initialize(message=nil) + super() + self.message = message + end + + MESSAGE = 1 + + FIELDS = { + MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + +class UnknownPartitionException < ::Thrift::Exception + include ::Thrift::Struct, ::Thrift::Struct_Union + def initialize(message=nil) + super() + self.message = message + end + + MESSAGE = 1 + + FIELDS = { + MESSAGE => {:type => ::Thrift::Types::STRING, :name => 'message'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self +end + class InvalidObjectException < ::Thrift::Exception include ::Thrift::Struct, ::Thrift::Struct_Union def initialize(message=nil) Modified: hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb (original) +++ hive/trunk/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb Mon Jun 20 09:28:55 2011 @@ -668,6 +668,47 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'partition_name_to_spec failed: unknown result') end + def markPartitionForEvent(db_name, tbl_name, part_vals, eventType) + send_markPartitionForEvent(db_name, tbl_name, part_vals, eventType) + recv_markPartitionForEvent() + end + + def send_markPartitionForEvent(db_name, tbl_name, part_vals, eventType) + send_message('markPartitionForEvent', MarkPartitionForEvent_args, :db_name => db_name, :tbl_name => tbl_name, :part_vals => part_vals, :eventType => eventType) + end + + def recv_markPartitionForEvent() + result = receive_message(MarkPartitionForEvent_result) + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise result.o3 unless result.o3.nil? + raise result.o4 unless result.o4.nil? + raise result.o5 unless result.o5.nil? + raise result.o6 unless result.o6.nil? + return + end + + def isPartitionMarkedForEvent(db_name, tbl_name, part_vals, eventType) + send_isPartitionMarkedForEvent(db_name, tbl_name, part_vals, eventType) + return recv_isPartitionMarkedForEvent() + end + + def send_isPartitionMarkedForEvent(db_name, tbl_name, part_vals, eventType) + send_message('isPartitionMarkedForEvent', IsPartitionMarkedForEvent_args, :db_name => db_name, :tbl_name => tbl_name, :part_vals => part_vals, :eventType => eventType) + end + + def recv_isPartitionMarkedForEvent() + result = receive_message(IsPartitionMarkedForEvent_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise result.o3 unless result.o3.nil? + raise result.o4 unless result.o4.nil? + raise result.o5 unless result.o5.nil? + raise result.o6 unless result.o6.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'isPartitionMarkedForEvent failed: unknown result') + end + def add_index(new_index, index_table) send_add_index(new_index, index_table) return recv_add_index() @@ -1488,6 +1529,48 @@ module ThriftHiveMetastore write_result(result, oprot, 'partition_name_to_spec', seqid) end + def process_markPartitionForEvent(seqid, iprot, oprot) + args = read_args(iprot, MarkPartitionForEvent_args) + result = MarkPartitionForEvent_result.new() + begin + @handler.markPartitionForEvent(args.db_name, args.tbl_name, args.part_vals, args.eventType) + rescue MetaException => o1 + result.o1 = o1 + rescue NoSuchObjectException => o2 + result.o2 = o2 + rescue UnknownDBException => o3 + result.o3 = o3 + rescue UnknownTableException => o4 + result.o4 = o4 + rescue UnknownPartitionException => o5 + result.o5 = o5 + rescue InvalidPartitionException => o6 + result.o6 = o6 + end + write_result(result, oprot, 'markPartitionForEvent', seqid) + end + + def process_isPartitionMarkedForEvent(seqid, iprot, oprot) + args = read_args(iprot, IsPartitionMarkedForEvent_args) + result = IsPartitionMarkedForEvent_result.new() + begin + result.success = @handler.isPartitionMarkedForEvent(args.db_name, args.tbl_name, args.part_vals, args.eventType) + rescue MetaException => o1 + result.o1 = o1 + rescue NoSuchObjectException => o2 + result.o2 = o2 + rescue UnknownDBException => o3 + result.o3 = o3 + rescue UnknownTableException => o4 + result.o4 = o4 + rescue UnknownPartitionException => o5 + result.o5 = o5 + rescue InvalidPartitionException => o6 + result.o6 = o6 + end + write_result(result, oprot, 'isPartitionMarkedForEvent', seqid) + end + def process_add_index(seqid, iprot, oprot) args = read_args(iprot, Add_index_args) result = Add_index_result.new() @@ -3216,6 +3299,110 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class MarkPartitionForEvent_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DB_NAME = 1 + TBL_NAME = 2 + PART_VALS = 3 + EVENTTYPE = 4 + + FIELDS = { + DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'}, + TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}, + PART_VALS => {:type => ::Thrift::Types::MAP, :name => 'part_vals', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, + EVENTTYPE => {:type => ::Thrift::Types::I32, :name => 'eventType', :enum_class => PartitionEventType} + } + + def struct_fields; FIELDS; end + + def validate + unless @eventType.nil? || PartitionEventType::VALID_VALUES.include?(@eventType) + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field eventType!') + end + end + + ::Thrift::Struct.generate_accessors self + end + + class MarkPartitionForEvent_result + include ::Thrift::Struct, ::Thrift::Struct_Union + O1 = 1 + O2 = 2 + O3 = 3 + O4 = 4 + O5 = 5 + O6 = 6 + + FIELDS = { + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => MetaException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => NoSuchObjectException}, + O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => UnknownDBException}, + O4 => {:type => ::Thrift::Types::STRUCT, :name => 'o4', :class => UnknownTableException}, + O5 => {:type => ::Thrift::Types::STRUCT, :name => 'o5', :class => UnknownPartitionException}, + O6 => {:type => ::Thrift::Types::STRUCT, :name => 'o6', :class => InvalidPartitionException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class IsPartitionMarkedForEvent_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DB_NAME = 1 + TBL_NAME = 2 + PART_VALS = 3 + EVENTTYPE = 4 + + FIELDS = { + DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'}, + TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}, + PART_VALS => {:type => ::Thrift::Types::MAP, :name => 'part_vals', :key => {:type => ::Thrift::Types::STRING}, :value => {:type => ::Thrift::Types::STRING}}, + EVENTTYPE => {:type => ::Thrift::Types::I32, :name => 'eventType', :enum_class => PartitionEventType} + } + + def struct_fields; FIELDS; end + + def validate + unless @eventType.nil? || PartitionEventType::VALID_VALUES.include?(@eventType) + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field eventType!') + end + end + + ::Thrift::Struct.generate_accessors self + end + + class IsPartitionMarkedForEvent_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + O2 = 2 + O3 = 3 + O4 = 4 + O5 = 5 + O6 = 6 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::BOOL, :name => 'success'}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => MetaException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => NoSuchObjectException}, + O3 => {:type => ::Thrift::Types::STRUCT, :name => 'o3', :class => UnknownDBException}, + O4 => {:type => ::Thrift::Types::STRUCT, :name => 'o4', :class => UnknownTableException}, + O5 => {:type => ::Thrift::Types::STRUCT, :name => 'o5', :class => UnknownPartitionException}, + O6 => {:type => ::Thrift::Types::STRUCT, :name => 'o6', :class => InvalidPartitionException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Add_index_args include ::Thrift::Struct, ::Thrift::Struct_Union NEW_INDEX = 1 Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Mon Jun 20 09:28:55 2011 @@ -54,9 +54,11 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.IndexAlreadyExistsException; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; @@ -66,6 +68,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; @@ -73,6 +76,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; import org.apache.hadoop.hive.metastore.hooks.JDOConnectionURLHook; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; @@ -527,6 +531,11 @@ public class HiveMetaStore extends Thrif + "[" + join(partVals, ",") + "]" ); } + public String startPartitionFunction(String function, String db, String tbl, + Map<String,String> partName) { + return startFunction(function, " : db=" + db + " tbl=" + tbl + "partition=" + partName); + } + public void endFunction(String function) { try { Metrics.endScope(function); @@ -3173,8 +3182,97 @@ public class HiveMetaStore extends Thrif } } + @Override + public void markPartitionForEvent(final String db_name, final String tbl_name, + final Map<String,String> partName, final PartitionEventType evtType) throws + MetaException,TException, NoSuchObjectException, UnknownDBException, UnknownTableException, + InvalidPartitionException, UnknownPartitionException { + + try { + startPartitionFunction("markPartitionForEvent", db_name, tbl_name, partName); + Table tbl = null; + try{ + tbl = executeWithRetry(new Command<Table>(){ + @Override + public Table run(RawStore ms) throws Exception { + return ms.markPartitionForEvent(db_name, tbl_name, partName, evtType); + } + }); + } catch (Exception original) { + LOG.error(original); + if (original instanceof NoSuchObjectException) { + throw (NoSuchObjectException)original; + } else if(original instanceof UnknownTableException){ + throw (UnknownTableException)original; + } else if(original instanceof UnknownDBException){ + throw (UnknownDBException)original; + } else if(original instanceof UnknownPartitionException){ + throw (UnknownPartitionException)original; + } else if(original instanceof InvalidPartitionException){ + throw (InvalidPartitionException)original; + } else if(original instanceof MetaException){ + throw (MetaException)original; + } else{ + MetaException me = new MetaException(original.toString()); + me.initCause(original); + throw me; + } + } + if (null == tbl) { + throw new UnknownTableException("Table: "+tbl_name + " not found."); + } else{ + for(MetaStoreEventListener listener : listeners){ + listener.onLoadPartitionDone(new LoadPartitionDoneEvent(true, this, tbl, partName)); + } + } + } + finally{ + endFunction("markPartitionForEvent"); + } + } + + @Override + public boolean isPartitionMarkedForEvent(final String db_name, final String tbl_name, + final Map<String,String> partName, final PartitionEventType evtType) throws + MetaException, NoSuchObjectException, UnknownDBException, UnknownTableException, + TException, UnknownPartitionException, InvalidPartitionException { + + startPartitionFunction("isPartitionMarkedForEvent", db_name, tbl_name, partName); + try { + return executeWithRetry(new Command<Boolean>(){ + @Override + public Boolean run(RawStore ms) throws Exception { + return ms.isPartitionMarkedForEvent(db_name, tbl_name, partName, evtType); + } + + }); + } catch (Exception original) { + LOG.error(original); + if (original instanceof NoSuchObjectException) { + throw (NoSuchObjectException)original; + } else if(original instanceof UnknownTableException){ + throw (UnknownTableException)original; + } else if(original instanceof UnknownDBException){ + throw (UnknownDBException)original; + } else if(original instanceof UnknownPartitionException){ + throw (UnknownPartitionException)original; + } else if(original instanceof InvalidPartitionException){ + throw (InvalidPartitionException)original; + } else if(original instanceof MetaException){ + throw (MetaException)original; + } else{ + MetaException me = new MetaException(original.toString()); + me.initCause(original); + throw me; + } + } + finally{ + endFunction("isPartitionMarkedForEvent"); + } + } } + /** * Discard a current delegation token. * @param tokenStrForm the token in string form Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java Mon Jun 20 09:28:55 2011 @@ -45,9 +45,11 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; @@ -56,6 +58,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.shims.ShimLoader; import org.apache.hadoop.hive.thrift.HadoopThriftAuthBridge; @@ -1137,4 +1140,24 @@ public class HiveMetaStoreClient impleme } } + @Override + public void markPartitionForEvent(String db_name, String tbl_name, Map<String,String> partKVs, PartitionEventType eventType) + throws MetaException, TException, NoSuchObjectException, UnknownDBException, UnknownTableException, + InvalidPartitionException, UnknownPartitionException { + assert db_name != null; + assert tbl_name != null; + assert partKVs != null; + client.markPartitionForEvent(db_name, tbl_name, partKVs, eventType); + } + + @Override + public boolean isPartitionMarkedForEvent(String db_name, String tbl_name, Map<String,String> partKVs, PartitionEventType eventType) + throws MetaException, NoSuchObjectException, UnknownTableException, UnknownDBException, TException, + InvalidPartitionException, UnknownPartitionException { + assert db_name != null; + assert tbl_name != null; + assert partKVs != null; + return client.isPartitionMarkedForEvent(db_name, tbl_name, partKVs, eventType); + } + } Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java Mon Jun 20 09:28:55 2011 @@ -30,15 +30,18 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; import org.apache.hadoop.hive.metastore.api.Role; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.thrift.TException; @@ -388,6 +391,40 @@ public interface IMetaStoreClient { List<String> groupNames) throws MetaException, TException, NoSuchObjectException; /** + * @param db_name + * @param tbl_name + * @param partKVs + * @param eventType + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + * @throws UnknownTableException + * @throws UnknownDBException + * @throws UnknownPartitionException + * @throws InvalidPartitionException + */ + public void markPartitionForEvent(String db_name, String tbl_name, Map<String,String> partKVs, + PartitionEventType eventType) throws MetaException, NoSuchObjectException, TException, + UnknownTableException, UnknownDBException, UnknownPartitionException, InvalidPartitionException; + + /** + * @param db_name + * @param tbl_name + * @param partKVs + * @param eventType + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + * @throws UnknownTableException + * @throws UnknownDBException + * @throws UnknownPartitionException + * @throws InvalidPartitionException + */ + public boolean isPartitionMarkedForEvent(String db_name, String tbl_name, Map<String,String> partKVs, + PartitionEventType eventType) throws MetaException, NoSuchObjectException, TException, + UnknownTableException, UnknownDBException, UnknownPartitionException, InvalidPartitionException; + + /** * @param tbl * @throws AlreadyExistsException * @throws InvalidObjectException @@ -396,6 +433,7 @@ public interface IMetaStoreClient { * @throws TException * @see org.apache.hadoop.hive.metastore.api.ThriftHiveMetastore.Iface#create_table(org.apache.hadoop.hive.metastore.api.Table) */ + public void createTable(Table tbl) throws AlreadyExistsException, InvalidObjectException, MetaException, NoSuchObjectException, TException; Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreEventListener.java Mon Jun 20 09:28:55 2011 @@ -27,6 +27,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.events.DropDatabaseEvent; import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; +import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; /** * This abstract class needs to be extended to provide implementation of actions that needs @@ -78,6 +79,12 @@ public abstract class MetaStoreEventList */ public abstract void onDropDatabase (DropDatabaseEvent dbEvent) throws MetaException; + /** + * @param partSetDoneEvent + * @throws MetaException + */ + public abstract void onLoadPartitionDone(LoadPartitionDoneEvent partSetDoneEvent) throws MetaException; + @Override public Configuration getConf() { return this.conf; Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Mon Jun 20 09:28:55 2011 @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.metastore; +import static org.apache.commons.lang.StringUtils.join; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -58,10 +60,12 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.HiveObjectType; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Order; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; @@ -72,6 +76,8 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MDatabase; import org.apache.hadoop.hive.metastore.model.MFieldSchema; @@ -80,6 +86,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.model.MOrder; import org.apache.hadoop.hive.metastore.model.MPartition; import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; +import org.apache.hadoop.hive.metastore.model.MPartitionEvent; import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; import org.apache.hadoop.hive.metastore.model.MRole; import org.apache.hadoop.hive.metastore.model.MRoleMap; @@ -3398,4 +3405,72 @@ public class ObjectStore implements RawS return mSecurityColumnList; } + @Override + public boolean isPartitionMarkedForEvent(String dbName, String tblName, + Map<String, String> partName, PartitionEventType evtType) throws UnknownTableException, + MetaException, InvalidPartitionException, UnknownPartitionException { + + Collection<MPartitionEvent> partEvents; + boolean success = false; + LOG.debug("Begin Executing isPartitionMarkedForEvent"); + try{ + openTransaction(); + Query query = pm.newQuery(MPartitionEvent.class, "dbName == t1 && tblName == t2 && partName == t3 && eventType == t4"); + query.declareParameters("java.lang.String t1, java.lang.String t2, java.lang.String t3, int t4"); + Table tbl = getTable(dbName, tblName); // Make sure dbName and tblName are valid. + if(null == tbl) { + throw new UnknownTableException("Table: "+ tblName + " is not found."); + } + partEvents = (Collection<MPartitionEvent>) query.executeWithArray(dbName, tblName, getPartitionStr(tbl, partName), evtType.getValue()); + pm.retrieveAll(partEvents); + success = commitTransaction(); + LOG.debug("Done executing isPartitionMarkedForEvent"); + } finally{ + if (!success) { + rollbackTransaction(); + } + } + return (partEvents != null && !partEvents.isEmpty()) ? true : false; + + } + + @Override + public Table markPartitionForEvent(String dbName, String tblName, Map<String,String> partName, + PartitionEventType evtType) throws MetaException, UnknownTableException, InvalidPartitionException, UnknownPartitionException { + + LOG.debug("Begin executing markPartitionForEvent"); + boolean success = false; + Table tbl = null; + try{ + openTransaction(); + tbl = getTable(dbName, tblName); // Make sure dbName and tblName are valid. + if(null == tbl) { + throw new UnknownTableException("Table: "+ tblName + " is not found."); + } + pm.makePersistent(new MPartitionEvent(dbName,tblName,getPartitionStr(tbl, partName), evtType.getValue())); + success = commitTransaction(); + LOG.debug("Done executing markPartitionForEvent"); + } finally { + if(!success) { + rollbackTransaction(); + } + } + return tbl; + } + + private String getPartitionStr(Table tbl, Map<String,String> partName) throws InvalidPartitionException{ + if(tbl.getPartitionKeysSize() != partName.size()){ + throw new InvalidPartitionException("Number of partition columns in table: "+ tbl.getPartitionKeysSize() + + " doesn't match with number of supplied partition values: "+partName.size()); + } + final List<String> storedVals = new ArrayList<String>(tbl.getPartitionKeysSize()); + for(FieldSchema partKey : tbl.getPartitionKeys()){ + String partVal = partName.get(partKey.getName()); + if(null == partVal) { + throw new InvalidPartitionException("No value found for partition column: "+partKey.getName()); + } + storedVals.add(partVal); + } + return join(storedVals,','); + } } Modified: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Mon Jun 20 09:28:55 2011 @@ -19,14 +19,17 @@ package org.apache.hadoop.hive.metastore; import java.util.List; +import java.util.Map; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Index; import org.apache.hadoop.hive.metastore.api.InvalidObjectException; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; import org.apache.hadoop.hive.metastore.api.PrincipalType; import org.apache.hadoop.hive.metastore.api.PrivilegeBag; @@ -34,6 +37,8 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.Type; import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; import org.apache.hadoop.hive.metastore.model.MDBPrivilege; import org.apache.hadoop.hive.metastore.model.MGlobalPrivilege; import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; @@ -163,6 +168,10 @@ public interface RawStore extends Config String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException; + public abstract Table markPartitionForEvent(String dbName, String tblName, Map<String,String> partVals, PartitionEventType evtType) throws MetaException, UnknownTableException, InvalidPartitionException, UnknownPartitionException; + + public abstract boolean isPartitionMarkedForEvent(String dbName, String tblName, Map<String, String> partName, PartitionEventType evtType) throws MetaException, UnknownTableException, InvalidPartitionException, UnknownPartitionException; + public abstract boolean addRole(String rowName, String ownerName) throws InvalidObjectException, MetaException, NoSuchObjectException; Added: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java?rev=1137561&view=auto ============================================================================== --- hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java (added) +++ hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java Mon Jun 20 09:28:55 2011 @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import java.util.Map; + +import org.apache.hadoop.hive.metastore.HiveMetaStore.HMSHandler; +import org.apache.hadoop.hive.metastore.api.Table; + +public class LoadPartitionDoneEvent extends ListenerEvent { + + private final Table table; + + private final Map<String,String> partSpec; + + public LoadPartitionDoneEvent(boolean status, HMSHandler handler, Table table, + Map<String,String> partSpec) { + super(status, handler); + this.table = table; + this.partSpec = partSpec; + } + + /** + * @return the tblName + */ + public Table getTable() { + return table; + } + + /** + * @return the partition Name + */ + public Map<String,String> getPartitionName() { + return partSpec; + } + +} Propchange: hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/events/LoadPartitionDoneEvent.java ------------------------------------------------------------------------------ svn:eol-style = native Added: hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionEvent.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionEvent.java?rev=1137561&view=auto ============================================================================== --- hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionEvent.java (added) +++ hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionEvent.java Mon Jun 20 09:28:55 2011 @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.model; + + +public class MPartitionEvent { + + private String dbName; + + private String tblName; + + private String partName; + + private long eventTime; + + private int eventType; + + public MPartitionEvent(String dbName, String tblName, String partitionName, int eventType) { + super(); + this.dbName = dbName; + this.tblName = tblName; + this.partName = partitionName; + this.eventType = eventType; + this.eventTime = System.currentTimeMillis(); + } + + public MPartitionEvent() {} + + /** + * @param dbName the dbName to set + */ + public void setDbName(String dbName) { + this.dbName = dbName; + } + + /** + * @param tblName the tblName to set + */ + public void setTblName(String tblName) { + this.tblName = tblName; + } + + /** + * @param partName the partSpec to set + */ + public void setPartName(String partName) { + this.partName = partName; + } + + /** + * @param eventTime the eventTime to set + */ + public void setEventTime(long createTime) { + this.eventTime = createTime; + } + + /** + * @param eventType the EventType to set + */ + public void setEventType(int eventType) { + this.eventType = eventType; + } + + /* (non-Javadoc) + * @see java.lang.Object#toString() + */ + @Override + public String toString() { + return "MPartitionEvent [dbName=" + dbName + ", tblName=" + tblName + ", partName=" + partName + + ", eventTime=" + eventTime + ", EventType=" + eventType + "]"; + } + + +} Propchange: hive/trunk/metastore/src/model/org/apache/hadoop/hive/metastore/model/MPartitionEvent.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: hive/trunk/metastore/src/model/package.jdo URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/model/package.jdo?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/model/package.jdo (original) +++ hive/trunk/metastore/src/model/package.jdo Mon Jun 20 09:28:55 2011 @@ -665,5 +665,32 @@ </field> </class> + <class name="MPartitionEvent" table="PARTITION_EVENTS" identity-type="datastore" detachable="true"> + + <index name="PartitionEventIndex" unique="false"> + <column name="PARTITION_NAME"/> + </index> + + <datastore-identity> + <column name="PART_NAME_ID"/> + </datastore-identity> + + <field name="dbName"> + <column name="DB_NAME" length="128" jdbc-type="VARCHAR"/> + </field> + <field name="tblName"> + <column name="TBL_NAME" length="128" jdbc-type="VARCHAR"/> + </field> + <field name="partName"> + <column name="PARTITION_NAME" length="767" jdbc-type="VARCHAR"/> + </field> + <field name="eventType"> + <column name="EVENT_TYPE" jdbc-type="integer"/> + </field> + <field name="eventTime"> + <column name="EVENT_TIME" jdbc-type="BIGINT"/> + </field> + + </class> </package> </jdo> Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java (original) +++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/DummyListener.java Mon Jun 20 09:28:55 2011 @@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; /** A dummy implementation for * {@link org.apache.hadoop.hive.metastore.hadooorg.apache.hadoop.hive.metastore.MetaStoreEventListener} @@ -72,4 +73,9 @@ public class DummyListener extends MetaS public void onDropTable(DropTableEvent table) throws MetaException { notifyList.add(table); } + + @Override + public void onLoadPartitionDone(LoadPartitionDoneEvent partEvent) throws MetaException { + notifyList.add(partEvent); + } } Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartition.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartition.java?rev=1137561&view=auto ============================================================================== --- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartition.java (added) +++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartition.java Mon Jun 20 09:28:55 2011 @@ -0,0 +1,102 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import java.util.HashMap; +import java.util.Map; + +import junit.framework.TestCase; + +import org.apache.hadoop.hive.cli.CliSessionState; +import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.InvalidPartitionException; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; +import org.apache.hadoop.hive.metastore.api.UnknownDBException; +import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; +import org.apache.hadoop.hive.metastore.api.UnknownTableException; +import org.apache.hadoop.hive.ql.CommandNeedRetryException; +import org.apache.hadoop.hive.ql.Driver; +import org.apache.hadoop.hive.ql.session.SessionState; +import org.apache.thrift.TException; + +public class TestMarkPartition extends TestCase{ + + protected HiveConf hiveConf; + private Driver driver; + + @Override + protected void setUp() throws Exception { + + super.setUp(); + hiveConf = new HiveConf(this.getClass()); + hiveConf.set(HiveConf.ConfVars.PREEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.POSTEXECHOOKS.varname, ""); + hiveConf.set(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY.varname, "false"); + SessionState.start(new CliSessionState(hiveConf)); + + } + + public void testMarkingPartitionSet() throws CommandNeedRetryException, MetaException, + TException, NoSuchObjectException, UnknownDBException, UnknownTableException, + InvalidPartitionException, UnknownPartitionException { + HiveMetaStoreClient msc = new HiveMetaStoreClient(hiveConf, null); + driver = new Driver(hiveConf); + driver.run("drop database if exists tmpdb cascade"); + driver.run("create database tmpdb"); + driver.run("use tmpdb"); + driver.run("drop table if exists tmptbl"); + driver.run("create table tmptbl (a string) partitioned by (b string)"); + driver.run("alter table tmptbl add partition (b='2011')"); + Map<String,String> kvs = new HashMap<String, String>(); + kvs.put("b", "'2011'"); + msc.markPartitionForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE); + assert msc.isPartitionMarkedForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE); + + kvs.put("b", "'2012'"); + assert !msc.isPartitionMarkedForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE); + try{ + msc.markPartitionForEvent("tmpdb", "tmptbl2", kvs, PartitionEventType.LOAD_DONE); + assert false; + } catch(Exception e){ + assert e instanceof UnknownTableException; + } + try{ + msc.isPartitionMarkedForEvent("tmpdb", "tmptbl2", kvs, PartitionEventType.LOAD_DONE); + assert false; + } catch(Exception e){ + assert e instanceof UnknownTableException; + } + kvs.put("a", "'2012'"); + try{ + msc.isPartitionMarkedForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE); + assert false; + } catch(Exception e){ + assert e instanceof InvalidPartitionException; + } + } + + @Override + protected void tearDown() throws Exception { + driver.run("drop database if exists tmpdb cascade"); + super.tearDown(); + } + +} Propchange: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartition.java ------------------------------------------------------------------------------ svn:eol-style = native Added: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java?rev=1137561&view=auto ============================================================================== --- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java (added) +++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java Mon Jun 20 09:28:55 2011 @@ -0,0 +1,49 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore; + +import org.apache.hadoop.hive.conf.HiveConf; + +public class TestMarkPartitionRemote extends TestMarkPartition{ + + private static class RunMS implements Runnable { + + @Override + public void run() { + try { + HiveMetaStore.main(new String[] { "29111" }); + } catch (Throwable e) { + e.printStackTrace(System.err); + assert false; + } + } + + } + @Override + protected void setUp() throws Exception { + super.setUp(); + Thread t = new Thread(new RunMS()); + t.setDaemon(true); + t.start(); + hiveConf.set("hive.metastore.local", "false"); + hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, "thrift://localhost:29111"); + hiveConf.setIntVar(HiveConf.ConfVars.METASTORETHRIFTRETRIES, 3); + Thread.sleep(30000); + } +} Propchange: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMarkPartitionRemote.java ------------------------------------------------------------------------------ svn:eol-style = native Modified: hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java URL: http://svn.apache.org/viewvc/hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java?rev=1137561&r1=1137560&r2=1137561&view=diff ============================================================================== --- hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java (original) +++ hive/trunk/metastore/src/test/org/apache/hadoop/hive/metastore/TestMetaStoreEventListener.java Mon Jun 20 09:28:55 2011 @@ -18,7 +18,9 @@ package org.apache.hadoop.hive.metastore; +import java.util.HashMap; import java.util.List; +import java.util.Map; import junit.framework.TestCase; @@ -27,6 +29,7 @@ import org.apache.hadoop.hive.conf.HiveC import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.metastore.api.Database; import org.apache.hadoop.hive.metastore.api.Partition; +import org.apache.hadoop.hive.metastore.api.PartitionEventType; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.events.AddPartitionEvent; import org.apache.hadoop.hive.metastore.events.CreateDatabaseEvent; @@ -35,6 +38,7 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.events.DropPartitionEvent; import org.apache.hadoop.hive.metastore.events.DropTableEvent; import org.apache.hadoop.hive.metastore.events.ListenerEvent; +import org.apache.hadoop.hive.metastore.events.LoadPartitionDoneEvent; import org.apache.hadoop.hive.ql.Driver; import org.apache.hadoop.hive.ql.session.SessionState; @@ -60,7 +64,6 @@ public class TestMetaStoreEventListener assert false; } } - } @Override @@ -119,17 +122,26 @@ public class TestMetaStoreEventListener assert partEvent.getStatus(); assertEquals(part, partEvent.getPartition()); - driver.run("alter table tmptbl drop partition (b='2011')"); + Map<String,String> kvs = new HashMap<String, String>(1); + kvs.put("b", "2011"); + msc.markPartitionForEvent("tmpdb", "tmptbl", kvs, PartitionEventType.LOAD_DONE); assertEquals(notifyList.size(), 4); - DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(3); + LoadPartitionDoneEvent partMarkEvent = (LoadPartitionDoneEvent)notifyList.get(3); + assert partMarkEvent.getStatus(); + assertEquals(partMarkEvent.getPartitionName(), kvs); + assertEquals(partMarkEvent.getTable().getTableName(), "tmptbl"); + + driver.run("alter table tmptbl drop partition (b='2011')"); + assertEquals(notifyList.size(), 5); + DropPartitionEvent dropPart = (DropPartitionEvent)notifyList.get(4); assert dropPart.getStatus(); assertEquals(part.getValues(), dropPart.getPartition().getValues()); assertEquals(part.getDbName(), dropPart.getPartition().getDbName()); assertEquals(part.getTableName(), dropPart.getPartition().getTableName()); driver.run("drop table tmptbl"); - assertEquals(notifyList.size(), 5); - DropTableEvent dropTbl = (DropTableEvent)notifyList.get(4); + assertEquals(notifyList.size(), 6); + DropTableEvent dropTbl = (DropTableEvent)notifyList.get(5); assert dropTbl.getStatus(); assertEquals(tbl.getTableName(), dropTbl.getTable().getTableName()); assertEquals(tbl.getDbName(), dropTbl.getTable().getDbName()); @@ -137,8 +149,8 @@ public class TestMetaStoreEventListener driver.run("drop database tmpdb"); - assertEquals(notifyList.size(), 6); - DropDatabaseEvent dropDB = (DropDatabaseEvent)notifyList.get(5); + assertEquals(notifyList.size(), 7); + DropDatabaseEvent dropDB = (DropDatabaseEvent)notifyList.get(6); assert dropDB.getStatus(); assertEquals(db, dropDB.getDatabase()); }
