Repository: hive Updated Branches: refs/heads/master a60fec2eb -> cdcc35e61
http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php ---------------------------------------------------------------------- diff --git a/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php b/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php index 75f4a47..d0289fe 100644 --- a/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php +++ b/metastore/src/gen/thrift/gen-php/metastore/ThriftHiveMetastore.php @@ -533,6 +533,15 @@ interface ThriftHiveMetastoreIf extends \FacebookServiceIf { /** * @param string $db_name * @param string $tbl_name + * @param string $filter + * @return int + * @throws \metastore\MetaException + * @throws \metastore\NoSuchObjectException + */ + public function get_num_partitions_by_filter($db_name, $tbl_name, $filter); + /** + * @param string $db_name + * @param string $tbl_name * @param string[] $names * @return \metastore\Partition[] * @throws \metastore\MetaException @@ -4485,6 +4494,65 @@ class ThriftHiveMetastoreClient extends \FacebookServiceClient implements \metas throw new \Exception("get_partitions_by_expr failed: unknown result"); } + public function get_num_partitions_by_filter($db_name, $tbl_name, $filter) + { + $this->send_get_num_partitions_by_filter($db_name, $tbl_name, $filter); + return $this->recv_get_num_partitions_by_filter(); + } + + public function send_get_num_partitions_by_filter($db_name, $tbl_name, $filter) + { + $args = new \metastore\ThriftHiveMetastore_get_num_partitions_by_filter_args(); + $args->db_name = $db_name; + $args->tbl_name = $tbl_name; + $args->filter = $filter; + $bin_accel = ($this->output_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_write_binary'); + if ($bin_accel) + { + thrift_protocol_write_binary($this->output_, 'get_num_partitions_by_filter', TMessageType::CALL, $args, $this->seqid_, $this->output_->isStrictWrite()); + } + else + { + $this->output_->writeMessageBegin('get_num_partitions_by_filter', TMessageType::CALL, $this->seqid_); + $args->write($this->output_); + $this->output_->writeMessageEnd(); + $this->output_->getTransport()->flush(); + } + } + + public function recv_get_num_partitions_by_filter() + { + $bin_accel = ($this->input_ instanceof TBinaryProtocolAccelerated) && function_exists('thrift_protocol_read_binary'); + if ($bin_accel) $result = thrift_protocol_read_binary($this->input_, '\metastore\ThriftHiveMetastore_get_num_partitions_by_filter_result', $this->input_->isStrictRead()); + else + { + $rseqid = 0; + $fname = null; + $mtype = 0; + + $this->input_->readMessageBegin($fname, $mtype, $rseqid); + if ($mtype == TMessageType::EXCEPTION) { + $x = new TApplicationException(); + $x->read($this->input_); + $this->input_->readMessageEnd(); + throw $x; + } + $result = new \metastore\ThriftHiveMetastore_get_num_partitions_by_filter_result(); + $result->read($this->input_); + $this->input_->readMessageEnd(); + } + if ($result->success !== null) { + return $result->success; + } + if ($result->o1 !== null) { + throw $result->o1; + } + if ($result->o2 !== null) { + throw $result->o2; + } + throw new \Exception("get_num_partitions_by_filter failed: unknown result"); + } + public function get_partitions_by_names($db_name, $tbl_name, array $names) { $this->send_get_partitions_by_names($db_name, $tbl_name, $names); @@ -23936,6 +24004,252 @@ class ThriftHiveMetastore_get_partitions_by_expr_result { } +class ThriftHiveMetastore_get_num_partitions_by_filter_args { + static $_TSPEC; + + /** + * @var string + */ + public $db_name = null; + /** + * @var string + */ + public $tbl_name = null; + /** + * @var string + */ + public $filter = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 1 => array( + 'var' => 'db_name', + 'type' => TType::STRING, + ), + 2 => array( + 'var' => 'tbl_name', + 'type' => TType::STRING, + ), + 3 => array( + 'var' => 'filter', + 'type' => TType::STRING, + ), + ); + } + if (is_array($vals)) { + if (isset($vals['db_name'])) { + $this->db_name = $vals['db_name']; + } + if (isset($vals['tbl_name'])) { + $this->tbl_name = $vals['tbl_name']; + } + if (isset($vals['filter'])) { + $this->filter = $vals['filter']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_get_num_partitions_by_filter_args'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 1: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->db_name); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->tbl_name); + } else { + $xfer += $input->skip($ftype); + } + break; + case 3: + if ($ftype == TType::STRING) { + $xfer += $input->readString($this->filter); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_num_partitions_by_filter_args'); + if ($this->db_name !== null) { + $xfer += $output->writeFieldBegin('db_name', TType::STRING, 1); + $xfer += $output->writeString($this->db_name); + $xfer += $output->writeFieldEnd(); + } + if ($this->tbl_name !== null) { + $xfer += $output->writeFieldBegin('tbl_name', TType::STRING, 2); + $xfer += $output->writeString($this->tbl_name); + $xfer += $output->writeFieldEnd(); + } + if ($this->filter !== null) { + $xfer += $output->writeFieldBegin('filter', TType::STRING, 3); + $xfer += $output->writeString($this->filter); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + +class ThriftHiveMetastore_get_num_partitions_by_filter_result { + static $_TSPEC; + + /** + * @var int + */ + public $success = null; + /** + * @var \metastore\MetaException + */ + public $o1 = null; + /** + * @var \metastore\NoSuchObjectException + */ + public $o2 = null; + + public function __construct($vals=null) { + if (!isset(self::$_TSPEC)) { + self::$_TSPEC = array( + 0 => array( + 'var' => 'success', + 'type' => TType::I32, + ), + 1 => array( + 'var' => 'o1', + 'type' => TType::STRUCT, + 'class' => '\metastore\MetaException', + ), + 2 => array( + 'var' => 'o2', + 'type' => TType::STRUCT, + 'class' => '\metastore\NoSuchObjectException', + ), + ); + } + if (is_array($vals)) { + if (isset($vals['success'])) { + $this->success = $vals['success']; + } + if (isset($vals['o1'])) { + $this->o1 = $vals['o1']; + } + if (isset($vals['o2'])) { + $this->o2 = $vals['o2']; + } + } + } + + public function getName() { + return 'ThriftHiveMetastore_get_num_partitions_by_filter_result'; + } + + public function read($input) + { + $xfer = 0; + $fname = null; + $ftype = 0; + $fid = 0; + $xfer += $input->readStructBegin($fname); + while (true) + { + $xfer += $input->readFieldBegin($fname, $ftype, $fid); + if ($ftype == TType::STOP) { + break; + } + switch ($fid) + { + case 0: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->success); + } else { + $xfer += $input->skip($ftype); + } + break; + case 1: + if ($ftype == TType::STRUCT) { + $this->o1 = new \metastore\MetaException(); + $xfer += $this->o1->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + case 2: + if ($ftype == TType::STRUCT) { + $this->o2 = new \metastore\NoSuchObjectException(); + $xfer += $this->o2->read($input); + } else { + $xfer += $input->skip($ftype); + } + break; + default: + $xfer += $input->skip($ftype); + break; + } + $xfer += $input->readFieldEnd(); + } + $xfer += $input->readStructEnd(); + return $xfer; + } + + public function write($output) { + $xfer = 0; + $xfer += $output->writeStructBegin('ThriftHiveMetastore_get_num_partitions_by_filter_result'); + if ($this->success !== null) { + $xfer += $output->writeFieldBegin('success', TType::I32, 0); + $xfer += $output->writeI32($this->success); + $xfer += $output->writeFieldEnd(); + } + if ($this->o1 !== null) { + $xfer += $output->writeFieldBegin('o1', TType::STRUCT, 1); + $xfer += $this->o1->write($output); + $xfer += $output->writeFieldEnd(); + } + if ($this->o2 !== null) { + $xfer += $output->writeFieldBegin('o2', TType::STRUCT, 2); + $xfer += $this->o2->write($output); + $xfer += $output->writeFieldEnd(); + } + $xfer += $output->writeFieldStop(); + $xfer += $output->writeStructEnd(); + return $xfer; + } + +} + class ThriftHiveMetastore_get_partitions_by_names_args { static $_TSPEC; @@ -39709,3 +40023,4 @@ class ThriftHiveMetastore_cache_file_metadata_result { } + http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote ---------------------------------------------------------------------- diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote index 9564ded..407f73c 100755 --- a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote +++ b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore-remote @@ -82,6 +82,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' get_partitions_by_filter(string db_name, string tbl_name, string filter, i16 max_parts)') print(' get_part_specs_by_filter(string db_name, string tbl_name, string filter, i32 max_parts)') print(' PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req)') + print(' i32 get_num_partitions_by_filter(string db_name, string tbl_name, string filter)') print(' get_partitions_by_names(string db_name, string tbl_name, names)') print(' void alter_partition(string db_name, string tbl_name, Partition new_part)') print(' void alter_partitions(string db_name, string tbl_name, new_parts)') @@ -573,6 +574,12 @@ elif cmd == 'get_partitions_by_expr': sys.exit(1) pp.pprint(client.get_partitions_by_expr(eval(args[0]),)) +elif cmd == 'get_num_partitions_by_filter': + if len(args) != 3: + print('get_num_partitions_by_filter requires 3 args') + sys.exit(1) + pp.pprint(client.get_num_partitions_by_filter(args[0],args[1],args[2],)) + elif cmd == 'get_partitions_by_names': if len(args) != 3: print('get_partitions_by_names requires 3 args') http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py ---------------------------------------------------------------------- diff --git a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py index 97acb5d..26d8a02 100644 --- a/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py +++ b/metastore/src/gen/thrift/gen-py/hive_metastore/ThriftHiveMetastore.py @@ -524,6 +524,15 @@ class Iface(fb303.FacebookService.Iface): """ pass + def get_num_partitions_by_filter(self, db_name, tbl_name, filter): + """ + Parameters: + - db_name + - tbl_name + - filter + """ + pass + def get_partitions_by_names(self, db_name, tbl_name, names): """ Parameters: @@ -3323,6 +3332,45 @@ class Client(fb303.FacebookService.Client, Iface): raise result.o2 raise TApplicationException(TApplicationException.MISSING_RESULT, "get_partitions_by_expr failed: unknown result") + def get_num_partitions_by_filter(self, db_name, tbl_name, filter): + """ + Parameters: + - db_name + - tbl_name + - filter + """ + self.send_get_num_partitions_by_filter(db_name, tbl_name, filter) + return self.recv_get_num_partitions_by_filter() + + def send_get_num_partitions_by_filter(self, db_name, tbl_name, filter): + self._oprot.writeMessageBegin('get_num_partitions_by_filter', TMessageType.CALL, self._seqid) + args = get_num_partitions_by_filter_args() + args.db_name = db_name + args.tbl_name = tbl_name + args.filter = filter + args.write(self._oprot) + self._oprot.writeMessageEnd() + self._oprot.trans.flush() + + def recv_get_num_partitions_by_filter(self): + iprot = self._iprot + (fname, mtype, rseqid) = iprot.readMessageBegin() + if mtype == TMessageType.EXCEPTION: + x = TApplicationException() + x.read(iprot) + iprot.readMessageEnd() + raise x + result = get_num_partitions_by_filter_result() + result.read(iprot) + iprot.readMessageEnd() + if result.success is not None: + return result.success + if result.o1 is not None: + raise result.o1 + if result.o2 is not None: + raise result.o2 + raise TApplicationException(TApplicationException.MISSING_RESULT, "get_num_partitions_by_filter failed: unknown result") + def get_partitions_by_names(self, db_name, tbl_name, names): """ Parameters: @@ -5973,6 +6021,7 @@ class Processor(fb303.FacebookService.Processor, Iface, TProcessor): self._processMap["get_partitions_by_filter"] = Processor.process_get_partitions_by_filter self._processMap["get_part_specs_by_filter"] = Processor.process_get_part_specs_by_filter self._processMap["get_partitions_by_expr"] = Processor.process_get_partitions_by_expr + self._processMap["get_num_partitions_by_filter"] = Processor.process_get_num_partitions_by_filter self._processMap["get_partitions_by_names"] = Processor.process_get_partitions_by_names self._processMap["alter_partition"] = Processor.process_alter_partition self._processMap["alter_partitions"] = Processor.process_alter_partitions @@ -7564,6 +7613,31 @@ class Processor(fb303.FacebookService.Processor, Iface, TProcessor): oprot.writeMessageEnd() oprot.trans.flush() + def process_get_num_partitions_by_filter(self, seqid, iprot, oprot): + args = get_num_partitions_by_filter_args() + args.read(iprot) + iprot.readMessageEnd() + result = get_num_partitions_by_filter_result() + try: + result.success = self._handler.get_num_partitions_by_filter(args.db_name, args.tbl_name, args.filter) + msg_type = TMessageType.REPLY + except (TTransport.TTransportException, KeyboardInterrupt, SystemExit): + raise + except MetaException as o1: + msg_type = TMessageType.REPLY + result.o1 = o1 + except NoSuchObjectException as o2: + msg_type = TMessageType.REPLY + result.o2 = o2 + except Exception as ex: + msg_type = TMessageType.EXCEPTION + logging.exception(ex) + result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') + oprot.writeMessageBegin("get_num_partitions_by_filter", msg_type, seqid) + result.write(oprot) + oprot.writeMessageEnd() + oprot.trans.flush() + def process_get_partitions_by_names(self, seqid, iprot, oprot): args = get_partitions_by_names_args() args.read(iprot) @@ -20186,6 +20260,189 @@ class get_partitions_by_expr_result: def __ne__(self, other): return not (self == other) +class get_num_partitions_by_filter_args: + """ + Attributes: + - db_name + - tbl_name + - filter + """ + + thrift_spec = ( + None, # 0 + (1, TType.STRING, 'db_name', None, None, ), # 1 + (2, TType.STRING, 'tbl_name', None, None, ), # 2 + (3, TType.STRING, 'filter', None, None, ), # 3 + ) + + def __init__(self, db_name=None, tbl_name=None, filter=None,): + self.db_name = db_name + self.tbl_name = tbl_name + self.filter = filter + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.db_name = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.tbl_name = iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.filter = iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('get_num_partitions_by_filter_args') + if self.db_name is not None: + oprot.writeFieldBegin('db_name', TType.STRING, 1) + oprot.writeString(self.db_name) + oprot.writeFieldEnd() + if self.tbl_name is not None: + oprot.writeFieldBegin('tbl_name', TType.STRING, 2) + oprot.writeString(self.tbl_name) + oprot.writeFieldEnd() + if self.filter is not None: + oprot.writeFieldBegin('filter', TType.STRING, 3) + oprot.writeString(self.filter) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.db_name) + value = (value * 31) ^ hash(self.tbl_name) + value = (value * 31) ^ hash(self.filter) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + +class get_num_partitions_by_filter_result: + """ + Attributes: + - success + - o1 + - o2 + """ + + thrift_spec = ( + (0, TType.I32, 'success', None, None, ), # 0 + (1, TType.STRUCT, 'o1', (MetaException, MetaException.thrift_spec), None, ), # 1 + (2, TType.STRUCT, 'o2', (NoSuchObjectException, NoSuchObjectException.thrift_spec), None, ), # 2 + ) + + def __init__(self, success=None, o1=None, o2=None,): + self.success = success + self.o1 = o1 + self.o2 = o2 + + def read(self, iprot): + if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: + fastbinary.decode_binary(self, iprot.trans, (self.__class__, self.thrift_spec)) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 0: + if ftype == TType.I32: + self.success = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 1: + if ftype == TType.STRUCT: + self.o1 = MetaException() + self.o1.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.o2 = NoSuchObjectException() + self.o2.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and self.thrift_spec is not None and fastbinary is not None: + oprot.trans.write(fastbinary.encode_binary(self, (self.__class__, self.thrift_spec))) + return + oprot.writeStructBegin('get_num_partitions_by_filter_result') + if self.success is not None: + oprot.writeFieldBegin('success', TType.I32, 0) + oprot.writeI32(self.success) + oprot.writeFieldEnd() + if self.o1 is not None: + oprot.writeFieldBegin('o1', TType.STRUCT, 1) + self.o1.write(oprot) + oprot.writeFieldEnd() + if self.o2 is not None: + oprot.writeFieldBegin('o2', TType.STRUCT, 2) + self.o2.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + + def __hash__(self): + value = 17 + value = (value * 31) ^ hash(self.success) + value = (value * 31) ^ hash(self.o1) + value = (value * 31) ^ hash(self.o2) + return value + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.iteritems()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + class get_partitions_by_names_args: """ Attributes: http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb ---------------------------------------------------------------------- diff --git a/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb b/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb index a90a180..9b9a27c 100644 --- a/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb +++ b/metastore/src/gen/thrift/gen-rb/thrift_hive_metastore.rb @@ -1004,6 +1004,23 @@ module ThriftHiveMetastore raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_partitions_by_expr failed: unknown result') end + def get_num_partitions_by_filter(db_name, tbl_name, filter) + send_get_num_partitions_by_filter(db_name, tbl_name, filter) + return recv_get_num_partitions_by_filter() + end + + def send_get_num_partitions_by_filter(db_name, tbl_name, filter) + send_message('get_num_partitions_by_filter', Get_num_partitions_by_filter_args, :db_name => db_name, :tbl_name => tbl_name, :filter => filter) + end + + def recv_get_num_partitions_by_filter() + result = receive_message(Get_num_partitions_by_filter_result) + return result.success unless result.success.nil? + raise result.o1 unless result.o1.nil? + raise result.o2 unless result.o2.nil? + raise ::Thrift::ApplicationException.new(::Thrift::ApplicationException::MISSING_RESULT, 'get_num_partitions_by_filter failed: unknown result') + end + def get_partitions_by_names(db_name, tbl_name, names) send_get_partitions_by_names(db_name, tbl_name, names) return recv_get_partitions_by_names() @@ -3007,6 +3024,19 @@ module ThriftHiveMetastore write_result(result, oprot, 'get_partitions_by_expr', seqid) end + def process_get_num_partitions_by_filter(seqid, iprot, oprot) + args = read_args(iprot, Get_num_partitions_by_filter_args) + result = Get_num_partitions_by_filter_result.new() + begin + result.success = @handler.get_num_partitions_by_filter(args.db_name, args.tbl_name, args.filter) + rescue ::MetaException => o1 + result.o1 = o1 + rescue ::NoSuchObjectException => o2 + result.o2 = o2 + end + write_result(result, oprot, 'get_num_partitions_by_filter', seqid) + end + def process_get_partitions_by_names(seqid, iprot, oprot) args = read_args(iprot, Get_partitions_by_names_args) result = Get_partitions_by_names_result.new() @@ -6176,6 +6206,46 @@ module ThriftHiveMetastore ::Thrift::Struct.generate_accessors self end + class Get_num_partitions_by_filter_args + include ::Thrift::Struct, ::Thrift::Struct_Union + DB_NAME = 1 + TBL_NAME = 2 + FILTER = 3 + + FIELDS = { + DB_NAME => {:type => ::Thrift::Types::STRING, :name => 'db_name'}, + TBL_NAME => {:type => ::Thrift::Types::STRING, :name => 'tbl_name'}, + FILTER => {:type => ::Thrift::Types::STRING, :name => 'filter'} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + + class Get_num_partitions_by_filter_result + include ::Thrift::Struct, ::Thrift::Struct_Union + SUCCESS = 0 + O1 = 1 + O2 = 2 + + FIELDS = { + SUCCESS => {:type => ::Thrift::Types::I32, :name => 'success'}, + O1 => {:type => ::Thrift::Types::STRUCT, :name => 'o1', :class => ::MetaException}, + O2 => {:type => ::Thrift::Types::STRUCT, :name => 'o2', :class => ::NoSuchObjectException} + } + + def struct_fields; FIELDS; end + + def validate + end + + ::Thrift::Struct.generate_accessors self + end + class Get_partitions_by_names_args include ::Thrift::Struct, ::Thrift::Struct_Union DB_NAME = 1 http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java index 81d7128..ace644b 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java @@ -4559,6 +4559,25 @@ public class HiveMetaStore extends ThriftHiveMetastore { } @Override + public int get_num_partitions_by_filter(final String dbName, + final String tblName, final String filter) + throws MetaException, NoSuchObjectException, TException { + startTableFunction("get_num_partitions_by_filter", dbName, tblName); + + int ret = -1; + Exception ex = null; + try { + ret = getMS().getNumPartitionsByFilter(dbName, tblName, filter); + } catch (Exception e) { + ex = e; + rethrowException(e); + } finally { + endFunction("get_num_partitions_by_filter", ret != -1, ex, tblName); + } + return ret; + } + + @Override public List<Partition> get_partitions_by_names(final String dbName, final String tblName, final List<String> partNames) throws MetaException, NoSuchObjectException, TException { http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index a17c6d8..09a6aea 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -1204,7 +1204,6 @@ public class HiveMetaStoreClient implements IMetaStoreClient { return !r.isSetHasUnknownPartitions() || r.isHasUnknownPartitions(); // Assume the worst. } - /** * @param name * @return the database @@ -1395,6 +1394,24 @@ public class HiveMetaStoreClient implements IMetaStoreClient { client.get_partition_names_ps(db_name, tbl_name, part_vals, max_parts)); } + /** + * Get number of partitions matching specified filter + * @param db_name the database name + * @param tbl_name the table name + * @param filter the filter string, + * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can + * be done only on string partition keys. + * @return number of partitions + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + */ + public int getNumPartitionsByFilter(String db_name, String tbl_name, + String filter) throws MetaException, + NoSuchObjectException, TException { + return client.get_num_partitions_by_filter(db_name, tbl_name, filter); + } + @Override public void alter_partition(String dbName, String tblName, Partition newPart) throws InvalidOperationException, MetaException, TException { http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 25e0d38..4284d54 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -530,6 +530,22 @@ public interface IMetaStoreClient { throws MetaException, TException, NoSuchObjectException; /** + * Get number of partitions matching specified filter + * @param dbName the database name + * @param tableName the table name + * @param filter the filter string, + * for example "part1 = \"p1_abc\" and part2 <= "\p2_test\"". Filtering can + * be done only on string partition keys. + * @return number of partitions + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + */ + public int getNumPartitionsByFilter(String dbName, String tableName, + String filter) throws MetaException, NoSuchObjectException, TException; + + + /** * Get list of partitions matching specified filter * @param db_name the database name * @param tbl_name the table name http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java index 36b316a..be54b9c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java @@ -385,6 +385,20 @@ class MetaStoreDirectSql { isViewTable, sqlFilter, params, joins, max); } + public int getNumPartitionsViaSqlFilter(Table table, ExpressionTree tree) throws MetaException { + List<Object> params = new ArrayList<Object>(); + List<String>joins = new ArrayList<String>(); + // Derby and Oracle do not interpret filters ANSI-properly in some cases and need a workaround. + boolean dbHasJoinCastBug = (dbType == DB.DERBY || dbType == DB.ORACLE); + String sqlFilter = PartitionFilterGenerator.generateSqlFilter( + table, tree, params, joins, dbHasJoinCastBug, defaultPartName); + if (sqlFilter == null) { + return 0; // Cannot make SQL filter to push down. + } + return getNumPartitionsViaSqlFilterInternal(table.getDbName(), table.getTableName(), sqlFilter, params, joins); + } + + /** * Gets all partitions of a table by using direct SQL queries. * @param dbName Metastore db name. @@ -807,6 +821,39 @@ class MetaStoreDirectSql { return orderedResult; } + private int getNumPartitionsViaSqlFilterInternal(String dbName, String tblName, + String sqlFilter, List<Object> paramsForFilter, + List<String> joinsForFilter) throws MetaException { + boolean doTrace = LOG.isDebugEnabled(); + dbName = dbName.toLowerCase(); + tblName = tblName.toLowerCase(); + + // Get number of partitions by doing count on PART_ID. + String queryText = "select count(\"PARTITIONS\".\"PART_ID\") from \"PARTITIONS\"" + + " inner join \"TBLS\" on \"PARTITIONS\".\"TBL_ID\" = \"TBLS\".\"TBL_ID\" " + + " and \"TBLS\".\"TBL_NAME\" = ? " + + " inner join \"DBS\" on \"TBLS\".\"DB_ID\" = \"DBS\".\"DB_ID\" " + + " and \"DBS\".\"NAME\" = ? " + + join(joinsForFilter, ' ') + + (sqlFilter == null ? "" : (" where " + sqlFilter)); + + Object[] params = new Object[paramsForFilter.size() + 2]; + params[0] = tblName; + params[1] = dbName; + for (int i = 0; i < paramsForFilter.size(); ++i) { + params[i + 2] = paramsForFilter.get(i); + } + + long start = doTrace ? System.nanoTime() : 0; + Query query = pm.newQuery("javax.jdo.query.SQL", queryText); + @SuppressWarnings("unchecked") + int sqlResult = extractSqlInt(query.executeWithArray(params)); + long queryTime = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, queryText, start, queryTime); + return sqlResult; + } + + private void timingTrace(boolean doTrace, String queryText, long start, long queryTime) { if (!doTrace) return; LOG.debug("Direct SQL query in " + (queryTime - start) / 1000000.0 + "ms + " + http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java index 05d5b57..e1b8b6c 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -2279,6 +2279,7 @@ public class ObjectStore implements RawStore, Configurable { } return result; } + @Override protected List<Partition> getJdoResult( GetHelper<List<Partition>> ctx) throws MetaException, NoSuchObjectException { @@ -2365,6 +2366,12 @@ public class ObjectStore implements RawStore, Configurable { return results; } + + private Integer getNumPartitionsViaOrmFilter(Table table, ExpressionTree tree, boolean isValidatedFilter) + throws MetaException { + return getPartitionsViaOrmFilter(table, tree, (short) -1, isValidatedFilter).size(); + } + /** * Gets partition names from the table via ORM (JDOQL) name filter. * @param dbName Database name. @@ -2646,6 +2653,36 @@ public class ObjectStore implements RawStore, Configurable { } } + @Override + public int getNumPartitionsByFilter(String dbName, String tblName, + String filter) throws MetaException, NoSuchObjectException { + return getNumPartitionsByFilterInternal(dbName, tblName, filter, + true, true); + } + + protected int getNumPartitionsByFilterInternal(String dbName, String tblName, + String filter, boolean allowSql, boolean allowJdo) + throws MetaException, NoSuchObjectException { + final ExpressionTree tree = (filter != null && !filter.isEmpty()) + ? PartFilterExprUtil.getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; + return new GetHelper<Integer>(dbName, tblName, allowSql, allowJdo) { + @Override + protected String describeResult() { + return null; + } + + @Override + protected Integer getSqlResult(GetHelper<Integer> ctx) throws MetaException { + return directSql.getNumPartitionsViaSqlFilter(ctx.getTable(), tree); + } + @Override + protected Integer getJdoResult( + GetHelper<Integer> ctx) throws MetaException, NoSuchObjectException { + return getNumPartitionsViaOrmFilter(ctx.getTable(), tree, true); + } + }.run(true); + } + protected List<Partition> getPartitionsByFilterInternal(String dbName, String tblName, String filter, final short maxParts, boolean allowSql, boolean allowJdo) throws MetaException, NoSuchObjectException { http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java index d228f24..cbd5957 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java @@ -223,6 +223,9 @@ public interface RawStore extends Configurable { byte[] expr, String defaultPartitionName, short maxParts, List<Partition> result) throws TException; + public abstract int getNumPartitionsByFilter(String dbName, String tblName, String filter) + throws MetaException, NoSuchObjectException; + public abstract List<Partition> getPartitionsByNames( String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException; http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java index fcf983f..0ffdbe0 100644 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java +++ b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/HBaseStore.java @@ -758,6 +758,21 @@ public class HBaseStore implements RawStore { } } + @Override + public int getNumPartitionsByFilter(String dbName, String tblName, String filter) + throws MetaException, NoSuchObjectException { + final ExpressionTree exprTree = (filter != null && !filter.isEmpty()) ? PartFilterExprUtil + .getFilterParser(filter).tree : ExpressionTree.EMPTY_TREE; + List<Partition> result = new ArrayList<Partition>(); + boolean commit = false; + openTransaction(); + try { + return getPartitionsByFilter(dbName, tblName, filter, Short.MAX_VALUE).size(); + } finally { + commitOrRoleBack(commit); + } + } + /** * Gets the partition names from a table, pruned using an expression. * @param table Table. http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java index 477a3be..660dd4e 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java @@ -310,6 +310,12 @@ public class DummyRawStoreControlledCommit implements RawStore, Configurable { } @Override + public int getNumPartitionsByFilter(String dbName, String tblName, + String filter) throws MetaException, NoSuchObjectException { + return objectStore.getNumPartitionsByFilter(dbName, tblName, filter); + } + + @Override public List<Partition> getPartitionsByNames(String dbName, String tblName, List<String> partNames) throws MetaException, NoSuchObjectException { return objectStore.getPartitionsByNames(dbName, tblName, partNames); http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java index ecabd5d..9103102 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java @@ -341,6 +341,12 @@ public class DummyRawStoreForJdoConnection implements RawStore { } @Override + public int getNumPartitionsByFilter(String dbName, String tblName, String filter) + throws MetaException, NoSuchObjectException { + return -1; + } + + @Override public Table markPartitionForEvent(String dbName, String tblName, Map<String, String> partVals, PartitionEventType evtType) throws MetaException, UnknownTableException, InvalidPartitionException, UnknownPartitionException { http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java index 1157033..2e1f5f4 100644 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java @@ -37,6 +37,7 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; +import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo; import org.junit.After; import org.junit.Assert; @@ -163,8 +164,8 @@ public class TestObjectStore { StorageDescriptor sd = new StorageDescriptor(null, "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); HashMap<String,String> tableParams = new HashMap<String,String>(); tableParams.put("EXTERNAL", "false"); - FieldSchema partitionKey1 = new FieldSchema("Country", "String", ""); - FieldSchema partitionKey2 = new FieldSchema("State", "String", ""); + FieldSchema partitionKey1 = new FieldSchema("Country", serdeConstants.STRING_TYPE_NAME, ""); + FieldSchema partitionKey2 = new FieldSchema("State", serdeConstants.STRING_TYPE_NAME, ""); Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, "viewOriginalText", "viewExpandedText", "MANAGED_TABLE"); objectStore.createTable(tbl1); HashMap<String, String> partitionParams = new HashMap<String, String>(); @@ -182,6 +183,12 @@ public class TestObjectStore { Assert.assertEquals(111, partitions.get(0).getCreateTime()); Assert.assertEquals(222, partitions.get(1).getCreateTime()); + int numPartitions = objectStore.getNumPartitionsByFilter(DB1, TABLE1, ""); + Assert.assertEquals(partitions.size(), numPartitions); + + numPartitions = objectStore.getNumPartitionsByFilter(DB1, TABLE1, "country = \"US\""); + Assert.assertEquals(2, numPartitions); + objectStore.dropPartition(DB1, TABLE1, value1); partitions = objectStore.getPartitions(DB1, TABLE1, 10); Assert.assertEquals(1, partitions.size()); http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 29df4f9..dca1198 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2373,6 +2373,30 @@ private void constructOneLBLocationMap(FileStatus fSta, return hasUnknownParts; } + /** + * Get a number of Partitions by filter. + * @param tbl The table containing the partitions. + * @param filter A string represent partition predicates. + * @return the number of partitions satisfying the partition predicates. + * @throws HiveException + * @throws MetaException + * @throws NoSuchObjectException + * @throws TException + */ + public int getNumPartitionsByFilter(Table tbl, String filter) + throws HiveException, MetaException, NoSuchObjectException, TException { + + if (!tbl.isPartitioned()) { + throw new HiveException("Partition spec should only be supplied for a " + + "partitioned table"); + } + + int numParts = getMSC().getNumPartitionsByFilter( + tbl.getDbName(), tbl.getTableName(), filter); + + return numParts; + } + public void validatePartitionNameCharacters(List<String> partVals) throws HiveException { try { getMSC().validatePartitionNameCharacters(partVals); http://git-wip-us.apache.org/repos/asf/hive/blob/cdcc35e6/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote ---------------------------------------------------------------------- diff --git a/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote b/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote index 4bd7a5a..ec21b6c 100755 --- a/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote +++ b/service/src/gen/thrift/gen-py/hive_service/ThriftHive-remote @@ -91,6 +91,7 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' get_partitions_by_filter(string db_name, string tbl_name, string filter, i16 max_parts)') print(' get_part_specs_by_filter(string db_name, string tbl_name, string filter, i32 max_parts)') print(' PartitionsByExprResult get_partitions_by_expr(PartitionsByExprRequest req)') + print(' i32 get_num_partitions_by_filter(string db_name, string tbl_name, string filter)') print(' get_partitions_by_names(string db_name, string tbl_name, names)') print(' void alter_partition(string db_name, string tbl_name, Partition new_part)') print(' void alter_partitions(string db_name, string tbl_name, new_parts)') @@ -636,6 +637,12 @@ elif cmd == 'get_partitions_by_expr': sys.exit(1) pp.pprint(client.get_partitions_by_expr(eval(args[0]),)) +elif cmd == 'get_num_partitions_by_filter': + if len(args) != 3: + print('get_num_partitions_by_filter requires 3 args') + sys.exit(1) + pp.pprint(client.get_num_partitions_by_filter(args[0],args[1],args[2],)) + elif cmd == 'get_partitions_by_names': if len(args) != 3: print('get_partitions_by_names requires 3 args')
