This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8f75876 HIVE-25753: Improving performance of
getLatestCommittedCompactionInfo (Yu-Wen Lai reviewed by Karen Coppage)
8f75876 is described below
commit 8f7587694ab9ba9dd152af274278074c31d0a453
Author: Yu-Wen Lai <[email protected]>
AuthorDate: Tue Nov 30 12:21:48 2021 -0800
HIVE-25753: Improving performance of getLatestCommittedCompactionInfo
(Yu-Wen Lai reviewed by Karen Coppage)
---
.../metastore/txn/TestCompactionTxnHandler.java | 39 ++++++++
.../gen/thrift/gen-cpp/hive_metastore_types.cpp | 42 ++++++--
.../src/gen/thrift/gen-cpp/hive_metastore_types.h | 12 ++-
.../GetLatestCommittedCompactionInfoRequest.java | 110 ++++++++++++++++++++-
.../GetLatestCommittedCompactionInfoRequest.php | 24 +++++
.../src/gen/thrift/gen-py/hive_metastore/ttypes.py | 14 ++-
.../src/gen/thrift/gen-rb/hive_metastore_types.rb | 4 +-
.../src/main/thrift/hive_metastore.thrift | 1 +
.../hadoop/hive/metastore/txn/TxnHandler.java | 8 +-
9 files changed, 233 insertions(+), 21 deletions(-)
diff --git
a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
index 9bfc324..a3b814e 100644
---
a/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
+++
b/ql/src/test/org/apache/hadoop/hive/metastore/txn/TestCompactionTxnHandler.java
@@ -330,6 +330,18 @@ public class TestCompactionTxnHandler {
CompactionInfoStruct lci2 = response.getCompactions().stream().
filter(c ->
c.getPartitionname().equals(anotherPartitionName)).findFirst().get();
assertEquals(3, lci2.getId());
+
+ // check the result is correct without setting partition names
+ rqst.unsetPartitionnames();
+ response = txnHandler.getLatestCommittedCompactionInfo(rqst);
+ assertNotNull(response);
+ assertEquals("Expecting a single compaction record for each partition", 2,
response.getCompactionsSize());
+ lci1 = response.getCompactions().stream()
+ .filter(c ->
c.getPartitionname().equals(partitionName)).findFirst().get();
+ assertEquals(1, lci1.getId());
+ lci2 = response.getCompactions().stream().
+ filter(c ->
c.getPartitionname().equals(anotherPartitionName)).findFirst().get();
+ assertEquals(3, lci2.getId());
}
@Test
@@ -366,6 +378,33 @@ public class TestCompactionTxnHandler {
}
@Test
+ public void testGetLatestCompactionWithIdFilter() throws Exception {
+ final String dbName = "foo";
+ final String tableName = "bar";
+ final String partitionName = "ds=today";
+ addSucceededCompaction(dbName, tableName, partitionName,
CompactionType.MINOR);
+ addSucceededCompaction(dbName, tableName, partitionName,
CompactionType.MINOR);
+ GetLatestCommittedCompactionInfoRequest rqst = new
GetLatestCommittedCompactionInfoRequest();
+ rqst.setDbname(dbName);
+ rqst.setTablename(tableName);
+ rqst.addToPartitionnames(partitionName);
+ GetLatestCommittedCompactionInfoResponse response =
txnHandler.getLatestCommittedCompactionInfo(rqst);
+
+ assertNotNull(response);
+ assertEquals("Expecting a single record", 1,
response.getCompactionsSize());
+ CompactionInfoStruct lci = response.getCompactions().get(0);
+ assertEquals("Expecting the second succeeded compaction record", 2,
lci.getId());
+ assertEquals(partitionName, lci.getPartitionname());
+ assertEquals(CompactionType.MINOR, lci.getType());
+
+ // response should only include compaction with id > 2
+ rqst.setLastCompactionId(2);
+ response = txnHandler.getLatestCommittedCompactionInfo(rqst);
+ assertNotNull(response);
+ assertEquals("Expecting no record", 0, response.getCompactionsSize());
+ }
+
+ @Test
public void testGetNoCompaction() throws Exception {
final String dbName = "foo";
final String tableName = "bar";
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
index 85ad737..d525faf 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.cpp
@@ -28149,6 +28149,11 @@ void
GetLatestCommittedCompactionInfoRequest::__set_partitionnames(const std::ve
this->partitionnames = val;
__isset.partitionnames = true;
}
+
+void GetLatestCommittedCompactionInfoRequest::__set_lastCompactionId(const
int64_t val) {
+ this->lastCompactionId = val;
+__isset.lastCompactionId = true;
+}
std::ostream& operator<<(std::ostream& out, const
GetLatestCommittedCompactionInfoRequest& obj)
{
obj.printTo(out);
@@ -28215,6 +28220,14 @@ uint32_t
GetLatestCommittedCompactionInfoRequest::read(::apache::thrift::protoco
xfer += iprot->skip(ftype);
}
break;
+ case 4:
+ if (ftype == ::apache::thrift::protocol::T_I64) {
+ xfer += iprot->readI64(this->lastCompactionId);
+ this->__isset.lastCompactionId = true;
+ } else {
+ xfer += iprot->skip(ftype);
+ }
+ break;
default:
xfer += iprot->skip(ftype);
break;
@@ -28257,6 +28270,11 @@ uint32_t
GetLatestCommittedCompactionInfoRequest::write(::apache::thrift::protoc
}
xfer += oprot->writeFieldEnd();
}
+ if (this->__isset.lastCompactionId) {
+ xfer += oprot->writeFieldBegin("lastCompactionId",
::apache::thrift::protocol::T_I64, 4);
+ xfer += oprot->writeI64(this->lastCompactionId);
+ xfer += oprot->writeFieldEnd();
+ }
xfer += oprot->writeFieldStop();
xfer += oprot->writeStructEnd();
return xfer;
@@ -28267,20 +28285,23 @@ void swap(GetLatestCommittedCompactionInfoRequest &a,
GetLatestCommittedCompacti
swap(a.dbname, b.dbname);
swap(a.tablename, b.tablename);
swap(a.partitionnames, b.partitionnames);
+ swap(a.lastCompactionId, b.lastCompactionId);
swap(a.__isset, b.__isset);
}
-GetLatestCommittedCompactionInfoRequest::GetLatestCommittedCompactionInfoRequest(const
GetLatestCommittedCompactionInfoRequest& other1007) {
- dbname = other1007.dbname;
- tablename = other1007.tablename;
- partitionnames = other1007.partitionnames;
- __isset = other1007.__isset;
+GetLatestCommittedCompactionInfoRequest::GetLatestCommittedCompactionInfoRequest(const
GetLatestCommittedCompactionInfoRequest& other1000) {
+ dbname = other1000.dbname;
+ tablename = other1000.tablename;
+ partitionnames = other1000.partitionnames;
+ lastCompactionId = other1000.lastCompactionId;
+ __isset = other1000.__isset;
}
-GetLatestCommittedCompactionInfoRequest&
GetLatestCommittedCompactionInfoRequest::operator=(const
GetLatestCommittedCompactionInfoRequest& other1008) {
- dbname = other1008.dbname;
- tablename = other1008.tablename;
- partitionnames = other1008.partitionnames;
- __isset = other1008.__isset;
+GetLatestCommittedCompactionInfoRequest&
GetLatestCommittedCompactionInfoRequest::operator=(const
GetLatestCommittedCompactionInfoRequest& other1001) {
+ dbname = other1001.dbname;
+ tablename = other1001.tablename;
+ partitionnames = other1001.partitionnames;
+ lastCompactionId = other1001.lastCompactionId;
+ __isset = other1001.__isset;
return *this;
}
void GetLatestCommittedCompactionInfoRequest::printTo(std::ostream& out) const
{
@@ -28289,6 +28310,7 @@ void
GetLatestCommittedCompactionInfoRequest::printTo(std::ostream& out) const {
out << "dbname=" << to_string(dbname);
out << ", " << "tablename=" << to_string(tablename);
out << ", " << "partitionnames="; (__isset.partitionnames ? (out <<
to_string(partitionnames)) : (out << "<null>"));
+ out << ", " << "lastCompactionId="; (__isset.lastCompactionId ? (out <<
to_string(lastCompactionId)) : (out << "<null>"));
out << ")";
}
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
index d383d56..1e05408 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-cpp/hive_metastore_types.h
@@ -10640,8 +10640,9 @@ void swap(ShowCompactResponse &a, ShowCompactResponse
&b);
std::ostream& operator<<(std::ostream& out, const ShowCompactResponse& obj);
typedef struct _GetLatestCommittedCompactionInfoRequest__isset {
- _GetLatestCommittedCompactionInfoRequest__isset() : partitionnames(false) {}
+ _GetLatestCommittedCompactionInfoRequest__isset() : partitionnames(false),
lastCompactionId(false) {}
bool partitionnames :1;
+ bool lastCompactionId :1;
} _GetLatestCommittedCompactionInfoRequest__isset;
class GetLatestCommittedCompactionInfoRequest : public virtual
::apache::thrift::TBase {
@@ -10649,13 +10650,14 @@ class GetLatestCommittedCompactionInfoRequest :
public virtual ::apache::thrift:
GetLatestCommittedCompactionInfoRequest(const
GetLatestCommittedCompactionInfoRequest&);
GetLatestCommittedCompactionInfoRequest& operator=(const
GetLatestCommittedCompactionInfoRequest&);
- GetLatestCommittedCompactionInfoRequest() : dbname(), tablename() {
+ GetLatestCommittedCompactionInfoRequest() : dbname(), tablename(),
lastCompactionId(0) {
}
virtual ~GetLatestCommittedCompactionInfoRequest() noexcept;
std::string dbname;
std::string tablename;
std::vector<std::string> partitionnames;
+ int64_t lastCompactionId;
_GetLatestCommittedCompactionInfoRequest__isset __isset;
@@ -10665,6 +10667,8 @@ class GetLatestCommittedCompactionInfoRequest : public
virtual ::apache::thrift:
void __set_partitionnames(const std::vector<std::string> & val);
+ void __set_lastCompactionId(const int64_t val);
+
bool operator == (const GetLatestCommittedCompactionInfoRequest & rhs) const
{
if (!(dbname == rhs.dbname))
@@ -10675,6 +10679,10 @@ class GetLatestCommittedCompactionInfoRequest : public
virtual ::apache::thrift:
return false;
else if (__isset.partitionnames && !(partitionnames == rhs.partitionnames))
return false;
+ if (__isset.lastCompactionId != rhs.__isset.lastCompactionId)
+ return false;
+ else if (__isset.lastCompactionId && !(lastCompactionId ==
rhs.lastCompactionId))
+ return false;
return true;
}
bool operator != (const GetLatestCommittedCompactionInfoRequest &rhs) const {
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java
index a3e71e5..09fc08a 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/GetLatestCommittedCompactionInfoRequest.java
@@ -14,6 +14,7 @@ package org.apache.hadoop.hive.metastore.api;
private static final org.apache.thrift.protocol.TField DBNAME_FIELD_DESC =
new org.apache.thrift.protocol.TField("dbname",
org.apache.thrift.protocol.TType.STRING, (short)1);
private static final org.apache.thrift.protocol.TField TABLENAME_FIELD_DESC
= new org.apache.thrift.protocol.TField("tablename",
org.apache.thrift.protocol.TType.STRING, (short)2);
private static final org.apache.thrift.protocol.TField
PARTITIONNAMES_FIELD_DESC = new
org.apache.thrift.protocol.TField("partitionnames",
org.apache.thrift.protocol.TType.LIST, (short)3);
+ private static final org.apache.thrift.protocol.TField
LAST_COMPACTION_ID_FIELD_DESC = new
org.apache.thrift.protocol.TField("lastCompactionId",
org.apache.thrift.protocol.TType.I64, (short)4);
private static final org.apache.thrift.scheme.SchemeFactory
STANDARD_SCHEME_FACTORY = new
GetLatestCommittedCompactionInfoRequestStandardSchemeFactory();
private static final org.apache.thrift.scheme.SchemeFactory
TUPLE_SCHEME_FACTORY = new
GetLatestCommittedCompactionInfoRequestTupleSchemeFactory();
@@ -21,12 +22,14 @@ package org.apache.hadoop.hive.metastore.api;
private @org.apache.thrift.annotation.Nullable java.lang.String dbname; //
required
private @org.apache.thrift.annotation.Nullable java.lang.String tablename;
// required
private @org.apache.thrift.annotation.Nullable
java.util.List<java.lang.String> partitionnames; // optional
+ private long lastCompactionId; // optional
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
DBNAME((short)1, "dbname"),
TABLENAME((short)2, "tablename"),
- PARTITIONNAMES((short)3, "partitionnames");
+ PARTITIONNAMES((short)3, "partitionnames"),
+ LAST_COMPACTION_ID((short)4, "lastCompactionId");
private static final java.util.Map<java.lang.String, _Fields> byName = new
java.util.HashMap<java.lang.String, _Fields>();
@@ -48,6 +51,8 @@ package org.apache.hadoop.hive.metastore.api;
return TABLENAME;
case 3: // PARTITIONNAMES
return PARTITIONNAMES;
+ case 4: // LAST_COMPACTION_ID
+ return LAST_COMPACTION_ID;
default:
return null;
}
@@ -89,7 +94,9 @@ package org.apache.hadoop.hive.metastore.api;
}
// isset id assignments
- private static final _Fields optionals[] = {_Fields.PARTITIONNAMES};
+ private static final int __LASTCOMPACTIONID_ISSET_ID = 0;
+ private byte __isset_bitfield = 0;
+ private static final _Fields optionals[] =
{_Fields.PARTITIONNAMES,_Fields.LAST_COMPACTION_ID};
public static final java.util.Map<_Fields,
org.apache.thrift.meta_data.FieldMetaData> metaDataMap;
static {
java.util.Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap =
new java.util.EnumMap<_Fields,
org.apache.thrift.meta_data.FieldMetaData>(_Fields.class);
@@ -100,6 +107,8 @@ package org.apache.hadoop.hive.metastore.api;
tmpMap.put(_Fields.PARTITIONNAMES, new
org.apache.thrift.meta_data.FieldMetaData("partitionnames",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
new
org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
+ tmpMap.put(_Fields.LAST_COMPACTION_ID, new
org.apache.thrift.meta_data.FieldMetaData("lastCompactionId",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(GetLatestCommittedCompactionInfoRequest.class,
metaDataMap);
}
@@ -120,6 +129,7 @@ package org.apache.hadoop.hive.metastore.api;
* Performs a deep copy on <i>other</i>.
*/
public
GetLatestCommittedCompactionInfoRequest(GetLatestCommittedCompactionInfoRequest
other) {
+ __isset_bitfield = other.__isset_bitfield;
if (other.isSetDbname()) {
this.dbname = other.dbname;
}
@@ -130,6 +140,7 @@ package org.apache.hadoop.hive.metastore.api;
java.util.List<java.lang.String> __this__partitionnames = new
java.util.ArrayList<java.lang.String>(other.partitionnames);
this.partitionnames = __this__partitionnames;
}
+ this.lastCompactionId = other.lastCompactionId;
}
public GetLatestCommittedCompactionInfoRequest deepCopy() {
@@ -141,6 +152,8 @@ package org.apache.hadoop.hive.metastore.api;
this.dbname = null;
this.tablename = null;
this.partitionnames = null;
+ setLastCompactionIdIsSet(false);
+ this.lastCompactionId = 0;
}
@org.apache.thrift.annotation.Nullable
@@ -231,6 +244,28 @@ package org.apache.hadoop.hive.metastore.api;
}
}
+ public long getLastCompactionId() {
+ return this.lastCompactionId;
+ }
+
+ public void setLastCompactionId(long lastCompactionId) {
+ this.lastCompactionId = lastCompactionId;
+ setLastCompactionIdIsSet(true);
+ }
+
+ public void unsetLastCompactionId() {
+ __isset_bitfield =
org.apache.thrift.EncodingUtils.clearBit(__isset_bitfield,
__LASTCOMPACTIONID_ISSET_ID);
+ }
+
+ /** Returns true if field lastCompactionId is set (has been assigned a
value) and false otherwise */
+ public boolean isSetLastCompactionId() {
+ return org.apache.thrift.EncodingUtils.testBit(__isset_bitfield,
__LASTCOMPACTIONID_ISSET_ID);
+ }
+
+ public void setLastCompactionIdIsSet(boolean value) {
+ __isset_bitfield =
org.apache.thrift.EncodingUtils.setBit(__isset_bitfield,
__LASTCOMPACTIONID_ISSET_ID, value);
+ }
+
public void setFieldValue(_Fields field,
@org.apache.thrift.annotation.Nullable java.lang.Object value) {
switch (field) {
case DBNAME:
@@ -257,6 +292,14 @@ package org.apache.hadoop.hive.metastore.api;
}
break;
+ case LAST_COMPACTION_ID:
+ if (value == null) {
+ unsetLastCompactionId();
+ } else {
+ setLastCompactionId((java.lang.Long)value);
+ }
+ break;
+
}
}
@@ -272,6 +315,9 @@ package org.apache.hadoop.hive.metastore.api;
case PARTITIONNAMES:
return getPartitionnames();
+ case LAST_COMPACTION_ID:
+ return getLastCompactionId();
+
}
throw new java.lang.IllegalStateException();
}
@@ -289,6 +335,8 @@ package org.apache.hadoop.hive.metastore.api;
return isSetTablename();
case PARTITIONNAMES:
return isSetPartitionnames();
+ case LAST_COMPACTION_ID:
+ return isSetLastCompactionId();
}
throw new java.lang.IllegalStateException();
}
@@ -333,6 +381,15 @@ package org.apache.hadoop.hive.metastore.api;
return false;
}
+ boolean this_present_lastCompactionId = true &&
this.isSetLastCompactionId();
+ boolean that_present_lastCompactionId = true &&
that.isSetLastCompactionId();
+ if (this_present_lastCompactionId || that_present_lastCompactionId) {
+ if (!(this_present_lastCompactionId && that_present_lastCompactionId))
+ return false;
+ if (this.lastCompactionId != that.lastCompactionId)
+ return false;
+ }
+
return true;
}
@@ -352,6 +409,10 @@ package org.apache.hadoop.hive.metastore.api;
if (isSetPartitionnames())
hashCode = hashCode * 8191 + partitionnames.hashCode();
+ hashCode = hashCode * 8191 + ((isSetLastCompactionId()) ? 131071 : 524287);
+ if (isSetLastCompactionId())
+ hashCode = hashCode * 8191 +
org.apache.thrift.TBaseHelper.hashCode(lastCompactionId);
+
return hashCode;
}
@@ -393,6 +454,16 @@ package org.apache.hadoop.hive.metastore.api;
return lastComparison;
}
}
+ lastComparison = java.lang.Boolean.compare(isSetLastCompactionId(),
other.isSetLastCompactionId());
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ if (isSetLastCompactionId()) {
+ lastComparison =
org.apache.thrift.TBaseHelper.compareTo(this.lastCompactionId,
other.lastCompactionId);
+ if (lastComparison != 0) {
+ return lastComparison;
+ }
+ }
return 0;
}
@@ -439,6 +510,12 @@ package org.apache.hadoop.hive.metastore.api;
}
first = false;
}
+ if (isSetLastCompactionId()) {
+ if (!first) sb.append(", ");
+ sb.append("lastCompactionId:");
+ sb.append(this.lastCompactionId);
+ first = false;
+ }
sb.append(")");
return sb.toString();
}
@@ -466,6 +543,8 @@ package org.apache.hadoop.hive.metastore.api;
private void readObject(java.io.ObjectInputStream in) throws
java.io.IOException, java.lang.ClassNotFoundException {
try {
+ // it doesn't seem like you should have to do this, but java
serialization is wacky, and doesn't call the default constructor.
+ __isset_bitfield = 0;
read(new org.apache.thrift.protocol.TCompactProtocol(new
org.apache.thrift.transport.TIOStreamTransport(in)));
} catch (org.apache.thrift.TException te) {
throw new java.io.IOException(te);
@@ -524,6 +603,14 @@ package org.apache.hadoop.hive.metastore.api;
org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
}
break;
+ case 4: // LAST_COMPACTION_ID
+ if (schemeField.type == org.apache.thrift.protocol.TType.I64) {
+ struct.lastCompactionId = iprot.readI64();
+ struct.setLastCompactionIdIsSet(true);
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
+ }
+ break;
default:
org.apache.thrift.protocol.TProtocolUtil.skip(iprot,
schemeField.type);
}
@@ -561,6 +648,11 @@ package org.apache.hadoop.hive.metastore.api;
oprot.writeFieldEnd();
}
}
+ if (struct.isSetLastCompactionId()) {
+ oprot.writeFieldBegin(LAST_COMPACTION_ID_FIELD_DESC);
+ oprot.writeI64(struct.lastCompactionId);
+ oprot.writeFieldEnd();
+ }
oprot.writeFieldStop();
oprot.writeStructEnd();
}
@@ -584,7 +676,10 @@ package org.apache.hadoop.hive.metastore.api;
if (struct.isSetPartitionnames()) {
optionals.set(0);
}
- oprot.writeBitSet(optionals, 1);
+ if (struct.isSetLastCompactionId()) {
+ optionals.set(1);
+ }
+ oprot.writeBitSet(optionals, 2);
if (struct.isSetPartitionnames()) {
{
oprot.writeI32(struct.partitionnames.size());
@@ -594,6 +689,9 @@ package org.apache.hadoop.hive.metastore.api;
}
}
}
+ if (struct.isSetLastCompactionId()) {
+ oprot.writeI64(struct.lastCompactionId);
+ }
}
@Override
@@ -603,7 +701,7 @@ package org.apache.hadoop.hive.metastore.api;
struct.setDbnameIsSet(true);
struct.tablename = iprot.readString();
struct.setTablenameIsSet(true);
- java.util.BitSet incoming = iprot.readBitSet(1);
+ java.util.BitSet incoming = iprot.readBitSet(2);
if (incoming.get(0)) {
{
org.apache.thrift.protocol.TList _list877 =
iprot.readListBegin(org.apache.thrift.protocol.TType.STRING);
@@ -617,6 +715,10 @@ package org.apache.hadoop.hive.metastore.api;
}
struct.setPartitionnamesIsSet(true);
}
+ if (incoming.get(1)) {
+ struct.lastCompactionId = iprot.readI64();
+ struct.setLastCompactionIdIsSet(true);
+ }
}
}
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php
index 2bbeda9..5eb0b47 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/GetLatestCommittedCompactionInfoRequest.php
@@ -40,6 +40,11 @@ class GetLatestCommittedCompactionInfoRequest
'type' => TType::STRING,
),
),
+ 4 => array(
+ 'var' => 'lastCompactionId',
+ 'isRequired' => false,
+ 'type' => TType::I64,
+ ),
);
/**
@@ -54,6 +59,10 @@ class GetLatestCommittedCompactionInfoRequest
* @var string[]
*/
public $partitionnames = null;
+ /**
+ * @var int
+ */
+ public $lastCompactionId = null;
public function __construct($vals = null)
{
@@ -67,6 +76,9 @@ class GetLatestCommittedCompactionInfoRequest
if (isset($vals['partitionnames'])) {
$this->partitionnames = $vals['partitionnames'];
}
+ if (isset($vals['lastCompactionId'])) {
+ $this->lastCompactionId = $vals['lastCompactionId'];
+ }
}
}
@@ -119,6 +131,13 @@ class GetLatestCommittedCompactionInfoRequest
$xfer += $input->skip($ftype);
}
break;
+ case 4:
+ if ($ftype == TType::I64) {
+ $xfer += $input->readI64($this->lastCompactionId);
+ } else {
+ $xfer += $input->skip($ftype);
+ }
+ break;
default:
$xfer += $input->skip($ftype);
break;
@@ -155,6 +174,11 @@ class GetLatestCommittedCompactionInfoRequest
$output->writeListEnd();
$xfer += $output->writeFieldEnd();
}
+ if ($this->lastCompactionId !== null) {
+ $xfer += $output->writeFieldBegin('lastCompactionId', TType::I64,
4);
+ $xfer += $output->writeI64($this->lastCompactionId);
+ $xfer += $output->writeFieldEnd();
+ }
$xfer += $output->writeFieldStop();
$xfer += $output->writeStructEnd();
return $xfer;
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
index 298b6ed..c4d1fc7 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py
@@ -16100,14 +16100,16 @@ class GetLatestCommittedCompactionInfoRequest(object):
- dbname
- tablename
- partitionnames
+ - lastCompactionId
"""
- def __init__(self, dbname=None, tablename=None, partitionnames=None,):
+ def __init__(self, dbname=None, tablename=None, partitionnames=None,
lastCompactionId=None,):
self.dbname = dbname
self.tablename = tablename
self.partitionnames = partitionnames
+ self.lastCompactionId = lastCompactionId
def read(self, iprot):
if iprot._fast_decode is not None and isinstance(iprot.trans,
TTransport.CReadableTransport) and self.thrift_spec is not None:
@@ -16138,6 +16140,11 @@ class GetLatestCommittedCompactionInfoRequest(object):
iprot.readListEnd()
else:
iprot.skip(ftype)
+ elif fid == 4:
+ if ftype == TType.I64:
+ self.lastCompactionId = iprot.readI64()
+ else:
+ iprot.skip(ftype)
else:
iprot.skip(ftype)
iprot.readFieldEnd()
@@ -16163,6 +16170,10 @@ class GetLatestCommittedCompactionInfoRequest(object):
oprot.writeString(iter773.encode('utf-8') if
sys.version_info[0] == 2 else iter773)
oprot.writeListEnd()
oprot.writeFieldEnd()
+ if self.lastCompactionId is not None:
+ oprot.writeFieldBegin('lastCompactionId', TType.I64, 4)
+ oprot.writeI64(self.lastCompactionId)
+ oprot.writeFieldEnd()
oprot.writeFieldStop()
oprot.writeStructEnd()
@@ -30484,6 +30495,7 @@ GetLatestCommittedCompactionInfoRequest.thrift_spec = (
(1, TType.STRING, 'dbname', 'UTF8', None, ), # 1
(2, TType.STRING, 'tablename', 'UTF8', None, ), # 2
(3, TType.LIST, 'partitionnames', (TType.STRING, 'UTF8', False), None, ),
# 3
+ (4, TType.I64, 'lastCompactionId', None, None, ), # 4
)
all_structs.append(GetLatestCommittedCompactionInfoResponse)
GetLatestCommittedCompactionInfoResponse.thrift_spec = (
diff --git
a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
index 5c78f49..f73250d 100644
---
a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
+++
b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb
@@ -4644,11 +4644,13 @@ class GetLatestCommittedCompactionInfoRequest
DBNAME = 1
TABLENAME = 2
PARTITIONNAMES = 3
+ LASTCOMPACTIONID = 4
FIELDS = {
DBNAME => {:type => ::Thrift::Types::STRING, :name => 'dbname'},
TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tablename'},
- PARTITIONNAMES => {:type => ::Thrift::Types::LIST, :name =>
'partitionnames', :element => {:type => ::Thrift::Types::STRING}, :optional =>
true}
+ PARTITIONNAMES => {:type => ::Thrift::Types::LIST, :name =>
'partitionnames', :element => {:type => ::Thrift::Types::STRING}, :optional =>
true},
+ LASTCOMPACTIONID => {:type => ::Thrift::Types::I64, :name =>
'lastCompactionId', :optional => true}
}
def struct_fields; FIELDS; end
diff --git
a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
index 20259cc..9edc43a 100644
---
a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
+++
b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift
@@ -1342,6 +1342,7 @@ struct GetLatestCommittedCompactionInfoRequest {
1: required string dbname,
2: required string tablename,
3: optional list<string> partitionnames,
+ 4: optional i64 lastCompactionId,
}
struct GetLatestCommittedCompactionInfoResponse {
diff --git
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
index caa3ae2..84a059c 100644
---
a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
+++
b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java
@@ -4011,7 +4011,7 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
.append(" FROM \"COMPACTION_QUEUE\"")
.append(" WHERE \"CQ_STATE\" = " +
quoteChar(READY_FOR_CLEANING))
.append(") AS compactions ")
- .append(" WHERE \"CC_DATABASE\"=? AND \"CC_TABLE\"=?");
+ .append(" WHERE \"CC_DATABASE\" = ? AND \"CC_TABLE\" = ?");
params.add(rqst.getDbname());
params.add(rqst.getTablename());
if (rqst.getPartitionnamesSize() > 0) {
@@ -4020,8 +4020,10 @@ abstract class TxnHandler implements TxnStore,
TxnStore.MutexAPI {
Collections.nCopies(rqst.getPartitionnamesSize(), "?")));
sb.append(")");
params.addAll(rqst.getPartitionnames());
- } else {
- sb.append(" AND \"CC_PARTITION\" IS NULL");
+ }
+ if (rqst.isSetLastCompactionId()) {
+ sb.append(" AND \"CC_ID\" > ?");
+ params.add(String.valueOf(rqst.getLastCompactionId()));
}
sb.append(" ORDER BY \"CC_ID\" DESC");
