Repository: hive Updated Branches: refs/heads/master dcc895016 -> f713140ba
HIVE-21036 extend OpenTxnRequest with transaction type (Igor Kryvenko via Eugene Koifman) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f713140b Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f713140b Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f713140b Branch: refs/heads/master Commit: f713140ba60d50017d2b97b1f5c5aea7217512dd Parents: dcc8950 Author: Igor Kryvenko <kryvenko7i...@gmail.com> Authored: Thu Jan 10 15:50:41 2019 -0800 Committer: Eugene Koifman <ekoif...@apache.org> Committed: Thu Jan 10 15:50:41 2019 -0800 ---------------------------------------------------------------------- .../hadoop/hive/ql/txn/compactor/Worker.java | 3 +- .../hive/metastore/api/OpenTxnRequest.java | 129 ++++++++++++++++++- .../hadoop/hive/metastore/api/TxnType.java | 51 ++++++++ .../src/gen/thrift/gen-php/metastore/Types.php | 36 ++++++ .../gen/thrift/gen-py/hive_metastore/ttypes.py | 35 ++++- .../gen/thrift/gen-rb/hive_metastore_types.rb | 16 ++- .../hive/metastore/HiveMetaStoreClient.java | 17 ++- .../hadoop/hive/metastore/IMetaStoreClient.java | 10 ++ .../src/main/thrift/hive_metastore.thrift | 8 ++ .../hadoop/hive/metastore/txn/TxnHandler.java | 13 -- .../HiveMetaStoreClientPreCatalog.java | 15 ++- .../hive/metastore/TestHiveMetaStoreTxns.java | 9 ++ 12 files changed, 315 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java index b645870..e3fab69 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/Worker.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Partition; import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TxnType; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.mapred.JobConf; import org.apache.thrift.TException; @@ -155,7 +156,7 @@ public class Worker extends RemoteCompactorThread implements MetaStoreThread { if (ci.runAs == null) { ci.runAs = findUserToRunAs(sd.getLocation(), t); } - long compactorTxnId = msc.openTxns(ci.runAs, 1).getTxn_ids().get(0); + long compactorTxnId = msc.openTxn(ci.runAs, TxnType.COMPACTION); heartbeater = new CompactionHeartbeater(compactorTxnId, fullTableName, conf); heartbeater.start(); http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java index 19b2c01..b5efffd 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/OpenTxnRequest.java @@ -44,6 +44,7 @@ import org.slf4j.LoggerFactory; private static final org.apache.thrift.protocol.TField AGENT_INFO_FIELD_DESC = new org.apache.thrift.protocol.TField("agentInfo", org.apache.thrift.protocol.TType.STRING, (short)4); private static final org.apache.thrift.protocol.TField REPL_POLICY_FIELD_DESC = new org.apache.thrift.protocol.TField("replPolicy", org.apache.thrift.protocol.TType.STRING, (short)5); private static final org.apache.thrift.protocol.TField REPL_SRC_TXN_IDS_FIELD_DESC = new org.apache.thrift.protocol.TField("replSrcTxnIds", org.apache.thrift.protocol.TType.LIST, (short)6); + private static final org.apache.thrift.protocol.TField TXN_TYPE_FIELD_DESC = new org.apache.thrift.protocol.TField("txn_type", org.apache.thrift.protocol.TType.I32, (short)7); private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>(); static { @@ -57,6 +58,7 @@ import org.slf4j.LoggerFactory; private String agentInfo; // optional private String replPolicy; // optional private List<Long> replSrcTxnIds; // optional + private TxnType txn_type; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -65,7 +67,12 @@ import org.slf4j.LoggerFactory; HOSTNAME((short)3, "hostname"), AGENT_INFO((short)4, "agentInfo"), REPL_POLICY((short)5, "replPolicy"), - REPL_SRC_TXN_IDS((short)6, "replSrcTxnIds"); + REPL_SRC_TXN_IDS((short)6, "replSrcTxnIds"), + /** + * + * @see TxnType + */ + TXN_TYPE((short)7, "txn_type"); private static final Map<String, _Fields> byName = new HashMap<String, _Fields>(); @@ -92,6 +99,8 @@ import org.slf4j.LoggerFactory; return REPL_POLICY; case 6: // REPL_SRC_TXN_IDS return REPL_SRC_TXN_IDS; + case 7: // TXN_TYPE + return TXN_TYPE; default: return null; } @@ -134,7 +143,7 @@ import org.slf4j.LoggerFactory; // isset id assignments private static final int __NUM_TXNS_ISSET_ID = 0; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.AGENT_INFO,_Fields.REPL_POLICY,_Fields.REPL_SRC_TXN_IDS}; + private static final _Fields optionals[] = {_Fields.AGENT_INFO,_Fields.REPL_POLICY,_Fields.REPL_SRC_TXN_IDS,_Fields.TXN_TYPE}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -151,6 +160,8 @@ import org.slf4j.LoggerFactory; tmpMap.put(_Fields.REPL_SRC_TXN_IDS, new org.apache.thrift.meta_data.FieldMetaData("replSrcTxnIds", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.I64)))); + tmpMap.put(_Fields.TXN_TYPE, new org.apache.thrift.meta_data.FieldMetaData("txn_type", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, TxnType.class))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(OpenTxnRequest.class, metaDataMap); } @@ -158,6 +169,8 @@ import org.slf4j.LoggerFactory; public OpenTxnRequest() { this.agentInfo = "Unknown"; + this.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.DEFAULT; + } public OpenTxnRequest( @@ -194,6 +207,9 @@ import org.slf4j.LoggerFactory; List<Long> __this__replSrcTxnIds = new ArrayList<Long>(other.replSrcTxnIds); this.replSrcTxnIds = __this__replSrcTxnIds; } + if (other.isSetTxn_type()) { + this.txn_type = other.txn_type; + } } public OpenTxnRequest deepCopy() { @@ -210,6 +226,8 @@ import org.slf4j.LoggerFactory; this.replPolicy = null; this.replSrcTxnIds = null; + this.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.DEFAULT; + } public int getNum_txns() { @@ -364,6 +382,37 @@ import org.slf4j.LoggerFactory; } } + /** + * + * @see TxnType + */ + public TxnType getTxn_type() { + return this.txn_type; + } + + /** + * + * @see TxnType + */ + public void setTxn_type(TxnType txn_type) { + this.txn_type = txn_type; + } + + public void unsetTxn_type() { + this.txn_type = null; + } + + /** Returns true if field txn_type is set (has been assigned a value) and false otherwise */ + public boolean isSetTxn_type() { + return this.txn_type != null; + } + + public void setTxn_typeIsSet(boolean value) { + if (!value) { + this.txn_type = null; + } + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case NUM_TXNS: @@ -414,6 +463,14 @@ import org.slf4j.LoggerFactory; } break; + case TXN_TYPE: + if (value == null) { + unsetTxn_type(); + } else { + setTxn_type((TxnType)value); + } + break; + } } @@ -437,6 +494,9 @@ import org.slf4j.LoggerFactory; case REPL_SRC_TXN_IDS: return getReplSrcTxnIds(); + case TXN_TYPE: + return getTxn_type(); + } throw new IllegalStateException(); } @@ -460,6 +520,8 @@ import org.slf4j.LoggerFactory; return isSetReplPolicy(); case REPL_SRC_TXN_IDS: return isSetReplSrcTxnIds(); + case TXN_TYPE: + return isSetTxn_type(); } throw new IllegalStateException(); } @@ -531,6 +593,15 @@ import org.slf4j.LoggerFactory; return false; } + boolean this_present_txn_type = true && this.isSetTxn_type(); + boolean that_present_txn_type = true && that.isSetTxn_type(); + if (this_present_txn_type || that_present_txn_type) { + if (!(this_present_txn_type && that_present_txn_type)) + return false; + if (!this.txn_type.equals(that.txn_type)) + return false; + } + return true; } @@ -568,6 +639,11 @@ import org.slf4j.LoggerFactory; if (present_replSrcTxnIds) list.add(replSrcTxnIds); + boolean present_txn_type = true && (isSetTxn_type()); + list.add(present_txn_type); + if (present_txn_type) + list.add(txn_type.getValue()); + return list.hashCode(); } @@ -639,6 +715,16 @@ import org.slf4j.LoggerFactory; return lastComparison; } } + lastComparison = Boolean.valueOf(isSetTxn_type()).compareTo(other.isSetTxn_type()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetTxn_type()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.txn_type, other.txn_type); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -708,6 +794,16 @@ import org.slf4j.LoggerFactory; } first = false; } + if (isSetTxn_type()) { + if (!first) sb.append(", "); + sb.append("txn_type:"); + if (this.txn_type == null) { + sb.append("null"); + } else { + sb.append(this.txn_type); + } + first = false; + } sb.append(")"); return sb.toString(); } @@ -823,6 +919,14 @@ import org.slf4j.LoggerFactory; org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 7: // TXN_TYPE + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.findByValue(iprot.readI32()); + struct.setTxn_typeIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -877,6 +981,13 @@ import org.slf4j.LoggerFactory; oprot.writeFieldEnd(); } } + if (struct.txn_type != null) { + if (struct.isSetTxn_type()) { + oprot.writeFieldBegin(TXN_TYPE_FIELD_DESC); + oprot.writeI32(struct.txn_type.getValue()); + oprot.writeFieldEnd(); + } + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -907,7 +1018,10 @@ import org.slf4j.LoggerFactory; if (struct.isSetReplSrcTxnIds()) { optionals.set(2); } - oprot.writeBitSet(optionals, 3); + if (struct.isSetTxn_type()) { + optionals.set(3); + } + oprot.writeBitSet(optionals, 4); if (struct.isSetAgentInfo()) { oprot.writeString(struct.agentInfo); } @@ -923,6 +1037,9 @@ import org.slf4j.LoggerFactory; } } } + if (struct.isSetTxn_type()) { + oprot.writeI32(struct.txn_type.getValue()); + } } @Override @@ -934,7 +1051,7 @@ import org.slf4j.LoggerFactory; struct.setUserIsSet(true); struct.hostname = iprot.readString(); struct.setHostnameIsSet(true); - BitSet incoming = iprot.readBitSet(3); + BitSet incoming = iprot.readBitSet(4); if (incoming.get(0)) { struct.agentInfo = iprot.readString(); struct.setAgentInfoIsSet(true); @@ -956,6 +1073,10 @@ import org.slf4j.LoggerFactory; } struct.setReplSrcTxnIdsIsSet(true); } + if (incoming.get(3)) { + struct.txn_type = org.apache.hadoop.hive.metastore.api.TxnType.findByValue(iprot.readI32()); + struct.setTxn_typeIsSet(true); + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java new file mode 100644 index 0000000..349f37f --- /dev/null +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/TxnType.java @@ -0,0 +1,51 @@ +/** + * Autogenerated by Thrift Compiler (0.9.3) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.metastore.api; + + +import java.util.Map; +import java.util.HashMap; +import org.apache.thrift.TEnum; + +public enum TxnType implements org.apache.thrift.TEnum { + DEFAULT(0), + REPL_CREATED(1), + READ_ONLY(2), + COMPACTION(3); + + private final int value; + + private TxnType(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + public static TxnType findByValue(int value) { + switch (value) { + case 0: + return DEFAULT; + case 1: + return REPL_CREATED; + case 2: + return READ_ONLY; + case 3: + return COMPACTION; + default: + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php index 2953a8f..13e287e 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-php/metastore/Types.php @@ -221,6 +221,19 @@ final class ResourceType { ); } +final class TxnType { + const DEFAULT = 0; + const REPL_CREATED = 1; + const READ_ONLY = 2; + const COMPACTION = 3; + static public $__names = array( + 0 => 'DEFAULT', + 1 => 'REPL_CREATED', + 2 => 'READ_ONLY', + 3 => 'COMPACTION', + ); +} + final class FileMetadataExprType { const ORC_SARG = 1; static public $__names = array( @@ -16778,6 +16791,10 @@ class OpenTxnRequest { * @var int[] */ public $replSrcTxnIds = null; + /** + * @var int + */ + public $txn_type = 0; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -16810,6 +16827,10 @@ class OpenTxnRequest { 'type' => TType::I64, ), ), + 7 => array( + 'var' => 'txn_type', + 'type' => TType::I32, + ), ); } if (is_array($vals)) { @@ -16831,6 +16852,9 @@ class OpenTxnRequest { if (isset($vals['replSrcTxnIds'])) { $this->replSrcTxnIds = $vals['replSrcTxnIds']; } + if (isset($vals['txn_type'])) { + $this->txn_type = $vals['txn_type']; + } } } @@ -16905,6 +16929,13 @@ class OpenTxnRequest { $xfer += $input->skip($ftype); } break; + case 7: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->txn_type); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -16960,6 +16991,11 @@ class OpenTxnRequest { } $xfer += $output->writeFieldEnd(); } + if ($this->txn_type !== null) { + $xfer += $output->writeFieldBegin('txn_type', TType::I32, 7); + $xfer += $output->writeI32($this->txn_type); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py index f8b8628..8f149d1 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -333,6 +333,26 @@ class ResourceType: "ARCHIVE": 3, } +class TxnType: + DEFAULT = 0 + REPL_CREATED = 1 + READ_ONLY = 2 + COMPACTION = 3 + + _VALUES_TO_NAMES = { + 0: "DEFAULT", + 1: "REPL_CREATED", + 2: "READ_ONLY", + 3: "COMPACTION", + } + + _NAMES_TO_VALUES = { + "DEFAULT": 0, + "REPL_CREATED": 1, + "READ_ONLY": 2, + "COMPACTION": 3, + } + class FileMetadataExprType: ORC_SARG = 1 @@ -11672,6 +11692,7 @@ class OpenTxnRequest: - agentInfo - replPolicy - replSrcTxnIds + - txn_type """ thrift_spec = ( @@ -11682,15 +11703,17 @@ class OpenTxnRequest: (4, TType.STRING, 'agentInfo', None, "Unknown", ), # 4 (5, TType.STRING, 'replPolicy', None, None, ), # 5 (6, TType.LIST, 'replSrcTxnIds', (TType.I64,None), None, ), # 6 + (7, TType.I32, 'txn_type', None, 0, ), # 7 ) - def __init__(self, num_txns=None, user=None, hostname=None, agentInfo=thrift_spec[4][4], replPolicy=None, replSrcTxnIds=None,): + def __init__(self, num_txns=None, user=None, hostname=None, agentInfo=thrift_spec[4][4], replPolicy=None, replSrcTxnIds=None, txn_type=thrift_spec[7][4],): self.num_txns = num_txns self.user = user self.hostname = hostname self.agentInfo = agentInfo self.replPolicy = replPolicy self.replSrcTxnIds = replSrcTxnIds + self.txn_type = txn_type def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -11736,6 +11759,11 @@ class OpenTxnRequest: iprot.readListEnd() else: iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I32: + self.txn_type = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -11773,6 +11801,10 @@ class OpenTxnRequest: oprot.writeI64(iter515) oprot.writeListEnd() oprot.writeFieldEnd() + if self.txn_type is not None: + oprot.writeFieldBegin('txn_type', TType.I32, 7) + oprot.writeI32(self.txn_type) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -11794,6 +11826,7 @@ class OpenTxnRequest: value = (value * 31) ^ hash(self.agentInfo) value = (value * 31) ^ hash(self.replPolicy) value = (value * 31) ^ hash(self.replSrcTxnIds) + value = (value * 31) ^ hash(self.txn_type) return value def __repr__(self): http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb index dfc5d7b..9e5f086 100644 --- a/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -155,6 +155,15 @@ module ResourceType VALID_VALUES = Set.new([JAR, FILE, ARCHIVE]).freeze end +module TxnType + DEFAULT = 0 + REPL_CREATED = 1 + READ_ONLY = 2 + COMPACTION = 3 + VALUE_MAP = {0 => "DEFAULT", 1 => "REPL_CREATED", 2 => "READ_ONLY", 3 => "COMPACTION"} + VALID_VALUES = Set.new([DEFAULT, REPL_CREATED, READ_ONLY, COMPACTION]).freeze +end + module FileMetadataExprType ORC_SARG = 1 VALUE_MAP = {1 => "ORC_SARG"} @@ -2604,6 +2613,7 @@ class OpenTxnRequest AGENTINFO = 4 REPLPOLICY = 5 REPLSRCTXNIDS = 6 + TXN_TYPE = 7 FIELDS = { NUM_TXNS => {:type => ::Thrift::Types::I32, :name => 'num_txns'}, @@ -2611,7 +2621,8 @@ class OpenTxnRequest HOSTNAME => {:type => ::Thrift::Types::STRING, :name => 'hostname'}, AGENTINFO => {:type => ::Thrift::Types::STRING, :name => 'agentInfo', :default => %q"Unknown", :optional => true}, REPLPOLICY => {:type => ::Thrift::Types::STRING, :name => 'replPolicy', :optional => true}, - REPLSRCTXNIDS => {:type => ::Thrift::Types::LIST, :name => 'replSrcTxnIds', :element => {:type => ::Thrift::Types::I64}, :optional => true} + REPLSRCTXNIDS => {:type => ::Thrift::Types::LIST, :name => 'replSrcTxnIds', :element => {:type => ::Thrift::Types::I64}, :optional => true}, + TXN_TYPE => {:type => ::Thrift::Types::I32, :name => 'txn_type', :default => 0, :optional => true, :enum_class => ::TxnType} } def struct_fields; FIELDS; end @@ -2620,6 +2631,9 @@ class OpenTxnRequest raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field num_txns is unset!') unless @num_txns raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field user is unset!') unless @user raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Required field hostname is unset!') unless @hostname + unless @txn_type.nil? || ::TxnType::VALID_VALUES.include?(@txn_type) + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field txn_type!') + end end ::Thrift::Struct.generate_accessors self http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 748b56b..3c81f93 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -2794,7 +2794,13 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public long openTxn(String user) throws TException { - OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null); + OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, null); + return txns.getTxn_ids().get(0); + } + + @Override + public long openTxn(String user, TxnType txnType) throws TException { + OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, txnType); return txns.getTxn_ids().get(0); } @@ -2802,17 +2808,17 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { public List<Long> replOpenTxn(String replPolicy, List<Long> srcTxnIds, String user) throws TException { // As this is called from replication task, the user is the user who has fired the repl command. // This is required for standalone metastore authentication. - OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds != null ? srcTxnIds.size() : 1, replPolicy, srcTxnIds); + OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds != null ? srcTxnIds.size() : 1, replPolicy, srcTxnIds, null); return txns.getTxn_ids(); } @Override public OpenTxnsResponse openTxns(String user, int numTxns) throws TException { - return openTxnsIntr(user, numTxns, null, null); + return openTxnsIntr(user, numTxns, null, null, null); } private OpenTxnsResponse openTxnsIntr(String user, int numTxns, String replPolicy, - List<Long> srcTxnIds) throws TException { + List<Long> srcTxnIds, TxnType txnType) throws TException { String hostname; try { hostname = InetAddress.getLocalHost().getHostName(); @@ -2830,6 +2836,9 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { } else { assert srcTxnIds == null; } + if(txnType != null) { + rqst.setTxn_type(txnType); + } return client.open_txns(rqst); } http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index c96bf0d..158740d 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -116,6 +116,7 @@ import org.apache.hadoop.hive.metastore.api.TableValidWriteIds; import org.apache.hadoop.hive.metastore.api.TxnAbortedException; import org.apache.hadoop.hive.metastore.api.TxnOpenException; import org.apache.hadoop.hive.metastore.api.TxnToWriteId; +import org.apache.hadoop.hive.metastore.api.TxnType; import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownPartitionException; @@ -2856,6 +2857,15 @@ public interface IMetaStoreClient { long openTxn(String user) throws TException; /** + * Initiate a transaction with given type. + * @param user User who is opening this transaction. + * @param txnType Type of needed transaction. + * @return transaction identifier + * @throws TException + */ + long openTxn(String user, TxnType txnType) throws TException; + + /** * Initiate a transaction at the target cluster. * @param replPolicy The replication policy to uniquely identify the source cluster. * @param srcTxnIds The list of transaction ids at the source cluster http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift index 7d09a5c..9576f87 100644 --- a/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/metastore-common/src/main/thrift/hive_metastore.thrift @@ -833,6 +833,13 @@ enum ResourceType { ARCHIVE = 3, } +enum TxnType { + DEFAULT = 0, + REPL_CREATED = 1, + READ_ONLY = 2, + COMPACTION = 3 +} + struct ResourceUri { 1: ResourceType resourceType, 2: string uri, @@ -883,6 +890,7 @@ struct OpenTxnRequest { 4: optional string agentInfo = "Unknown", 5: optional string replPolicy, 6: optional list<i64> replSrcTxnIds, + 7: optional TxnType txn_type = TxnType.DEFAULT, } struct OpenTxnsResponse { http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java index 2a62903..91a9ab4 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/txn/TxnHandler.java @@ -166,19 +166,6 @@ abstract class TxnHandler implements TxnStore, TxnStore.MutexAPI { //todo: make these like OperationType and remove above char constants enum TxnStatus {OPEN, ABORTED, COMMITTED, UNKNOWN} - public enum TxnType { - DEFAULT(0), REPL_CREATED(1), READ_ONLY(2); - - private final int value; - TxnType(int value) { - this.value = value; - } - - public int getValue() { - return value; - } - } - // Lock states static final protected char LOCK_ACQUIRED = 'a'; static final protected char LOCK_WAITING = 'w'; http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index a3f5e2c..898a94d 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -2227,20 +2227,26 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } @Override + public long openTxn(String user, TxnType txnType) throws TException { + OpenTxnsResponse txns = openTxnsIntr(user, 1, null, null, txnType); + return txns.getTxn_ids().get(0); + } + + @Override public OpenTxnsResponse openTxns(String user, int numTxns) throws TException { - return openTxnsIntr(user, numTxns, null, null); + return openTxnsIntr(user, numTxns, null, null, null); } @Override public List<Long> replOpenTxn(String replPolicy, List<Long> srcTxnIds, String user) throws TException { // As this is called from replication task, the user is the user who has fired the repl command. // This is required for standalone metastore authentication. - OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds.size(), replPolicy, srcTxnIds); + OpenTxnsResponse txns = openTxnsIntr(user, srcTxnIds.size(), replPolicy, srcTxnIds, null); return txns.getTxn_ids(); } private OpenTxnsResponse openTxnsIntr(String user, int numTxns, String replPolicy, - List<Long> srcTxnIds) throws TException { + List<Long> srcTxnIds, TxnType txnType) throws TException { String hostname; try { hostname = InetAddress.getLocalHost().getHostName(); @@ -2258,6 +2264,9 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } else { assert srcTxnIds == null; } + if (txnType != null) { + rqst.setTxn_type(txnType); + } return client.open_txns(rqst); } http://git-wip-us.apache.org/repos/asf/hive/blob/f713140b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java index 2a70ec3..4cdc035 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetaStoreTxns.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hive.metastore.api.LockResponse; import org.apache.hadoop.hive.metastore.api.LockState; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.TxnType; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -298,6 +299,14 @@ public class TestHiveMetaStoreTxns { Assert.assertTrue(sawFive); } + @Test + public void testOpenTxnWithType() throws Exception { + long txnId = client.openTxn("me", TxnType.DEFAULT); + client.commitTxn(txnId); + ValidTxnList validTxns = client.getValidTxns(); + Assert.assertTrue(validTxns.isTxnValid(txnId)); + } + @Before public void setUp() throws Exception { MetaStoreTestUtils.setConfForStandloneMode(conf);