Repository: hive Updated Branches: refs/heads/master 1faadb074 -> 6e9b63e48
http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h index 4c09bc8..44a2c83 100644 --- a/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h +++ b/standalone-metastore/src/gen/thrift/gen-cpp/hive_metastore_types.h @@ -132,6 +132,16 @@ struct EventRequestType { extern const std::map<int, const char*> _EventRequestType_VALUES_TO_NAMES; +struct BucketingVersion { + enum type { + INVALID_BUCKETING = 0, + JAVA_BUCKETING = 1, + MURMUR_BUCKETING = 2 + }; +}; + +extern const std::map<int, const char*> _BucketingVersion_VALUES_TO_NAMES; + struct FunctionType { enum type { JAVA = 1 @@ -2372,7 +2382,7 @@ inline std::ostream& operator<<(std::ostream& out, const StorageDescriptor& obj) } typedef struct _Table__isset { - _Table__isset() : tableName(false), dbName(false), owner(false), createTime(false), lastAccessTime(false), retention(false), sd(false), partitionKeys(false), parameters(false), viewOriginalText(false), viewExpandedText(false), tableType(false), privileges(false), temporary(true), rewriteEnabled(false), creationMetadata(false) {} + _Table__isset() : tableName(false), dbName(false), owner(false), createTime(false), lastAccessTime(false), retention(false), sd(false), partitionKeys(false), parameters(false), viewOriginalText(false), viewExpandedText(false), tableType(false), privileges(false), temporary(true), rewriteEnabled(false), creationMetadata(false), bucketingVersion(true), loadInBucketedTable(true) {} bool tableName :1; bool dbName :1; bool owner :1; @@ -2389,6 +2399,8 @@ typedef struct _Table__isset { bool temporary :1; bool rewriteEnabled :1; bool creationMetadata :1; + bool bucketingVersion :1; + bool loadInBucketedTable :1; } _Table__isset; class Table { @@ -2396,7 +2408,9 @@ class Table { Table(const Table&); Table& operator=(const Table&); - Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType(), temporary(false), rewriteEnabled(0) { + Table() : tableName(), dbName(), owner(), createTime(0), lastAccessTime(0), retention(0), viewOriginalText(), viewExpandedText(), tableType(), temporary(false), rewriteEnabled(0), bucketingVersion((BucketingVersion::type)1), loadInBucketedTable(false) { + bucketingVersion = (BucketingVersion::type)1; + } virtual ~Table() throw(); @@ -2416,6 +2430,8 @@ class Table { bool temporary; bool rewriteEnabled; CreationMetadata creationMetadata; + BucketingVersion::type bucketingVersion; + bool loadInBucketedTable; _Table__isset __isset; @@ -2451,6 +2467,10 @@ class Table { void __set_creationMetadata(const CreationMetadata& val); + void __set_bucketingVersion(const BucketingVersion::type val); + + void __set_loadInBucketedTable(const bool val); + bool operator == (const Table & rhs) const { if (!(tableName == rhs.tableName)) @@ -2493,6 +2513,14 @@ class Table { return false; else if (__isset.creationMetadata && !(creationMetadata == rhs.creationMetadata)) return false; + if (__isset.bucketingVersion != rhs.__isset.bucketingVersion) + return false; + else if (__isset.bucketingVersion && !(bucketingVersion == rhs.bucketingVersion)) + return false; + if (__isset.loadInBucketedTable != rhs.__isset.loadInBucketedTable) + return false; + else if (__isset.loadInBucketedTable && !(loadInBucketedTable == rhs.loadInBucketedTable)) + return false; return true; } bool operator != (const Table &rhs) const { http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java new file mode 100644 index 0000000..b7de161 --- /dev/null +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/BucketingVersion.java @@ -0,0 +1,48 @@ +/** + * Autogenerated by Thrift Compiler (0.9.3) + * + * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING + * @generated + */ +package org.apache.hadoop.hive.metastore.api; + + +import java.util.Map; +import java.util.HashMap; +import org.apache.thrift.TEnum; + +public enum BucketingVersion implements org.apache.thrift.TEnum { + INVALID_BUCKETING(0), + JAVA_BUCKETING(1), + MURMUR_BUCKETING(2); + + private final int value; + + private BucketingVersion(int value) { + this.value = value; + } + + /** + * Get the integer value of this enum value, as defined in the Thrift IDL. + */ + public int getValue() { + return value; + } + + /** + * Find a the enum type by its integer value, as defined in the Thrift IDL. + * @return null if the value is not found. + */ + public static BucketingVersion findByValue(int value) { + switch (value) { + case 0: + return INVALID_BUCKETING; + case 1: + return JAVA_BUCKETING; + case 2: + return MURMUR_BUCKETING; + default: + return null; + } + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java index a132e5e..068e542 100644 --- a/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java +++ b/standalone-metastore/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/Table.java @@ -54,6 +54,8 @@ import org.slf4j.LoggerFactory; private static final org.apache.thrift.protocol.TField TEMPORARY_FIELD_DESC = new org.apache.thrift.protocol.TField("temporary", org.apache.thrift.protocol.TType.BOOL, (short)14); private static final org.apache.thrift.protocol.TField REWRITE_ENABLED_FIELD_DESC = new org.apache.thrift.protocol.TField("rewriteEnabled", org.apache.thrift.protocol.TType.BOOL, (short)15); private static final org.apache.thrift.protocol.TField CREATION_METADATA_FIELD_DESC = new org.apache.thrift.protocol.TField("creationMetadata", org.apache.thrift.protocol.TType.STRUCT, (short)16); + private static final org.apache.thrift.protocol.TField BUCKETING_VERSION_FIELD_DESC = new org.apache.thrift.protocol.TField("bucketingVersion", org.apache.thrift.protocol.TType.I32, (short)17); + private static final org.apache.thrift.protocol.TField LOAD_IN_BUCKETED_TABLE_FIELD_DESC = new org.apache.thrift.protocol.TField("loadInBucketedTable", org.apache.thrift.protocol.TType.BOOL, (short)18); private static final Map<Class<? extends IScheme>, SchemeFactory> schemes = new HashMap<Class<? extends IScheme>, SchemeFactory>(); static { @@ -77,6 +79,8 @@ import org.slf4j.LoggerFactory; private boolean temporary; // optional private boolean rewriteEnabled; // optional private CreationMetadata creationMetadata; // optional + private BucketingVersion bucketingVersion; // optional + private boolean loadInBucketedTable; // optional /** The set of fields this struct contains, along with convenience methods for finding and manipulating them. */ public enum _Fields implements org.apache.thrift.TFieldIdEnum { @@ -95,7 +99,13 @@ import org.slf4j.LoggerFactory; PRIVILEGES((short)13, "privileges"), TEMPORARY((short)14, "temporary"), REWRITE_ENABLED((short)15, "rewriteEnabled"), - CREATION_METADATA((short)16, "creationMetadata"); + CREATION_METADATA((short)16, "creationMetadata"), + /** + * + * @see BucketingVersion + */ + BUCKETING_VERSION((short)17, "bucketingVersion"), + LOAD_IN_BUCKETED_TABLE((short)18, "loadInBucketedTable"); private static final Map<String, _Fields> byName = new HashMap<String, _Fields>(); @@ -142,6 +152,10 @@ import org.slf4j.LoggerFactory; return REWRITE_ENABLED; case 16: // CREATION_METADATA return CREATION_METADATA; + case 17: // BUCKETING_VERSION + return BUCKETING_VERSION; + case 18: // LOAD_IN_BUCKETED_TABLE + return LOAD_IN_BUCKETED_TABLE; default: return null; } @@ -187,8 +201,9 @@ import org.slf4j.LoggerFactory; private static final int __RETENTION_ISSET_ID = 2; private static final int __TEMPORARY_ISSET_ID = 3; private static final int __REWRITEENABLED_ISSET_ID = 4; + private static final int __LOADINBUCKETEDTABLE_ISSET_ID = 5; private byte __isset_bitfield = 0; - private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.TEMPORARY,_Fields.REWRITE_ENABLED,_Fields.CREATION_METADATA}; + private static final _Fields optionals[] = {_Fields.PRIVILEGES,_Fields.TEMPORARY,_Fields.REWRITE_ENABLED,_Fields.CREATION_METADATA,_Fields.BUCKETING_VERSION,_Fields.LOAD_IN_BUCKETED_TABLE}; public static final Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> metaDataMap; static { Map<_Fields, org.apache.thrift.meta_data.FieldMetaData> tmpMap = new EnumMap<_Fields, org.apache.thrift.meta_data.FieldMetaData>(_Fields.class); @@ -227,6 +242,10 @@ import org.slf4j.LoggerFactory; new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); tmpMap.put(_Fields.CREATION_METADATA, new org.apache.thrift.meta_data.FieldMetaData("creationMetadata", org.apache.thrift.TFieldRequirementType.OPTIONAL, new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRUCT , "CreationMetadata"))); + tmpMap.put(_Fields.BUCKETING_VERSION, new org.apache.thrift.meta_data.FieldMetaData("bucketingVersion", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.EnumMetaData(org.apache.thrift.protocol.TType.ENUM, BucketingVersion.class))); + tmpMap.put(_Fields.LOAD_IN_BUCKETED_TABLE, new org.apache.thrift.meta_data.FieldMetaData("loadInBucketedTable", org.apache.thrift.TFieldRequirementType.OPTIONAL, + new org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.BOOL))); metaDataMap = Collections.unmodifiableMap(tmpMap); org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Table.class, metaDataMap); } @@ -234,6 +253,10 @@ import org.slf4j.LoggerFactory; public Table() { this.temporary = false; + this.bucketingVersion = org.apache.hadoop.hive.metastore.api.BucketingVersion.JAVA_BUCKETING; + + this.loadInBucketedTable = false; + } public Table( @@ -316,6 +339,10 @@ import org.slf4j.LoggerFactory; if (other.isSetCreationMetadata()) { this.creationMetadata = other.creationMetadata; } + if (other.isSetBucketingVersion()) { + this.bucketingVersion = other.bucketingVersion; + } + this.loadInBucketedTable = other.loadInBucketedTable; } public Table deepCopy() { @@ -345,6 +372,10 @@ import org.slf4j.LoggerFactory; setRewriteEnabledIsSet(false); this.rewriteEnabled = false; this.creationMetadata = null; + this.bucketingVersion = org.apache.hadoop.hive.metastore.api.BucketingVersion.JAVA_BUCKETING; + + this.loadInBucketedTable = false; + } public String getTableName() { @@ -736,6 +767,59 @@ import org.slf4j.LoggerFactory; } } + /** + * + * @see BucketingVersion + */ + public BucketingVersion getBucketingVersion() { + return this.bucketingVersion; + } + + /** + * + * @see BucketingVersion + */ + public void setBucketingVersion(BucketingVersion bucketingVersion) { + this.bucketingVersion = bucketingVersion; + } + + public void unsetBucketingVersion() { + this.bucketingVersion = null; + } + + /** Returns true if field bucketingVersion is set (has been assigned a value) and false otherwise */ + public boolean isSetBucketingVersion() { + return this.bucketingVersion != null; + } + + public void setBucketingVersionIsSet(boolean value) { + if (!value) { + this.bucketingVersion = null; + } + } + + public boolean isLoadInBucketedTable() { + return this.loadInBucketedTable; + } + + public void setLoadInBucketedTable(boolean loadInBucketedTable) { + this.loadInBucketedTable = loadInBucketedTable; + setLoadInBucketedTableIsSet(true); + } + + public void unsetLoadInBucketedTable() { + __isset_bitfield = EncodingUtils.clearBit(__isset_bitfield, __LOADINBUCKETEDTABLE_ISSET_ID); + } + + /** Returns true if field loadInBucketedTable is set (has been assigned a value) and false otherwise */ + public boolean isSetLoadInBucketedTable() { + return EncodingUtils.testBit(__isset_bitfield, __LOADINBUCKETEDTABLE_ISSET_ID); + } + + public void setLoadInBucketedTableIsSet(boolean value) { + __isset_bitfield = EncodingUtils.setBit(__isset_bitfield, __LOADINBUCKETEDTABLE_ISSET_ID, value); + } + public void setFieldValue(_Fields field, Object value) { switch (field) { case TABLE_NAME: @@ -866,6 +950,22 @@ import org.slf4j.LoggerFactory; } break; + case BUCKETING_VERSION: + if (value == null) { + unsetBucketingVersion(); + } else { + setBucketingVersion((BucketingVersion)value); + } + break; + + case LOAD_IN_BUCKETED_TABLE: + if (value == null) { + unsetLoadInBucketedTable(); + } else { + setLoadInBucketedTable((Boolean)value); + } + break; + } } @@ -919,6 +1019,12 @@ import org.slf4j.LoggerFactory; case CREATION_METADATA: return getCreationMetadata(); + case BUCKETING_VERSION: + return getBucketingVersion(); + + case LOAD_IN_BUCKETED_TABLE: + return isLoadInBucketedTable(); + } throw new IllegalStateException(); } @@ -962,6 +1068,10 @@ import org.slf4j.LoggerFactory; return isSetRewriteEnabled(); case CREATION_METADATA: return isSetCreationMetadata(); + case BUCKETING_VERSION: + return isSetBucketingVersion(); + case LOAD_IN_BUCKETED_TABLE: + return isSetLoadInBucketedTable(); } throw new IllegalStateException(); } @@ -1123,6 +1233,24 @@ import org.slf4j.LoggerFactory; return false; } + boolean this_present_bucketingVersion = true && this.isSetBucketingVersion(); + boolean that_present_bucketingVersion = true && that.isSetBucketingVersion(); + if (this_present_bucketingVersion || that_present_bucketingVersion) { + if (!(this_present_bucketingVersion && that_present_bucketingVersion)) + return false; + if (!this.bucketingVersion.equals(that.bucketingVersion)) + return false; + } + + boolean this_present_loadInBucketedTable = true && this.isSetLoadInBucketedTable(); + boolean that_present_loadInBucketedTable = true && that.isSetLoadInBucketedTable(); + if (this_present_loadInBucketedTable || that_present_loadInBucketedTable) { + if (!(this_present_loadInBucketedTable && that_present_loadInBucketedTable)) + return false; + if (this.loadInBucketedTable != that.loadInBucketedTable) + return false; + } + return true; } @@ -1210,6 +1338,16 @@ import org.slf4j.LoggerFactory; if (present_creationMetadata) list.add(creationMetadata); + boolean present_bucketingVersion = true && (isSetBucketingVersion()); + list.add(present_bucketingVersion); + if (present_bucketingVersion) + list.add(bucketingVersion.getValue()); + + boolean present_loadInBucketedTable = true && (isSetLoadInBucketedTable()); + list.add(present_loadInBucketedTable); + if (present_loadInBucketedTable) + list.add(loadInBucketedTable); + return list.hashCode(); } @@ -1381,6 +1519,26 @@ import org.slf4j.LoggerFactory; return lastComparison; } } + lastComparison = Boolean.valueOf(isSetBucketingVersion()).compareTo(other.isSetBucketingVersion()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetBucketingVersion()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.bucketingVersion, other.bucketingVersion); + if (lastComparison != 0) { + return lastComparison; + } + } + lastComparison = Boolean.valueOf(isSetLoadInBucketedTable()).compareTo(other.isSetLoadInBucketedTable()); + if (lastComparison != 0) { + return lastComparison; + } + if (isSetLoadInBucketedTable()) { + lastComparison = org.apache.thrift.TBaseHelper.compareTo(this.loadInBucketedTable, other.loadInBucketedTable); + if (lastComparison != 0) { + return lastComparison; + } + } return 0; } @@ -1516,6 +1674,22 @@ import org.slf4j.LoggerFactory; } first = false; } + if (isSetBucketingVersion()) { + if (!first) sb.append(", "); + sb.append("bucketingVersion:"); + if (this.bucketingVersion == null) { + sb.append("null"); + } else { + sb.append(this.bucketingVersion); + } + first = false; + } + if (isSetLoadInBucketedTable()) { + if (!first) sb.append(", "); + sb.append("loadInBucketedTable:"); + sb.append(this.loadInBucketedTable); + first = false; + } sb.append(")"); return sb.toString(); } @@ -1721,6 +1895,22 @@ import org.slf4j.LoggerFactory; org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } break; + case 17: // BUCKETING_VERSION + if (schemeField.type == org.apache.thrift.protocol.TType.I32) { + struct.bucketingVersion = org.apache.hadoop.hive.metastore.api.BucketingVersion.findByValue(iprot.readI32()); + struct.setBucketingVersionIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; + case 18: // LOAD_IN_BUCKETED_TABLE + if (schemeField.type == org.apache.thrift.protocol.TType.BOOL) { + struct.loadInBucketedTable = iprot.readBool(); + struct.setLoadInBucketedTableIsSet(true); + } else { + org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); + } + break; default: org.apache.thrift.protocol.TProtocolUtil.skip(iprot, schemeField.type); } @@ -1827,6 +2017,18 @@ import org.slf4j.LoggerFactory; oprot.writeFieldEnd(); } } + if (struct.bucketingVersion != null) { + if (struct.isSetBucketingVersion()) { + oprot.writeFieldBegin(BUCKETING_VERSION_FIELD_DESC); + oprot.writeI32(struct.bucketingVersion.getValue()); + oprot.writeFieldEnd(); + } + } + if (struct.isSetLoadInBucketedTable()) { + oprot.writeFieldBegin(LOAD_IN_BUCKETED_TABLE_FIELD_DESC); + oprot.writeBool(struct.loadInBucketedTable); + oprot.writeFieldEnd(); + } oprot.writeFieldStop(); oprot.writeStructEnd(); } @@ -1893,7 +2095,13 @@ import org.slf4j.LoggerFactory; if (struct.isSetCreationMetadata()) { optionals.set(15); } - oprot.writeBitSet(optionals, 16); + if (struct.isSetBucketingVersion()) { + optionals.set(16); + } + if (struct.isSetLoadInBucketedTable()) { + optionals.set(17); + } + oprot.writeBitSet(optionals, 18); if (struct.isSetTableName()) { oprot.writeString(struct.tableName); } @@ -1955,12 +2163,18 @@ import org.slf4j.LoggerFactory; if (struct.isSetCreationMetadata()) { struct.creationMetadata.write(oprot); } + if (struct.isSetBucketingVersion()) { + oprot.writeI32(struct.bucketingVersion.getValue()); + } + if (struct.isSetLoadInBucketedTable()) { + oprot.writeBool(struct.loadInBucketedTable); + } } @Override public void read(org.apache.thrift.protocol.TProtocol prot, Table struct) throws org.apache.thrift.TException { TTupleProtocol iprot = (TTupleProtocol) prot; - BitSet incoming = iprot.readBitSet(16); + BitSet incoming = iprot.readBitSet(18); if (incoming.get(0)) { struct.tableName = iprot.readString(); struct.setTableNameIsSet(true); @@ -2049,6 +2263,14 @@ import org.slf4j.LoggerFactory; struct.creationMetadata.read(iprot); struct.setCreationMetadataIsSet(true); } + if (incoming.get(16)) { + struct.bucketingVersion = org.apache.hadoop.hive.metastore.api.BucketingVersion.findByValue(iprot.readI32()); + struct.setBucketingVersionIsSet(true); + } + if (incoming.get(17)) { + struct.loadInBucketedTable = iprot.readBool(); + struct.setLoadInBucketedTableIsSet(true); + } } } http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php index a5b578e..687911e 100644 --- a/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php +++ b/standalone-metastore/src/gen/thrift/gen-php/metastore/Types.php @@ -142,6 +142,17 @@ final class EventRequestType { ); } +final class BucketingVersion { + const INVALID_BUCKETING = 0; + const JAVA_BUCKETING = 1; + const MURMUR_BUCKETING = 2; + static public $__names = array( + 0 => 'INVALID_BUCKETING', + 1 => 'JAVA_BUCKETING', + 2 => 'MURMUR_BUCKETING', + ); +} + final class FunctionType { const JAVA = 1; static public $__names = array( @@ -5042,6 +5053,14 @@ class Table { * @var \metastore\CreationMetadata */ public $creationMetadata = null; + /** + * @var int + */ + public $bucketingVersion = 1; + /** + * @var bool + */ + public $loadInBucketedTable = false; public function __construct($vals=null) { if (!isset(self::$_TSPEC)) { @@ -5126,6 +5145,14 @@ class Table { 'type' => TType::STRUCT, 'class' => '\metastore\CreationMetadata', ), + 17 => array( + 'var' => 'bucketingVersion', + 'type' => TType::I32, + ), + 18 => array( + 'var' => 'loadInBucketedTable', + 'type' => TType::BOOL, + ), ); } if (is_array($vals)) { @@ -5177,6 +5204,12 @@ class Table { if (isset($vals['creationMetadata'])) { $this->creationMetadata = $vals['creationMetadata']; } + if (isset($vals['bucketingVersion'])) { + $this->bucketingVersion = $vals['bucketingVersion']; + } + if (isset($vals['loadInBucketedTable'])) { + $this->loadInBucketedTable = $vals['loadInBucketedTable']; + } } } @@ -5338,6 +5371,20 @@ class Table { $xfer += $input->skip($ftype); } break; + case 17: + if ($ftype == TType::I32) { + $xfer += $input->readI32($this->bucketingVersion); + } else { + $xfer += $input->skip($ftype); + } + break; + case 18: + if ($ftype == TType::BOOL) { + $xfer += $input->readBool($this->loadInBucketedTable); + } else { + $xfer += $input->skip($ftype); + } + break; default: $xfer += $input->skip($ftype); break; @@ -5465,6 +5512,16 @@ class Table { $xfer += $this->creationMetadata->write($output); $xfer += $output->writeFieldEnd(); } + if ($this->bucketingVersion !== null) { + $xfer += $output->writeFieldBegin('bucketingVersion', TType::I32, 17); + $xfer += $output->writeI32($this->bucketingVersion); + $xfer += $output->writeFieldEnd(); + } + if ($this->loadInBucketedTable !== null) { + $xfer += $output->writeFieldBegin('loadInBucketedTable', TType::BOOL, 18); + $xfer += $output->writeBool($this->loadInBucketedTable); + $xfer += $output->writeFieldEnd(); + } $xfer += $output->writeFieldStop(); $xfer += $output->writeStructEnd(); return $xfer; http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py index 5598859..86c1937 100644 --- a/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py +++ b/standalone-metastore/src/gen/thrift/gen-py/hive_metastore/ttypes.py @@ -211,6 +211,23 @@ class EventRequestType: "DELETE": 3, } +class BucketingVersion: + INVALID_BUCKETING = 0 + JAVA_BUCKETING = 1 + MURMUR_BUCKETING = 2 + + _VALUES_TO_NAMES = { + 0: "INVALID_BUCKETING", + 1: "JAVA_BUCKETING", + 2: "MURMUR_BUCKETING", + } + + _NAMES_TO_VALUES = { + "INVALID_BUCKETING": 0, + "JAVA_BUCKETING": 1, + "MURMUR_BUCKETING": 2, + } + class FunctionType: JAVA = 1 @@ -3468,6 +3485,8 @@ class Table: - temporary - rewriteEnabled - creationMetadata + - bucketingVersion + - loadInBucketedTable """ thrift_spec = ( @@ -3488,9 +3507,11 @@ class Table: (14, TType.BOOL, 'temporary', None, False, ), # 14 (15, TType.BOOL, 'rewriteEnabled', None, None, ), # 15 (16, TType.STRUCT, 'creationMetadata', (CreationMetadata, CreationMetadata.thrift_spec), None, ), # 16 + (17, TType.I32, 'bucketingVersion', None, 1, ), # 17 + (18, TType.BOOL, 'loadInBucketedTable', None, False, ), # 18 ) - def __init__(self, tableName=None, dbName=None, owner=None, createTime=None, lastAccessTime=None, retention=None, sd=None, partitionKeys=None, parameters=None, viewOriginalText=None, viewExpandedText=None, tableType=None, privileges=None, temporary=thrift_spec[14][4], rewriteEnabled=None, creationMetadata=None,): + def __init__(self, tableName=None, dbName=None, owner=None, createTime=None, lastAccessTime=None, retention=None, sd=None, partitionKeys=None, parameters=None, viewOriginalText=None, viewExpandedText=None, tableType=None, privileges=None, temporary=thrift_spec[14][4], rewriteEnabled=None, creationMetadata=None, bucketingVersion=thrift_spec[17][4], loadInBucketedTable=thrift_spec[18][4],): self.tableName = tableName self.dbName = dbName self.owner = owner @@ -3507,6 +3528,8 @@ class Table: self.temporary = temporary self.rewriteEnabled = rewriteEnabled self.creationMetadata = creationMetadata + self.bucketingVersion = bucketingVersion + self.loadInBucketedTable = loadInBucketedTable def read(self, iprot): if iprot.__class__ == TBinaryProtocol.TBinaryProtocolAccelerated and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None and fastbinary is not None: @@ -3612,6 +3635,16 @@ class Table: self.creationMetadata.read(iprot) else: iprot.skip(ftype) + elif fid == 17: + if ftype == TType.I32: + self.bucketingVersion = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 18: + if ftype == TType.BOOL: + self.loadInBucketedTable = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -3693,6 +3726,14 @@ class Table: oprot.writeFieldBegin('creationMetadata', TType.STRUCT, 16) self.creationMetadata.write(oprot) oprot.writeFieldEnd() + if self.bucketingVersion is not None: + oprot.writeFieldBegin('bucketingVersion', TType.I32, 17) + oprot.writeI32(self.bucketingVersion) + oprot.writeFieldEnd() + if self.loadInBucketedTable is not None: + oprot.writeFieldBegin('loadInBucketedTable', TType.BOOL, 18) + oprot.writeBool(self.loadInBucketedTable) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -3718,6 +3759,8 @@ class Table: value = (value * 31) ^ hash(self.temporary) value = (value * 31) ^ hash(self.rewriteEnabled) value = (value * 31) ^ hash(self.creationMetadata) + value = (value * 31) ^ hash(self.bucketingVersion) + value = (value * 31) ^ hash(self.loadInBucketedTable) return value def __repr__(self): http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb index bc58cfe..2c3edb7 100644 --- a/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb +++ b/standalone-metastore/src/gen/thrift/gen-rb/hive_metastore_types.rb @@ -98,6 +98,14 @@ module EventRequestType VALID_VALUES = Set.new([INSERT, UPDATE, DELETE]).freeze end +module BucketingVersion + INVALID_BUCKETING = 0 + JAVA_BUCKETING = 1 + MURMUR_BUCKETING = 2 + VALUE_MAP = {0 => "INVALID_BUCKETING", 1 => "JAVA_BUCKETING", 2 => "MURMUR_BUCKETING"} + VALID_VALUES = Set.new([INVALID_BUCKETING, JAVA_BUCKETING, MURMUR_BUCKETING]).freeze +end + module FunctionType JAVA = 1 VALUE_MAP = {1 => "JAVA"} @@ -810,6 +818,8 @@ class Table TEMPORARY = 14 REWRITEENABLED = 15 CREATIONMETADATA = 16 + BUCKETINGVERSION = 17 + LOADINBUCKETEDTABLE = 18 FIELDS = { TABLENAME => {:type => ::Thrift::Types::STRING, :name => 'tableName'}, @@ -827,12 +837,17 @@ class Table PRIVILEGES => {:type => ::Thrift::Types::STRUCT, :name => 'privileges', :class => ::PrincipalPrivilegeSet, :optional => true}, TEMPORARY => {:type => ::Thrift::Types::BOOL, :name => 'temporary', :default => false, :optional => true}, REWRITEENABLED => {:type => ::Thrift::Types::BOOL, :name => 'rewriteEnabled', :optional => true}, - CREATIONMETADATA => {:type => ::Thrift::Types::STRUCT, :name => 'creationMetadata', :class => ::CreationMetadata, :optional => true} + CREATIONMETADATA => {:type => ::Thrift::Types::STRUCT, :name => 'creationMetadata', :class => ::CreationMetadata, :optional => true}, + BUCKETINGVERSION => {:type => ::Thrift::Types::I32, :name => 'bucketingVersion', :default => 1, :optional => true, :enum_class => ::BucketingVersion}, + LOADINBUCKETEDTABLE => {:type => ::Thrift::Types::BOOL, :name => 'loadInBucketedTable', :default => false, :optional => true} } def struct_fields; FIELDS; end def validate + unless @bucketingVersion.nil? || ::BucketingVersion::VALID_VALUES.include?(@bucketingVersion) + raise ::Thrift::ProtocolException.new(::Thrift::ProtocolException::UNKNOWN, 'Invalid value of field bucketingVersion!') + end end ::Thrift::Struct.generate_accessors self http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index d58ed67..7003fa8 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -1688,6 +1688,8 @@ public class ObjectStore implements RawStore, Configurable { convertToFieldSchemas(mtbl.getPartitionKeys()), convertMap(mtbl.getParameters()), mtbl.getViewOriginalText(), mtbl.getViewExpandedText(), tableType); t.setRewriteEnabled(mtbl.isRewriteEnabled()); + t.setBucketingVersion(mtbl.getBucketingVersion()); + t.setLoadInBucketedTable(mtbl.isLoadInBucketedTable()); return t; } @@ -1726,7 +1728,8 @@ public class ObjectStore implements RawStore, Configurable { .getCreateTime(), tbl.getLastAccessTime(), tbl.getRetention(), convertToMFieldSchemas(tbl.getPartitionKeys()), tbl.getParameters(), tbl.getViewOriginalText(), tbl.getViewExpandedText(), tbl.isRewriteEnabled(), - tableType); + tableType, + tbl.getBucketingVersion(), tbl.isLoadInBucketedTable()); } private List<MFieldSchema> convertToMFieldSchemas(List<FieldSchema> keys) { http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java index a38a125..d97826b 100644 --- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java +++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/model/MTable.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hive.metastore.model; +import org.apache.hadoop.hive.metastore.api.BucketingVersion; + import java.util.List; import java.util.Map; @@ -36,6 +38,8 @@ public class MTable { private String viewExpandedText; private boolean rewriteEnabled; private String tableType; + private BucketingVersion bucketingVersion; + private boolean loadInBucketedTable; public MTable() {} @@ -56,7 +60,8 @@ public class MTable { public MTable(String tableName, MDatabase database, MStorageDescriptor sd, String owner, int createTime, int lastAccessTime, int retention, List<MFieldSchema> partitionKeys, Map<String, String> parameters, String viewOriginalText, String viewExpandedText, - boolean rewriteEnabled, String tableType) { + boolean rewriteEnabled, + String tableType, BucketingVersion bucketingVersion, boolean loadInBucketedTable) { this.tableName = tableName; this.database = database; this.sd = sd; @@ -70,6 +75,8 @@ public class MTable { this.viewExpandedText = viewExpandedText; this.rewriteEnabled = rewriteEnabled; this.tableType = tableType; + this.bucketingVersion = bucketingVersion; + this.loadInBucketedTable = loadInBucketedTable; } /** @@ -253,4 +260,32 @@ public class MTable { public String getTableType() { return tableType; } + + /** + * @param bucketingVersion used in bucketed table + */ + public void setBucketingVersion(BucketingVersion bucketingVersion) { + this.bucketingVersion = bucketingVersion; + } + + /** + * @return the bucketingVersion + */ + public BucketingVersion getBucketingVersion() { + return bucketingVersion; + } + /** + * @param loadInBucketedTable + */ + public void setLoadInBucketedTable(boolean loadInBucketedTable) { + this.loadInBucketedTable = loadInBucketedTable; + } + + /** + * @return the expertMode + */ + public boolean isLoadInBucketedTable() { + return loadInBucketedTable; + } + } http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/main/thrift/hive_metastore.thrift ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/main/thrift/hive_metastore.thrift b/standalone-metastore/src/main/thrift/hive_metastore.thrift index 371b975..cf4f64f 100644 --- a/standalone-metastore/src/main/thrift/hive_metastore.thrift +++ b/standalone-metastore/src/main/thrift/hive_metastore.thrift @@ -310,6 +310,13 @@ struct StorageDescriptor { 12: optional bool storedAsSubDirectories // stored as subdirectories or not } +// Hash version for bucketing table +enum BucketingVersion { + INVALID_BUCKETING = 0, + JAVA_BUCKETING = 1, + MURMUR_BUCKETING = 2, +} + // table information struct Table { 1: string tableName, // name of the table @@ -327,7 +334,9 @@ struct Table { 13: optional PrincipalPrivilegeSet privileges, 14: optional bool temporary=false, 15: optional bool rewriteEnabled, // rewrite enabled or not - 16: optional CreationMetadata creationMetadata // only for MVs, it stores table names used and txn list at MV creation + 16: optional CreationMetadata creationMetadata, // only for MVs, it stores table names used and txn list at MV creation + 17: optional BucketingVersion bucketingVersion = BucketingVersion.JAVA_BUCKETING, // For bucketed table only. Default : 2, for existing tables, 1, for new tables 2. + 18: optional bool loadInBucketedTable = false // For bucketed table only. Default : false. true if user loads data using âload dataâ command. } struct Partition { http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java index bd61df6..09bed5d 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/cache/TestCachedStore.java @@ -29,21 +29,7 @@ import org.apache.hadoop.hive.metastore.MetaStoreTestUtils; import org.apache.hadoop.hive.metastore.ObjectStore; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.TestObjectStore.MockPartitionExpressionProxy; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.BasicTxnInfo; -import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; @@ -225,6 +211,8 @@ public class TestCachedStore { Table tbl1 = new Table(tblName1, dbName, tblOwner, 0, 0, 0, sd, new ArrayList<>(), tblParams, null, null, TableType.MANAGED_TABLE.toString()); + tbl1.setBucketingVersion(BucketingVersion.JAVA_BUCKETING); + tbl1.setLoadInBucketedTable(false); cachedStore.createTable(tbl1); tbl1 = cachedStore.getTable(dbName, tblName1); http://git-wip-us.apache.org/repos/asf/hive/blob/6e9b63e4/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java index 00f38ee..f10be72 100644 --- a/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java +++ b/standalone-metastore/src/test/java/org/apache/hadoop/hive/metastore/client/TestTablesCreateDropAlterTruncate.java @@ -21,18 +21,7 @@ package org.apache.hadoop.hive.metastore.client; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; -import org.apache.hadoop.hive.metastore.api.EnvironmentContext; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.InvalidOperationException; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.SkewedInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; +import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder; import org.apache.hadoop.hive.metastore.client.builder.PartitionBuilder; import org.apache.hadoop.hive.metastore.client.builder.TableBuilder; @@ -224,6 +213,9 @@ public class TestTablesCreateDropAlterTruncate { public void testCreateGetDeleteTable() throws Exception { // Try to create a table with all of the parameters set Table table = getTableWithAllParametersSet(); + // Set parameters set outside + table.setBucketingVersion(BucketingVersion.MURMUR_BUCKETING); + table.setLoadInBucketedTable(false); client.createTable(table); Table createdTable = client.getTable(table.getDbName(), table.getTableName()); // The createTime will be set on the server side, so the comparison should skip it @@ -684,6 +676,9 @@ public class TestTablesCreateDropAlterTruncate { // Partition keys can not be set, but getTableWithAllParametersSet is added one, so remove for // this test newTable.setPartitionKeys(originalTable.getPartitionKeys()); + // Set the optional bucketingVersion and expertMode with default values + newTable.setBucketingVersion(BucketingVersion.JAVA_BUCKETING); + newTable.setLoadInBucketedTable(false); client.alter_table(originalDatabase, originalTableName, newTable); Table alteredTable = client.getTable(originalDatabase, originalTableName);
