This is an automated email from the ASF dual-hosted git repository. mgergely pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push: new 029cab2 HIVE-22192 Remove ObjectPair classes 029cab2 is described below commit 029cab297a9ae40d249f63040721f93857398648 Author: miklosgergely <mgerg...@cloudera.com> AuthorDate: Wed Sep 11 10:33:46 2019 +0200 HIVE-22192 Remove ObjectPair classes --- .../org/apache/hadoop/hive/common/ObjectPair.java | 86 ---------------------- .../hive/hcatalog/api/HCatClientHMSImpl.java | 4 +- .../AlterTableDropPartitionOperation.java | 6 +- .../apache/hadoop/hive/ql/exec/FooterBuffer.java | 18 +++-- .../hadoop/hive/ql/exec/MapJoinOperator.java | 14 +--- .../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 17 +++-- .../ql/exec/persistence/KeyValueContainer.java | 18 ++--- .../hive/ql/exec/spark/HiveKVResultCache.java | 56 +++++++------- .../hive/ql/exec/spark/session/SparkSession.java | 4 +- .../ql/exec/spark/session/SparkSessionImpl.java | 9 +-- .../apache/hadoop/hive/ql/hooks/LineageLogger.java | 12 +-- .../org/apache/hadoop/hive/ql/metadata/Hive.java | 76 ++++++++++++++++--- .../hive/ql/optimizer/ConvertJoinMapJoin.java | 2 +- .../hadoop/hive/ql/optimizer/MapJoinProcessor.java | 16 ++-- .../hive/ql/optimizer/lineage/LineageCtx.java | 20 ++--- .../hive/ql/optimizer/ppr/PartExprEvalUtils.java | 10 +-- .../hive/ql/optimizer/ppr/PartitionPruner.java | 4 +- .../spark/SetSparkReducerParallelism.java | 10 +-- .../hadoop/hive/ql/parse/CalcitePlanner.java | 2 +- .../apache/hadoop/hive/ql/parse/QBSubQuery.java | 32 ++++---- .../hadoop/hive/ql/parse/SemanticAnalyzer.java | 65 ++++++++-------- .../hive/ql/parse/spark/GenSparkProcContext.java | 8 +- .../hadoop/hive/ql/parse/spark/GenSparkWork.java | 13 ++-- .../TestReflectionObjectInspectors.java | 17 ++--- .../hadoop/hive/metastore/HiveMetaStoreClient.java | 19 ++--- .../hadoop/hive/metastore/IMetaStoreClient.java | 14 ++-- .../hadoop/hive/metastore/utils/ObjectPair.java | 86 ---------------------- .../org/apache/hadoop/hive/metastore/Msck.java | 10 +-- .../apache/hadoop/hive/metastore/ObjectStore.java | 75 ++++++++----------- .../metastore/HiveMetaStoreClientPreCatalog.java | 19 +++-- 30 files changed, 290 insertions(+), 452 deletions(-) diff --git a/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java b/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java deleted file mode 100644 index 50b5b3f..0000000 --- a/common/src/java/org/apache/hadoop/hive/common/ObjectPair.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.common; - - - -public class ObjectPair<F, S> { - private F first; - private S second; - - public ObjectPair() {} - - /** - * Creates a pair. Constructor doesn't infer template args but - * the method does, so the code becomes less ugly. - */ - public static <T1, T2> ObjectPair<T1, T2> create(T1 f, T2 s) { - return new ObjectPair<T1, T2>(f, s); - } - - public ObjectPair(F first, S second) { - this.first = first; - this.second = second; - } - - public F getFirst() { - return first; - } - - public void setFirst(F first) { - this.first = first; - } - - public S getSecond() { - return second; - } - - public void setSecond(S second) { - this.second = second; - } - - @Override - public boolean equals(Object that) { - if (that == null) { - return false; - } - if (that instanceof ObjectPair) { - return this.equals((ObjectPair<F, S>)that); - } - return false; - } - - public boolean equals(ObjectPair<F, S> that) { - if (that == null) { - return false; - } - - return this.getFirst().equals(that.getFirst()) && - this.getSecond().equals(that.getSecond()); - } - - @Override - public int hashCode() { - return first.hashCode() * 31 + second.hashCode(); - } - - public String toString() { - return first + ":" + second; - } -} diff --git a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java index a06191d..1cb35bb 100644 --- a/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java +++ b/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatClientHMSImpl.java @@ -27,6 +27,7 @@ import java.util.Map; import javax.annotation.Nullable; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.common.classification.InterfaceAudience; @@ -51,7 +52,6 @@ import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.UnknownDBException; import org.apache.hadoop.hive.metastore.api.UnknownTableException; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.hadoop.hive.ql.exec.FunctionRegistry; import org.apache.hadoop.hive.ql.exec.SerializationUtilities; import org.apache.hadoop.hive.ql.parse.SemanticException; @@ -584,7 +584,7 @@ public class HCatClientHMSImpl extends HCatClient { throws SemanticException, TException { LOG.info("HCatClient: Dropping partitions using partition-predicate Expressions."); ExprNodeGenericFuncDesc partitionExpression = new ExpressionBuilder(table, partitionSpec).build(); - ObjectPair<Integer, byte[]> serializedPartitionExpression = new ObjectPair<>(partitionSpec.size(), + Pair<Integer, byte[]> serializedPartitionExpression = Pair.of(partitionSpec.size(), SerializationUtilities.serializeExpressionToKryo(partitionExpression)); hmsClient.dropPartitions(table.getDbName(), table.getTableName(), Arrays.asList(serializedPartitionExpression), deleteData && !isExternal(table), // Delete data? diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableDropPartitionOperation.java b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableDropPartitionOperation.java index 13c9416..6f0dfba 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableDropPartitionOperation.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/ddl/table/partition/AlterTableDropPartitionOperation.java @@ -21,9 +21,9 @@ package org.apache.hadoop.hive.ql.ddl.table.partition; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.metastore.PartitionDropOptions; import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.hadoop.hive.ql.ddl.DDLOperation; import org.apache.hadoop.hive.ql.ddl.DDLOperationContext; import org.apache.hadoop.hive.ql.ddl.DDLUtils; @@ -106,9 +106,9 @@ public class AlterTableDropPartitionOperation extends DDLOperation<AlterTableDro // ifExists is currently verified in DDLSemanticAnalyzer String[] names = Utilities.getDbTableName(desc.getTableName()); - List<ObjectPair<Integer, byte[]>> partitionExpressions = new ArrayList<>(desc.getPartSpecs().size()); + List<Pair<Integer, byte[]>> partitionExpressions = new ArrayList<>(desc.getPartSpecs().size()); for (AlterTableDropPartitionDesc.PartitionDesc partSpec : desc.getPartSpecs()) { - partitionExpressions.add(new ObjectPair<>(partSpec.getPrefixLength(), + partitionExpressions.add(Pair.of(partSpec.getPrefixLength(), SerializationUtilities.serializeExpressionToKryo(partSpec.getPartSpec()))); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java index 8ead797..8668b1d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java @@ -20,8 +20,9 @@ package org.apache.hadoop.hive.ql.exec; import java.io.IOException; import java.util.ArrayList; +import java.util.List; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.JobConf; @@ -29,7 +30,7 @@ import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.util.ReflectionUtils; public class FooterBuffer { - private ArrayList<ObjectPair<WritableComparable, Writable>> buffer; + private List<Pair<WritableComparable, Writable>> buffer; private int cur; public FooterBuffer() { @@ -70,9 +71,10 @@ public class FooterBuffer { if (!notEOF) { return false; } - ObjectPair<WritableComparable, Writable> tem = new ObjectPair<>(); - tem.setFirst(ReflectionUtils.copy(job, key, tem.getFirst())); - tem.setSecond(ReflectionUtils.copy(job, value, tem.getSecond())); + + WritableComparable left = ReflectionUtils.copy(job, key, null); + Writable right = ReflectionUtils.copy(job, value, null); + Pair<WritableComparable, Writable> tem = Pair.of(left, right); buffer.add(tem); } this.cur = 0; @@ -98,9 +100,9 @@ public class FooterBuffer { */ public boolean updateBuffer(JobConf job, RecordReader recordreader, WritableComparable key, Writable value) throws IOException { - key = ReflectionUtils.copy(job, buffer.get(cur).getFirst(), key); - value = ReflectionUtils.copy(job, buffer.get(cur).getSecond(), value); - boolean notEOF = recordreader.next(buffer.get(cur).getFirst(), buffer.get(cur).getSecond()); + key = ReflectionUtils.copy(job, buffer.get(cur).getKey(), key); + value = ReflectionUtils.copy(job, buffer.get(cur).getValue(), value); + boolean notEOF = recordreader.next(buffer.get(cur).getKey(), buffer.get(cur).getValue()); if (notEOF) { cur = (++cur) % buffer.size(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java index b0b4ec8..0643a54 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java @@ -23,13 +23,11 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Future; -import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.lang.ArrayUtils; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.Constants; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; @@ -134,7 +132,6 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem // Only used in bucket map join. private transient int numBuckets = -1; private transient int bucketId = -1; - private transient ReentrantLock subCacheLock = new ReentrantLock(); /** Kryo ctor. */ protected MapJoinOperator() { @@ -680,8 +677,6 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem */ NonMatchedSmallTableIterator nonMatchedIterator = substituteSmallTable.createNonMatchedSmallTableIterator(matchTracker); - int nonMatchedKeyCount = 0; - int nonMatchedValueCount = 0; while (nonMatchedIterator.isNext()) { List<Object> keyObjList = nonMatchedIterator.getCurrentKey(); @@ -729,10 +724,7 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem // FUTURE: Support residual filters for non-equi joins. internalForward(standardCopyRow, outputObjInspector); - nonMatchedValueCount++; } - - nonMatchedKeyCount++; } } @@ -938,9 +930,9 @@ public class MapJoinOperator extends AbstractMapJoinOperator<MapJoinDesc> implem KeyValueHelper writeHelper = container.getWriteHelper(); while (kvContainer.hasNext()) { - ObjectPair<HiveKey, BytesWritable> pair = kvContainer.next(); - Writable key = pair.getFirst(); - Writable val = pair.getSecond(); + Pair<HiveKey, BytesWritable> pair = kvContainer.next(); + Writable key = pair.getLeft(); + Writable val = pair.getRight(); writeHelper.setKeyValue(key, val); restoredHashMap.put(writeHelper, -1); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java index bfdb7d2..c09bf53 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java @@ -28,9 +28,9 @@ import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.CompilationOpContext; import org.apache.hadoop.hive.ql.exec.persistence.RowContainer; @@ -696,7 +696,7 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp // index of FetchOperator which is providing smallest one transient Integer currentMinSegment; - transient ObjectPair<List<Object>, InspectableObject>[] keys; + transient MutablePair<List<Object>, InspectableObject>[] keys; public MergeQueue(String alias, FetchWork fetchWork, JobConf jobConf, Operator<? extends OperatorDesc> forwardOp, @@ -737,7 +737,7 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp private FetchOperator[] segmentsForSize(int segmentLen) { if (segments == null || segments.length < segmentLen) { FetchOperator[] newSegments = new FetchOperator[segmentLen]; - ObjectPair<List<Object>, InspectableObject>[] newKeys = new ObjectPair[segmentLen]; + MutablePair<List<Object>, InspectableObject>[] newKeys = new MutablePair[segmentLen]; if (segments != null) { System.arraycopy(segments, 0, newSegments, 0, segments.length); System.arraycopy(keys, 0, newKeys, 0, keys.length); @@ -760,7 +760,7 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp @Override protected boolean lessThan(Object a, Object b) { - return compareKeys(keys[(Integer) a].getFirst(), keys[(Integer)b].getFirst()) < 0; + return compareKeys(keys[(Integer) a].getLeft(), keys[(Integer)b].getLeft()) < 0; } public final InspectableObject getNextRow() throws IOException { @@ -775,7 +775,8 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp return null; } counter++; - return keys[currentMinSegment = current].getSecond(); + currentMinSegment = current; + return keys[currentMinSegment].getRight(); } private void adjustPriorityQueue(Integer current) throws IOException { @@ -816,7 +817,7 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp while (nextRow != null) { sinkOp.reset(); if (keys[current] == null) { - keys[current] = new ObjectPair<List<Object>, InspectableObject>(); + keys[current] = new MutablePair<List<Object>, InspectableObject>(); } // Pass the row though the operator tree. It is guaranteed that not more than 1 row can @@ -827,8 +828,8 @@ public class SMBMapJoinOperator extends AbstractMapJoinOperator<SMBJoinDesc> imp // It is possible that the row got absorbed in the operator tree. if (nextRow.o != null) { // todo this should be changed to be evaluated lazily, especially for single segment case - keys[current].setFirst(JoinUtil.computeKeys(nextRow.o, keyFields, keyFieldOIs)); - keys[current].setSecond(nextRow); + keys[current].setLeft(JoinUtil.computeKeys(nextRow.o, keyFields, keyFieldOIs)); + keys[current].setRight(nextRow); return true; } nextRow = segments[current].getNextRow(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java index 3f00dc6..6d06bff 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/KeyValueContainer.java @@ -23,9 +23,9 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.hive.ql.metadata.HiveException; import org.apache.hadoop.io.BytesWritable; @@ -47,7 +47,7 @@ public class KeyValueContainer { @VisibleForTesting static final int IN_MEMORY_NUM_ROWS = 1024; - private ObjectPair<HiveKey, BytesWritable>[] readBuffer; + private MutablePair<HiveKey, BytesWritable>[] readBuffer; private boolean readBufferUsed = false; // indicates if read buffer has data private int rowsInReadBuffer = 0; // number of rows in the temporary read buffer private int readCursor = 0; // cursor during reading @@ -60,9 +60,9 @@ public class KeyValueContainer { private Output output; public KeyValueContainer(String spillLocalDirs) { - readBuffer = new ObjectPair[IN_MEMORY_NUM_ROWS]; + readBuffer = new MutablePair[IN_MEMORY_NUM_ROWS]; for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) { - readBuffer[i] = new ObjectPair<HiveKey, BytesWritable>(); + readBuffer[i] = new MutablePair<HiveKey, BytesWritable>(); } try { setupOutput(spillLocalDirs); @@ -158,7 +158,7 @@ public class KeyValueContainer { return readBufferUsed || rowsOnDisk > 0; } - public ObjectPair<HiveKey, BytesWritable> next() { + public MutablePair<HiveKey, BytesWritable> next() { Preconditions.checkState(hasNext()); if (!readBufferUsed) { try { @@ -186,9 +186,9 @@ public class KeyValueContainer { } for (int i = 0; i < rowsInReadBuffer; i++) { - ObjectPair<HiveKey, BytesWritable> pair = readBuffer[i]; - pair.setFirst(readHiveKey(input)); - pair.setSecond(readValue(input)); + MutablePair<HiveKey, BytesWritable> pair = readBuffer[i]; + pair.setLeft(readHiveKey(input)); + pair.setRight(readValue(input)); } if (input.eof()) { @@ -206,7 +206,7 @@ public class KeyValueContainer { } } - ObjectPair<HiveKey, BytesWritable> row = readBuffer[readCursor]; + MutablePair<HiveKey, BytesWritable> row = readBuffer[readCursor]; if (++readCursor >= rowsInReadBuffer) { readBufferUsed = false; rowsInReadBuffer = 0; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java index 413946b..253e310 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HiveKVResultCache.java @@ -24,8 +24,8 @@ import java.io.IOException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.commons.lang3.tuple.MutablePair; import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.ql.io.HiveKey; import org.apache.hadoop.io.BytesWritable; @@ -49,8 +49,8 @@ class HiveKVResultCache { @VisibleForTesting static final int IN_MEMORY_NUM_ROWS = 1024; - private ObjectPair<HiveKey, BytesWritable>[] writeBuffer; - private ObjectPair<HiveKey, BytesWritable>[] readBuffer; + private MutablePair<HiveKey, BytesWritable>[] writeBuffer; + private MutablePair<HiveKey, BytesWritable>[] readBuffer; private File parentFile; private File tmpFile; @@ -67,16 +67,16 @@ class HiveKVResultCache { private Output output; public HiveKVResultCache() { - writeBuffer = new ObjectPair[IN_MEMORY_NUM_ROWS]; - readBuffer = new ObjectPair[IN_MEMORY_NUM_ROWS]; + writeBuffer = new MutablePair[IN_MEMORY_NUM_ROWS]; + readBuffer = new MutablePair[IN_MEMORY_NUM_ROWS]; for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) { - writeBuffer[i] = new ObjectPair<HiveKey, BytesWritable>(); - readBuffer[i] = new ObjectPair<HiveKey, BytesWritable>(); + writeBuffer[i] = new MutablePair<HiveKey, BytesWritable>(); + readBuffer[i] = new MutablePair<HiveKey, BytesWritable>(); } } private void switchBufferAndResetCursor() { - ObjectPair<HiveKey, BytesWritable>[] tmp = readBuffer; + MutablePair<HiveKey, BytesWritable>[] tmp = readBuffer; rowsInReadBuffer = writeCursor; readBuffer = writeBuffer; readBufferUsed = true; @@ -152,11 +152,11 @@ class HiveKVResultCache { setupOutput(); } for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) { - ObjectPair<HiveKey, BytesWritable> pair = writeBuffer[i]; - writeHiveKey(output, pair.getFirst()); - writeValue(output, pair.getSecond()); - pair.setFirst(null); - pair.setSecond(null); + MutablePair<HiveKey, BytesWritable> pair = writeBuffer[i]; + writeHiveKey(output, pair.getLeft()); + writeValue(output, pair.getRight()); + pair.setLeft(null); + pair.setRight(null); } writeCursor = 0; } catch (Exception e) { @@ -165,9 +165,9 @@ class HiveKVResultCache { } } } - ObjectPair<HiveKey, BytesWritable> pair = writeBuffer[writeCursor++]; - pair.setFirst(key); - pair.setSecond(value); + MutablePair<HiveKey, BytesWritable> pair = writeBuffer[writeCursor++]; + pair.setLeft(key); + pair.setRight(value); } public synchronized void clear() { @@ -224,9 +224,9 @@ class HiveKVResultCache { if (input != null) { // Load next batch from disk for (int i = 0; i < IN_MEMORY_NUM_ROWS; i++) { - ObjectPair<HiveKey, BytesWritable> pair = readBuffer[i]; - pair.setFirst(readHiveKey(input)); - pair.setSecond(readValue(input)); + MutablePair<HiveKey, BytesWritable> pair = readBuffer[i]; + pair.setLeft(readHiveKey(input)); + pair.setRight(readValue(input)); } if (input.eof()) { input.close(); @@ -236,11 +236,10 @@ class HiveKVResultCache { readBufferUsed = true; readCursor = 0; } else if (writeCursor == 1) { - ObjectPair<HiveKey, BytesWritable> pair = writeBuffer[0]; - Tuple2<HiveKey, BytesWritable> row = new Tuple2<HiveKey, BytesWritable>( - pair.getFirst(), pair.getSecond()); - pair.setFirst(null); - pair.setSecond(null); + MutablePair<HiveKey, BytesWritable> pair = writeBuffer[0]; + Tuple2<HiveKey, BytesWritable> row = new Tuple2<HiveKey, BytesWritable>(pair.getLeft(), pair.getRight()); + pair.setLeft(null); + pair.setRight(null); writeCursor = 0; return row; } else { @@ -252,11 +251,10 @@ class HiveKVResultCache { throw new RuntimeException("Failed to load rows from disk", e); } } - ObjectPair<HiveKey, BytesWritable> pair = readBuffer[readCursor]; - Tuple2<HiveKey, BytesWritable> row = new Tuple2<HiveKey, BytesWritable>( - pair.getFirst(), pair.getSecond()); - pair.setFirst(null); - pair.setSecond(null); + MutablePair<HiveKey, BytesWritable> pair = readBuffer[readCursor]; + Tuple2<HiveKey, BytesWritable> row = new Tuple2<HiveKey, BytesWritable>(pair.getLeft(), pair.getRight()); + pair.setLeft(null); + pair.setRight(null); if (++readCursor >= rowsInReadBuffer) { readBufferUsed = false; rowsInReadBuffer = 0; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java index 62f88c4..b29be48 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSession.java @@ -17,8 +17,8 @@ */ package org.apache.hadoop.hive.ql.exec.spark.session; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.exec.spark.status.SparkJobRef; @@ -50,7 +50,7 @@ public interface SparkSession { * @return an object pair, the first element is the shuffle memory per task in bytes, * the second element is the number of total cores usable by the client */ - ObjectPair<Long, Integer> getMemoryAndCores() throws Exception; + Pair<Long, Integer> getMemoryAndCores() throws Exception; /** * @return true if the session is open and ready to submit jobs. diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java index 1d251ed..7e64342 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/session/SparkSessionImpl.java @@ -32,6 +32,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; @@ -41,7 +42,6 @@ import org.apache.hadoop.hive.ql.session.SessionState; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.DriverContext; import org.apache.hadoop.hive.ql.exec.spark.HiveSparkClient; @@ -153,14 +153,14 @@ public class SparkSessionImpl implements SparkSession { } @Override - public ObjectPair<Long, Integer> getMemoryAndCores() throws Exception { + public Pair<Long, Integer> getMemoryAndCores() throws Exception { closeLock.readLock().lock(); try { SparkConf sparkConf = hiveSparkClient.getSparkConf(); int numExecutors = hiveSparkClient.getExecutorCount(); // at start-up, we may be unable to get number of executors if (numExecutors <= 0) { - return new ObjectPair<Long, Integer>(-1L, -1); + return Pair.of(-1L, -1); } int executorMemoryInMB = Utils.memoryStringToMb( sparkConf.get("spark.executor.memory", "512m")); @@ -183,8 +183,7 @@ public class SparkSessionImpl implements SparkSession { LOG.info("Hive on Spark application currently has number of executors: " + numExecutors + ", total cores: " + totalCores + ", memory per executor: " + executorMemoryInMB + " mb, memoryFraction: " + memoryFraction); - return new ObjectPair<Long, Integer>(Long.valueOf(memoryPerTaskInBytes), - Integer.valueOf(totalCores)); + return Pair.of(Long.valueOf(memoryPerTaskInBytes), Integer.valueOf(totalCores)); } finally { closeLock.readLock().unlock(); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java index a9d845a..ec4c04c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java @@ -26,7 +26,7 @@ import com.google.gson.stream.JsonWriter; import org.apache.commons.collections.SetUtils; import org.apache.commons.io.output.StringBuilderWriter; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.Warehouse; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -249,15 +249,15 @@ public class LineageLogger implements ExecuteWithHookContext { */ @VisibleForTesting public static List<Edge> getEdges(QueryPlan plan, Index index) { - LinkedHashMap<String, ObjectPair<SelectOperator, - org.apache.hadoop.hive.ql.metadata.Table>> finalSelOps = index.getFinalSelectOps(); + Map<String, Pair<SelectOperator, org.apache.hadoop.hive.ql.metadata.Table>> finalSelOps = + index.getFinalSelectOps(); Map<String, Vertex> vertexCache = new LinkedHashMap<String, Vertex>(); List<Edge> edges = new ArrayList<Edge>(); - for (ObjectPair<SelectOperator, + for (Pair<SelectOperator, org.apache.hadoop.hive.ql.metadata.Table> pair: finalSelOps.values()) { List<FieldSchema> fieldSchemas = plan.getResultSchema().getFieldSchemas(); - SelectOperator finalSelOp = pair.getFirst(); - org.apache.hadoop.hive.ql.metadata.Table t = pair.getSecond(); + SelectOperator finalSelOp = pair.getLeft(); + org.apache.hadoop.hive.ql.metadata.Table t = pair.getRight(); String destTableName = null; List<String> colNames = null; if (t != null) { diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index 522c20a..db8cc6c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -80,6 +80,7 @@ import org.apache.calcite.rel.core.TableScan; import org.apache.calcite.rex.RexBuilder; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang3.exception.ExceptionUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; @@ -90,7 +91,6 @@ import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.TableName; import org.apache.hadoop.hive.common.ValidReaderWriteIdList; @@ -115,8 +115,64 @@ import org.apache.hadoop.hive.metastore.RetryingMetaStoreClient; import org.apache.hadoop.hive.metastore.SynchronizedMetaStoreClient; import org.apache.hadoop.hive.metastore.TableType; import org.apache.hadoop.hive.metastore.Warehouse; +import org.apache.hadoop.hive.metastore.api.AggrStats; +import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; +import org.apache.hadoop.hive.metastore.api.CheckConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.CmRecycleRequest; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; +import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.CompactionResponse; +import org.apache.hadoop.hive.metastore.api.CompactionType; +import org.apache.hadoop.hive.metastore.api.CreationMetadata; +import org.apache.hadoop.hive.metastore.api.Database; +import org.apache.hadoop.hive.metastore.api.DefaultConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.EnvironmentContext; +import org.apache.hadoop.hive.metastore.api.FieldSchema; +import org.apache.hadoop.hive.metastore.api.FireEventRequest; +import org.apache.hadoop.hive.metastore.api.FireEventRequestData; +import org.apache.hadoop.hive.metastore.api.ForeignKeysRequest; +import org.apache.hadoop.hive.metastore.api.Function; +import org.apache.hadoop.hive.metastore.api.GetOpenTxnsInfoResponse; +import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalRequest; +import org.apache.hadoop.hive.metastore.api.GetRoleGrantsForPrincipalResponse; +import org.apache.hadoop.hive.metastore.api.HiveObjectPrivilege; +import org.apache.hadoop.hive.metastore.api.HiveObjectRef; +import org.apache.hadoop.hive.metastore.api.HiveObjectType; +import org.apache.hadoop.hive.metastore.api.InsertEventRequestData; +import org.apache.hadoop.hive.metastore.api.InvalidOperationException; +import org.apache.hadoop.hive.metastore.api.Materialization; +import org.apache.hadoop.hive.metastore.api.MetaException; +import org.apache.hadoop.hive.metastore.api.MetadataPpdResult; +import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; +import org.apache.hadoop.hive.metastore.api.NotNullConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.PrimaryKeysRequest; +import org.apache.hadoop.hive.metastore.api.PrincipalPrivilegeSet; +import org.apache.hadoop.hive.metastore.api.PrincipalType; +import org.apache.hadoop.hive.metastore.api.PrivilegeBag; +import org.apache.hadoop.hive.metastore.api.Role; +import org.apache.hadoop.hive.metastore.api.RolePrincipalGrant; +import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; +import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; +import org.apache.hadoop.hive.metastore.api.SQLForeignKey; +import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; +import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; +import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; +import org.apache.hadoop.hive.metastore.api.SetPartitionsStatsRequest; +import org.apache.hadoop.hive.metastore.api.ShowCompactResponse; +import org.apache.hadoop.hive.metastore.api.SkewedInfo; +import org.apache.hadoop.hive.metastore.api.UniqueConstraintsRequest; +import org.apache.hadoop.hive.metastore.api.WMFullResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMMapping; +import org.apache.hadoop.hive.metastore.api.WMNullablePool; +import org.apache.hadoop.hive.metastore.api.WMNullableResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMPool; +import org.apache.hadoop.hive.metastore.api.WMResourcePlan; +import org.apache.hadoop.hive.metastore.api.WMTrigger; +import org.apache.hadoop.hive.metastore.api.WMValidateResourcePlanResponse; +import org.apache.hadoop.hive.metastore.api.WriteNotificationLogRequest; +import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.metastore.ReplChangeManager; -import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; @@ -3322,7 +3378,7 @@ private void constructOneLBLocationMap(FileStatus fSta, } public List<Partition> dropPartitions(String dbName, String tableName, - List<org.apache.hadoop.hive.metastore.utils.ObjectPair<Integer, byte[]>> partitionExpressions, + List<Pair<Integer, byte[]>> partitionExpressions, PartitionDropOptions dropOptions) throws HiveException { try { Table table = getTable(dbName, tableName); @@ -3810,7 +3866,7 @@ private void constructOneLBLocationMap(FileStatus fSta, if (!fullDestStatus.getFileStatus().isDirectory()) { throw new HiveException(destf + " is not a directory."); } - final List<Future<ObjectPair<Path, Path>>> futures = new LinkedList<>(); + final List<Future<Pair<Path, Path>>> futures = new LinkedList<>(); final ExecutorService pool = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25) > 0 ? Executors.newFixedThreadPool(conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 25), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Move-Thread-%d").build()) : null; @@ -3864,9 +3920,9 @@ private void constructOneLBLocationMap(FileStatus fSta, } else { // future only takes final or seemingly final values. Make a final copy of taskId final int finalTaskId = acidRename ? taskId++ : -1; - futures.add(pool.submit(new Callable<ObjectPair<Path, Path>>() { + futures.add(pool.submit(new Callable<Pair<Path, Path>>() { @Override - public ObjectPair<Path, Path> call() throws HiveException { + public Pair<Path, Path> call() throws HiveException { SessionState.setCurrentSessionState(parentSession); try { @@ -3876,7 +3932,7 @@ private void constructOneLBLocationMap(FileStatus fSta, if (null != newFiles) { newFiles.add(destPath); } - return ObjectPair.create(srcP, destPath); + return Pair.of(srcP, destPath); } catch (Exception e) { throw getHiveException(e, msg); } @@ -3887,10 +3943,10 @@ private void constructOneLBLocationMap(FileStatus fSta, } if (null != pool) { pool.shutdown(); - for (Future<ObjectPair<Path, Path>> future : futures) { + for (Future<Pair<Path, Path>> future : futures) { try { - ObjectPair<Path, Path> pair = future.get(); - LOG.debug("Moved src: {}, to dest: {}", pair.getFirst().toString(), pair.getSecond().toString()); + Pair<Path, Path> pair = future.get(); + LOG.debug("Moved src: {}, to dest: {}", pair.getLeft().toString(), pair.getRight().toString()); } catch (Exception e) { throw handlePoolException(pool, e); } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java index de61be8..c937b62 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java @@ -522,7 +522,7 @@ public class ConvertJoinMapJoin implements NodeProcessor { mapJoinDesc = new MapJoinDesc( MapJoinProcessor.getKeys(joinOp.getConf().isLeftInputJoin(), - joinOp.getConf().getBaseSrc(), joinOp).getSecond(), + joinOp.getConf().getBaseSrc(), joinOp).getRight(), null, joinDesc.getExprs(), null, null, joinDesc.getOutputColumnNames(), mapJoinConversionPos, joinDesc.getConds(), joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java index 5ed43c7..a9506c8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java @@ -30,8 +30,8 @@ import java.util.Map.Entry; import java.util.Set; import java.util.Stack; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.AbstractMapJoinOperator; @@ -561,8 +561,6 @@ public class MapJoinProcessor extends Transform { public static boolean isFullOuterEnabledForDynamicPartitionHashJoin(HiveConf hiveConf, JoinOperator joinOp) throws SemanticException { - JoinDesc joinDesc = joinOp.getConf(); - return true; } @@ -1230,7 +1228,7 @@ public class MapJoinProcessor extends Transform { } - public static ObjectPair<List<ReduceSinkOperator>, Map<Byte, List<ExprNodeDesc>>> getKeys( + public static Pair<List<ReduceSinkOperator>, Map<Byte, List<ExprNodeDesc>>> getKeys( boolean leftInputJoin, String[] baseSrc, JoinOperator op) { // Walk over all the sources (which are guaranteed to be reduce sink @@ -1264,8 +1262,7 @@ public class MapJoinProcessor extends Transform { keyExprMap.put(pos, keyCols); } - return new ObjectPair<List<ReduceSinkOperator>, Map<Byte,List<ExprNodeDesc>>>( - oldReduceSinkParentOps, keyExprMap); + return Pair.of(oldReduceSinkParentOps, keyExprMap); } public static MapJoinDesc getMapJoinDesc(HiveConf hconf, @@ -1287,9 +1284,8 @@ public class MapJoinProcessor extends Transform { Map<Byte, List<ExprNodeDesc>> valueExprs = op.getConf().getExprs(); Map<Byte, List<ExprNodeDesc>> newValueExprs = new HashMap<Byte, List<ExprNodeDesc>>(); - ObjectPair<List<ReduceSinkOperator>, Map<Byte,List<ExprNodeDesc>>> pair = - getKeys(leftInputJoin, baseSrc, op); - List<ReduceSinkOperator> oldReduceSinkParentOps = pair.getFirst(); + Pair<List<ReduceSinkOperator>, Map<Byte, List<ExprNodeDesc>>> pair = getKeys(leftInputJoin, baseSrc, op); + List<ReduceSinkOperator> oldReduceSinkParentOps = pair.getLeft(); for (Map.Entry<Byte, List<ExprNodeDesc>> entry : valueExprs.entrySet()) { byte tag = entry.getKey(); Operator<?> terminal = oldReduceSinkParentOps.get(tag); @@ -1318,7 +1314,7 @@ public class MapJoinProcessor extends Transform { Map<Byte, int[]> valueIndices = new HashMap<Byte, int[]>(); // get the join keys from old parent ReduceSink operators - Map<Byte, List<ExprNodeDesc>> keyExprMap = pair.getSecond(); + Map<Byte, List<ExprNodeDesc>> keyExprMap = pair.getRight(); if (!adjustParentsChildren) { // Since we did not remove reduce sink parents, keep the original value expressions diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java index ad1dec9..58b90d8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java @@ -26,7 +26,7 @@ import java.util.Map; import java.util.Set; import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; import org.apache.hadoop.hive.ql.exec.Operator; @@ -55,8 +55,7 @@ public class LineageCtx implements NodeProcessorCtx { * dependency vector for that tuple. This is used to generate the * dependency vectors during the walk of the operator tree. */ - private final Map<Operator<? extends OperatorDesc>, - LinkedHashMap<ColumnInfo, Dependency>> depMap; + private final Map<Operator<? extends OperatorDesc>, Map<ColumnInfo, Dependency>> depMap; /** * A map from operator to the conditions strings. @@ -67,18 +66,17 @@ public class LineageCtx implements NodeProcessorCtx { * A map from a final select operator id to the select operator * and the corresponding target table in case an insert into query. */ - private LinkedHashMap<String, ObjectPair<SelectOperator, Table>> finalSelectOps; + private Map<String, Pair<SelectOperator, Table>> finalSelectOps; /** * Constructor. */ public Index() { depMap = - new LinkedHashMap<Operator<? extends OperatorDesc>, - LinkedHashMap<ColumnInfo, Dependency>>(); + new LinkedHashMap<Operator<? extends OperatorDesc>, Map<ColumnInfo, Dependency>>(); condMap = new HashMap<Operator<? extends OperatorDesc>, Set<Predicate>>(); finalSelectOps = - new LinkedHashMap<String, ObjectPair<SelectOperator, Table>>(); + new LinkedHashMap<String, Pair<SelectOperator, Table>>(); } /** @@ -128,7 +126,7 @@ public class LineageCtx implements NodeProcessorCtx { */ public void putDependency(Operator<? extends OperatorDesc> op, ColumnInfo col, Dependency dep) { - LinkedHashMap<ColumnInfo, Dependency> colMap = depMap.get(op); + Map<ColumnInfo, Dependency> colMap = depMap.get(op); if (colMap == null) { colMap = new LinkedHashMap<ColumnInfo, Dependency>(); depMap.put(op, colMap); @@ -204,13 +202,11 @@ public class LineageCtx implements NodeProcessorCtx { FileSinkOperator fso = (FileSinkOperator) sinkOp; table = fso.getConf().getTable(); } - finalSelectOps.put(operatorId, - new ObjectPair<SelectOperator, Table>(sop, table)); + finalSelectOps.put(operatorId, Pair.of(sop, table)); } } - public LinkedHashMap<String, - ObjectPair<SelectOperator, Table>> getFinalSelectOps() { + public Map<String, Pair<SelectOperator, Table>> getFinalSelectOps() { return finalSelectOps; } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java index 636e0a2..1b9db01 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartExprEvalUtils.java @@ -24,7 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Properties; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator; import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory; @@ -102,7 +102,7 @@ public class PartExprEvalUtils { .getPrimitiveJavaObject(evaluateResultO); } - static public ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> prepareExpr( + public static Pair<PrimitiveObjectInspector, ExprNodeEvaluator> prepareExpr( ExprNodeGenericFuncDesc expr, List<String> partColumnNames, List<PrimitiveTypeInfo> partColumnTypeInfos) throws HiveException { // Create the row object @@ -116,12 +116,12 @@ public class PartExprEvalUtils { ExprNodeEvaluator evaluator = ExprNodeEvaluatorFactory.get(expr); ObjectInspector evaluateResultOI = evaluator.initialize(objectInspector); - return ObjectPair.create((PrimitiveObjectInspector)evaluateResultOI, evaluator); + return Pair.of((PrimitiveObjectInspector)evaluateResultOI, evaluator); } static public Object evaluateExprOnPart( - ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> pair, Object partColValues) + Pair<PrimitiveObjectInspector, ExprNodeEvaluator> pair, Object partColValues) throws HiveException { - return pair.getFirst().getPrimitiveJavaObject(pair.getSecond().evaluate(partColValues)); + return pair.getLeft().getPrimitiveJavaObject(pair.getRight().evaluate(partColValues)); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java index 7c8d752..673d858 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.StrictChecks; import org.apache.hadoop.hive.metastore.IMetaStoreClient; @@ -551,7 +551,7 @@ public class PartitionPruner extends Transform { List<PrimitiveTypeInfo> partColumnTypeInfos, ExprNodeGenericFuncDesc prunerExpr, String defaultPartitionName, List<String> partNames) throws HiveException, MetaException { // Prepare the expression to filter on the columns. - ObjectPair<PrimitiveObjectInspector, ExprNodeEvaluator> handle = + Pair<PrimitiveObjectInspector, ExprNodeEvaluator> handle = PartExprEvalUtils.prepareExpr(prunerExpr, partColumnNames, partColumnTypeInfos); // Filter the name list. Removing elements one by one can be slow on e.g. ArrayList, diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java index ab87c79..341422f 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SetSparkReducerParallelism.java @@ -27,7 +27,7 @@ import java.util.Stack; import org.apache.hadoop.hive.ql.plan.ExprNodeDesc; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -66,7 +66,7 @@ public class SetSparkReducerParallelism implements NodeProcessor { private static final String SPARK_DYNAMIC_ALLOCATION_ENABLED = "spark.dynamicAllocation.enabled"; // Spark memory per task, and total number of cores - private ObjectPair<Long, Integer> sparkMemoryAndCores; + private Pair<Long, Integer> sparkMemoryAndCores; private final boolean useOpStats; public SetSparkReducerParallelism(HiveConf conf) { @@ -169,15 +169,15 @@ public class SetSparkReducerParallelism implements NodeProcessor { getSparkMemoryAndCores(context); if (sparkMemoryAndCores != null && - sparkMemoryAndCores.getFirst() > 0 && sparkMemoryAndCores.getSecond() > 0) { + sparkMemoryAndCores.getLeft() > 0 && sparkMemoryAndCores.getRight() > 0) { // warn the user if bytes per reducer is much larger than memory per task - if ((double) sparkMemoryAndCores.getFirst() / bytesPerReducer < 0.5) { + if ((double) sparkMemoryAndCores.getLeft() / bytesPerReducer < 0.5) { LOG.warn("Average load of a reducer is much larger than its available memory. " + "Consider decreasing hive.exec.reducers.bytes.per.reducer"); } // If there are more cores, use the number of cores - numReducers = Math.max(numReducers, sparkMemoryAndCores.getSecond()); + numReducers = Math.max(numReducers, sparkMemoryAndCores.getRight()); } numReducers = Math.min(numReducers, maxReducers); LOG.info("Set parallelism for reduce sink " + sink + " to: " + numReducers + diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java index 8d9718f..43dfcee 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java @@ -3825,7 +3825,7 @@ public class CalcitePlanner extends SemanticAnalyzer { int groupingColsSize = gbExprNDescLst.size(); List<Long> groupingSets = null; if (cubeRollupGrpSetPresent) { - groupingSets = getGroupByGroupingSetsForClause(qbp, detsClauseName).getSecond(); + groupingSets = getGroupByGroupingSetsForClause(qbp, detsClauseName).getRight(); } // 6. Construct aggregation function Info diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java index 51448a4..200e814 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/QBSubQuery.java @@ -23,7 +23,7 @@ import java.util.List; import java.util.Set; import java.util.Stack; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.Context; import org.apache.hadoop.hive.ql.ErrorMsg; import org.apache.hadoop.hive.ql.exec.ColumnInfo; @@ -279,14 +279,14 @@ public class QBSubQuery implements ISubQueryJoinInfo { * 3. All other expressions have a Type based on their children. * An Expr w/o children is assumed to refer to neither. */ - private ObjectPair<ExprType,ColumnInfo> analyzeExpr(ASTNode expr) { + private Pair<ExprType, ColumnInfo> analyzeExpr(ASTNode expr) { ColumnInfo cInfo = null; if ( forHavingClause ) { try { cInfo = parentQueryRR.getExpression(expr); if ( cInfo != null) { - return ObjectPair.create(ExprType.REFERS_PARENT, cInfo); - } + return Pair.of(ExprType.REFERS_PARENT, cInfo); + } } catch(SemanticException se) { } } @@ -294,19 +294,19 @@ public class QBSubQuery implements ISubQueryJoinInfo { ASTNode dot = firstDot(expr); cInfo = resolveDot(dot); if ( cInfo != null ) { - return ObjectPair.create(ExprType.REFERS_PARENT, cInfo); + return Pair.of(ExprType.REFERS_PARENT, cInfo); } - return ObjectPair.create(ExprType.REFERS_SUBQUERY, null); + return Pair.of(ExprType.REFERS_SUBQUERY, null); } else if ( expr.getType() == HiveParser.TOK_TABLE_OR_COL ) { - return ObjectPair.create(ExprType.REFERS_SUBQUERY, null); + return Pair.of(ExprType.REFERS_SUBQUERY, null); } else { ExprType exprType = ExprType.REFERS_NONE; int cnt = expr.getChildCount(); for(int i=0; i < cnt; i++) { ASTNode child = (ASTNode) expr.getChild(i); - exprType = exprType.combine(analyzeExpr(child).getFirst()); + exprType = exprType.combine(analyzeExpr(child).getLeft()); } - return ObjectPair.create(exprType, null); + return Pair.of(exprType, null); } } @@ -321,17 +321,17 @@ public class QBSubQuery implements ISubQueryJoinInfo { if(conjunct.getChildCount() == 2) { ASTNode left = (ASTNode) conjunct.getChild(0); ASTNode right = (ASTNode) conjunct.getChild(1); - ObjectPair<ExprType,ColumnInfo> leftInfo = analyzeExpr(left); - ObjectPair<ExprType,ColumnInfo> rightInfo = analyzeExpr(right); + Pair<ExprType, ColumnInfo> leftInfo = analyzeExpr(left); + Pair<ExprType, ColumnInfo> rightInfo = analyzeExpr(right); return new Conjunct(left, right, - leftInfo.getFirst(), rightInfo.getFirst(), - leftInfo.getSecond(), rightInfo.getSecond()); + leftInfo.getLeft(), rightInfo.getLeft(), + leftInfo.getRight(), rightInfo.getRight()); } else { - ObjectPair<ExprType,ColumnInfo> sqExprInfo = analyzeExpr(conjunct); + Pair<ExprType, ColumnInfo> sqExprInfo = analyzeExpr(conjunct); return new Conjunct(conjunct, null, - sqExprInfo.getFirst(), null, - sqExprInfo.getSecond(), sqExprInfo.getSecond()); + sqExprInfo.getLeft(), null, + sqExprInfo.getRight(), sqExprInfo.getRight()); } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java index 4548276..050d09d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java @@ -67,7 +67,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.hive.common.FileUtils; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.common.StatsSetupConst; import org.apache.hadoop.hive.common.StatsSetupConst.StatDB; import org.apache.hadoop.hive.common.StringInternUtils; @@ -2094,8 +2093,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // For eg: for a query like 'select * from V3', where V3 -> V2, V2 -> V1, V1 -> T // keeps track of full view name and read entity corresponding to alias V3, V3:V2, V3:V2:V1. // This is needed for tracking the dependencies for inputs, along with their parents. - Map<String, ObjectPair<String, ReadEntity>> aliasToViewInfo = - new HashMap<String, ObjectPair<String, ReadEntity>>(); + Map<String, Pair<String, ReadEntity>> aliasToViewInfo = + new HashMap<String, Pair<String, ReadEntity>>(); /* * used to capture view to SQ conversions. This is used to check for @@ -2171,7 +2170,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } ReadEntity viewInput = new ReadEntity(tab, parentInput, !qb.isInsideView()); viewInput = PlanUtils.addInput(inputs, viewInput); - aliasToViewInfo.put(alias, new ObjectPair<String, ReadEntity>(fullViewName, viewInput)); + aliasToViewInfo.put(alias, Pair.of(fullViewName, viewInput)); String aliasId = getAliasId(alias, qb); if (aliasId != null) { aliasId = aliasId.replace(SemanticAnalyzer.SUBQUERY_TAG_1, "") @@ -2221,8 +2220,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { boolean wasCTE = sqAliasToCTEName.containsKey(alias); ReadEntity newParentInput = null; if (wasView) { - viewsExpanded.add(aliasToViewInfo.get(alias).getFirst()); - newParentInput = aliasToViewInfo.get(alias).getSecond(); + viewsExpanded.add(aliasToViewInfo.get(alias).getLeft()); + newParentInput = aliasToViewInfo.get(alias).getRight(); } else if (wasCTE) { ctesExpanded.add(sqAliasToCTEName.get(alias)); } @@ -4137,7 +4136,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // This function returns the grouping sets along with the grouping expressions // Even if rollups and cubes are present in the query, they are converted to // grouping sets at this point - ObjectPair<List<ASTNode>, List<Long>> getGroupByGroupingSetsForClause( + Pair<List<ASTNode>, List<Long>> getGroupByGroupingSetsForClause( QBParseInfo parseInfo, String dest) throws SemanticException { List<Long> groupingSets = new ArrayList<Long>(); List<ASTNode> groupByExprs = getGroupByForClause(parseInfo, dest); @@ -4154,7 +4153,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_SIZE_LIMIT.getMsg()); } - return new ObjectPair<List<ASTNode>, List<Long>>(groupByExprs, groupingSets); + return Pair.of(groupByExprs, groupingSets); } protected List<Long> getGroupingSets(List<ASTNode> groupByExpr, QBParseInfo parseInfo, @@ -6115,11 +6114,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QBParseInfo parseInfo = qb.getParseInfo(); int numReducers = -1; - ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = - getGroupByGroupingSetsForClause(parseInfo, dest); + Pair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); - List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); + List<ASTNode> grpByExprs = grpByExprsGroupingSets.getLeft(); + List<Long> groupingSets = grpByExprsGroupingSets.getRight(); if (grpByExprs.isEmpty()) { numReducers = 1; @@ -6164,10 +6162,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { List<ExprNodeDesc.ExprNodeDescEqualityWrapper> whereExpressions = new ArrayList<ExprNodeDesc.ExprNodeDescEqualityWrapper>(); for (String dest : dests) { - ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = + Pair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); - List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); + List<Long> groupingSets = grpByExprsGroupingSets.getRight(); if (!groupingSets.isEmpty()) { throw new SemanticException(ErrorMsg.HIVE_GROUPING_SETS_AGGR_NOMAPAGGR_MULTIGBY.getMsg()); } @@ -6301,11 +6299,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QBParseInfo parseInfo = qb.getParseInfo(); - ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = - getGroupByGroupingSetsForClause(parseInfo, dest); + Pair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); - List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); + List<ASTNode> grpByExprs = grpByExprsGroupingSets.getLeft(); + List<Long> groupingSets = grpByExprsGroupingSets.getRight(); // Grouping sets are not allowed // This restriction can be lifted in future. @@ -6496,11 +6493,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { Operator inputOperatorInfo) throws SemanticException { QBParseInfo parseInfo = qb.getParseInfo(); - ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = - getGroupByGroupingSetsForClause(parseInfo, dest); + Pair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); - List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); + List<ASTNode> grpByExprs = grpByExprsGroupingSets.getLeft(); + List<Long> groupingSets = grpByExprsGroupingSets.getRight(); boolean groupingSetsPresent = !groupingSets.isEmpty(); int newMRJobGroupingSetsThreshold = @@ -6665,11 +6661,10 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { QBParseInfo parseInfo = qb.getParseInfo(); - ObjectPair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = - getGroupByGroupingSetsForClause(parseInfo, dest); + Pair<List<ASTNode>, List<Long>> grpByExprsGroupingSets = getGroupByGroupingSetsForClause(parseInfo, dest); - List<ASTNode> grpByExprs = grpByExprsGroupingSets.getFirst(); - List<Long> groupingSets = grpByExprsGroupingSets.getSecond(); + List<ASTNode> grpByExprs = grpByExprsGroupingSets.getLeft(); + List<Long> groupingSets = grpByExprsGroupingSets.getRight(); boolean groupingSetsPresent = !groupingSets.isEmpty(); if (groupingSetsPresent) { @@ -10433,12 +10428,12 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } - private ObjectPair<Integer, int[]> findMergePos(QBJoinTree node, QBJoinTree target) { + private Pair<Integer, int[]> findMergePos(QBJoinTree node, QBJoinTree target) { int res = -1; String leftAlias = node.getLeftAlias(); if (leftAlias == null && (!node.getNoOuterJoin() || !target.getNoOuterJoin())) { // Cross with outer join: currently we do not merge - return new ObjectPair(-1, null); + return Pair.of(-1, null); } ArrayList<ASTNode> nodeCondn = node.getExpressions().get(0); @@ -10458,7 +10453,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } if ( targetCondn == null || (nodeCondn.size() != targetCondn.size())) { - return new ObjectPair(-1, null); + return Pair.of(-1, null); } /* @@ -10481,17 +10476,17 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { } } if ( tgtToNodeExprMap[i] == -1) { - return new ObjectPair(-1, null); + return Pair.of(-1, null); } } for(j=0; j < nodeCondn.size(); j++) { if ( !nodeFiltersMapped[j]) { - return new ObjectPair(-1, null); + return Pair.of(-1, null); } } - return new ObjectPair(res, tgtToNodeExprMap); + return Pair.of(res, tgtToNodeExprMap); } boolean isCBOExecuted() { @@ -10565,8 +10560,8 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { // Outer joins with post-filtering conditions cannot be merged break; } - ObjectPair<Integer, int[]> mergeDetails = findMergePos(node, target); - int pos = mergeDetails.getFirst(); + Pair<Integer, int[]> mergeDetails = findMergePos(node, target); + int pos = mergeDetails.getLeft(); if (pos >= 0) { // for outer joins, it should not exceed 16 aliases (short type) if (!node.getNoOuterJoin() || !target.getNoOuterJoin()) { @@ -10576,7 +10571,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer { continue; } } - mergeJoins(qb, node, target, pos, mergeDetails.getSecond()); + mergeJoins(qb, node, target, pos, mergeDetails.getRight()); trees.set(j, null); mergedQBJTree = true; continue; // continue merging with next alias diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java index ede9abc..e60f715 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkProcContext.java @@ -18,8 +18,8 @@ package org.apache.hadoop.hive.ql.parse.spark; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.DependencyCollectionTask; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -92,8 +92,7 @@ public class GenSparkProcContext implements NodeProcessorCtx { // map that keeps track of the last operator of a task to the following work // of this operator. This is used for connecting them later. - public final Map<ReduceSinkOperator, ObjectPair<SparkEdgeProperty, ReduceWork>> - leafOpToFollowingWorkInfo; + public final Map<ReduceSinkOperator, Pair<SparkEdgeProperty, ReduceWork>> leafOpToFollowingWorkInfo; // a map that keeps track of work that need to be linked while // traversing an operator tree @@ -149,7 +148,6 @@ public class GenSparkProcContext implements NodeProcessorCtx { public final Set<Operator<?>> clonedPruningTableScanSet; - @SuppressWarnings("unchecked") public GenSparkProcContext(HiveConf conf, ParseContext parseContext, List<Task<MoveWork>> moveTask, @@ -167,7 +165,7 @@ public class GenSparkProcContext implements NodeProcessorCtx { this.currentTask = SparkUtilities.createSparkTask(conf); this.rootTasks.add(currentTask); this.leafOpToFollowingWorkInfo = - new LinkedHashMap<ReduceSinkOperator, ObjectPair<SparkEdgeProperty, ReduceWork>>(); + new LinkedHashMap<ReduceSinkOperator, Pair<SparkEdgeProperty, ReduceWork>>(); this.linkOpWithWorkMap = new LinkedHashMap<Operator<?>, Map<BaseWork, SparkEdgeProperty>>(); this.linkWorkWithReduceSinkMap = new LinkedHashMap<BaseWork, List<ReduceSinkOperator>>(); this.smbMapJoinCtxMap = new HashMap<SMBMapJoinOperator, SparkSMBMapJoinInfo>(); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java index 04b1202..4fcc0c2 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/spark/GenSparkWork.java @@ -27,19 +27,17 @@ import java.util.Stack; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator; import org.apache.hadoop.hive.ql.exec.MapJoinOperator; import org.apache.hadoop.hive.ql.exec.Operator; import org.apache.hadoop.hive.ql.exec.OperatorFactory; import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator; import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator; -import org.apache.hadoop.hive.ql.exec.TableScanOperator; import org.apache.hadoop.hive.ql.lib.Node; import org.apache.hadoop.hive.ql.lib.NodeProcessor; import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx; import org.apache.hadoop.hive.ql.optimizer.GenMapRedUtils; -import org.apache.hadoop.hive.ql.optimizer.spark.SparkSortMergeJoinFactory; import org.apache.hadoop.hive.ql.parse.SemanticException; import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.plan.MapWork; @@ -216,7 +214,7 @@ public class GenSparkWork implements NodeProcessor { rsOp.getConf().setOutputName(reduceWork.getName()); GenMapRedUtils.setKeyAndValueDesc(reduceWork, rsOp); - context.leafOpToFollowingWorkInfo.put(rsOp, ObjectPair.create(edgeProp, reduceWork)); + context.leafOpToFollowingWorkInfo.put(rsOp, Pair.of(edgeProp, reduceWork)); LOG.debug("Removing " + parent + " as parent from " + root); root.removeParent(parent); } @@ -242,10 +240,9 @@ public class GenSparkWork implements NodeProcessor { // Also note: the concept of leaf and root is reversed in hive for historical // reasons. Roots are data sources, leaves are data sinks. I know. if (context.leafOpToFollowingWorkInfo.containsKey(operator)) { - ObjectPair<SparkEdgeProperty, ReduceWork> childWorkInfo = context. - leafOpToFollowingWorkInfo.get(operator); - SparkEdgeProperty edgeProp = childWorkInfo.getFirst(); - ReduceWork childWork = childWorkInfo.getSecond(); + Pair<SparkEdgeProperty, ReduceWork> childWorkInfo = context.leafOpToFollowingWorkInfo.get(operator); + SparkEdgeProperty edgeProp = childWorkInfo.getLeft(); + ReduceWork childWork = childWorkInfo.getRight(); LOG.debug("Second pass. Leaf operator: " + operator + " has common downstream work:" + childWork); diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java index 251b31a..ef33054 100644 --- a/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java +++ b/serde/src/test/org/apache/hadoop/hive/serde2/objectinspector/TestReflectionObjectInspectors.java @@ -29,10 +29,9 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import org.apache.commons.lang.mutable.MutableObject; -import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category; import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaConstantStringObjectInspector; -import org.apache.hadoop.hive.serde2.objectinspector.primitive.JavaStringObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; import org.apache.hadoop.hive.serde2.thrift.test.Complex; import com.google.common.collect.Lists; @@ -146,23 +145,21 @@ public class TestReflectionObjectInspectors { @SuppressWarnings("unchecked") public void run() { Future<ObjectInspector>[] results = (Future<ObjectInspector>[])new Future[workerCount]; - ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>[] types = - (ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>[])new ObjectPair[] { - new ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>(Complex.class, - ObjectInspectorFactory.ObjectInspectorOptions.THRIFT), - new ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions>(MyStruct.class, - ObjectInspectorFactory.ObjectInspectorOptions.JAVA), + Pair<Type, ObjectInspectorFactory.ObjectInspectorOptions>[] types = + (Pair<Type, ObjectInspectorFactory.ObjectInspectorOptions>[])new Pair[] { + Pair.of(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT), + Pair.of(MyStruct.class, ObjectInspectorFactory.ObjectInspectorOptions.JAVA), }; try { for (int i = 0; i < 20; i++) { // repeat 20 times - for (final ObjectPair<Type, ObjectInspectorFactory.ObjectInspectorOptions> t: types) { + for (final Pair<Type, ObjectInspectorFactory.ObjectInspectorOptions> t: types) { ObjectInspectorFactory.objectInspectorCache.asMap().clear(); for (int k = 0; k < workerCount; k++) { results[k] = executorService.schedule(new Callable<ObjectInspector>() { @Override public ObjectInspector call() throws Exception { return ObjectInspectorFactory.getReflectionObjectInspector( - t.getFirst(), t.getSecond()); + t.getLeft(), t.getRight()); } }, 50, TimeUnit.MILLISECONDS); } diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java index 12de51c..a906f61 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClient.java @@ -19,9 +19,7 @@ package org.apache.hadoop.hive.metastore; import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_DATABASE_NAME; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.CAT_NAME; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.parseDbName; import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.prependCatalogToDbName; import java.io.IOException; @@ -54,6 +52,7 @@ import javax.security.auth.login.LoginException; import com.google.common.base.Preconditions; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -70,7 +69,6 @@ import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.utils.FilterUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.hive.metastore.utils.LogUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -319,7 +317,6 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { List<URI> metastoreURIArray = new ArrayList<URI>(); try { - int i = 0; for (String s : metastoreUrisString) { URI tmpUri = new URI(s); if (tmpUri.getScheme() == null) { @@ -1481,7 +1478,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, + List<Pair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws TException { return dropPartitions(getDefaultCatalog(conf), dbName, tblName, partExprs, options); @@ -1489,7 +1486,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists, boolean needResult) throws NoSuchObjectException, MetaException, TException { return dropPartitions(getDefaultCatalog(conf), dbName, tblName, partExprs, @@ -1502,7 +1499,7 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists) throws NoSuchObjectException, MetaException, TException { // By default, we need the results from dropPartitions(); return dropPartitions(getDefaultCatalog(conf), dbName, tblName, partExprs, @@ -1513,14 +1510,14 @@ public class HiveMetaStoreClient implements IMetaStoreClient, AutoCloseable { @Override public List<Partition> dropPartitions(String catName, String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, + List<Pair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws TException { RequestPartsSpec rps = new RequestPartsSpec(); List<DropPartitionsExpr> exprs = new ArrayList<>(partExprs.size()); - for (ObjectPair<Integer, byte[]> partExpr : partExprs) { + for (Pair<Integer, byte[]> partExpr : partExprs) { DropPartitionsExpr dpe = new DropPartitionsExpr(); - dpe.setExpr(partExpr.getSecond()); - dpe.setPartArchiveLevel(partExpr.getFirst()); + dpe.setExpr(partExpr.getRight()); + dpe.setPartArchiveLevel(partExpr.getLeft()); exprs.add(dpe); } rps.setExprs(exprs); diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java index 0129a89..6c7d80e 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/IMetaStoreClient.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -34,7 +35,6 @@ import org.apache.hadoop.hive.common.classification.RetrySemantics; import org.apache.hadoop.hive.metastore.annotation.NoReconnect; import org.apache.hadoop.hive.metastore.api.*; import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.thrift.TException; /** @@ -1892,7 +1892,7 @@ public interface IMetaStoreClient { * @throws TException Thrift transport error. */ List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists) throws NoSuchObjectException, MetaException, TException; /** @@ -1913,7 +1913,7 @@ public interface IMetaStoreClient { * @throws TException Thrift transport error. */ default List<Partition> dropPartitions(String catName, String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists) throws NoSuchObjectException, MetaException, TException { return dropPartitions(catName, dbName, tblName, partExprs, @@ -1943,7 +1943,7 @@ public interface IMetaStoreClient { */ @Deprecated List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists, boolean needResults) throws NoSuchObjectException, MetaException, TException; /** @@ -1966,7 +1966,7 @@ public interface IMetaStoreClient { * @throws TException Thrift transport error. */ default List<Partition> dropPartitions(String catName, String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists, boolean needResults) throws NoSuchObjectException, MetaException, TException { return dropPartitions(catName, dbName, tblName, partExprs, @@ -1988,7 +1988,7 @@ public interface IMetaStoreClient { * @throws TException On failure */ List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, + List<Pair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws NoSuchObjectException, MetaException, TException; @@ -2005,7 +2005,7 @@ public interface IMetaStoreClient { * @throws TException On failure */ List<Partition> dropPartitions(String catName, String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, + List<Pair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws NoSuchObjectException, MetaException, TException; diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/ObjectPair.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/ObjectPair.java deleted file mode 100644 index 5b49a25..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/utils/ObjectPair.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore.utils; - - - -public class ObjectPair<F, S> { - private F first; - private S second; - - public ObjectPair() {} - - /** - * Creates a pair. Constructor doesn't infer template args but - * the method does, so the code becomes less ugly. - */ - public static <T1, T2> ObjectPair<T1, T2> create(T1 f, T2 s) { - return new ObjectPair<>(f, s); - } - - public ObjectPair(F first, S second) { - this.first = first; - this.second = second; - } - - public F getFirst() { - return first; - } - - public void setFirst(F first) { - this.first = first; - } - - public S getSecond() { - return second; - } - - public void setSecond(S second) { - this.second = second; - } - - @Override - public boolean equals(Object that) { - if (that == null) { - return false; - } - if (that instanceof ObjectPair) { - return this.equals((ObjectPair<F, S>)that); - } - return false; - } - - public boolean equals(ObjectPair<F, S> that) { - if (that == null) { - return false; - } - - return this.getFirst().equals(that.getFirst()) && - this.getSecond().equals(that.getSecond()); - } - - @Override - public int hashCode() { - return first.hashCode() * 31 + second.hashCode(); - } - - public String toString() { - return first + ":" + second; - } -} diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java index d75d709..fab83b6 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java @@ -31,6 +31,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -45,7 +46,6 @@ import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.hadoop.hive.metastore.utils.RetryUtilities; import org.apache.hadoop.security.UserGroupInformation; import org.apache.thrift.TException; @@ -467,7 +467,7 @@ public class Msck { // so 3rd parameter (deleteData) is set to false // msck is doing a clean up of hms. if for some reason the partition is already // deleted, then it is good. So, the last parameter ifexists is set to true - List<ObjectPair<Integer, byte[]>> partExprs = getPartitionExpr(dropParts); + List<Pair<Integer, byte[]>> partExprs = getPartitionExpr(dropParts); metastoreClient.dropPartitions(table.getCatName(), table.getDbName(), table.getTableName(), partExprs, dropOptions); // if last batch is successful remove it from partsNotInFs @@ -480,8 +480,8 @@ public class Msck { } } - private List<ObjectPair<Integer, byte[]>> getPartitionExpr(final List<String> parts) throws MetaException { - List<ObjectPair<Integer, byte[]>> expr = new ArrayList<>(parts.size()); + private List<Pair<Integer, byte[]>> getPartitionExpr(final List<String> parts) throws MetaException { + List<Pair<Integer, byte[]>> expr = new ArrayList<>(parts.size()); for (int i = 0; i < parts.size(); i++) { String partName = parts.get(i); Map<String, String> partSpec = Warehouse.makeSpecFromName(partName); @@ -489,7 +489,7 @@ public class Msck { if (LOG.isDebugEnabled()) { LOG.debug("Generated partExpr: {} for partName: {}", partExpr, partName); } - expr.add(new ObjectPair<>(i, partExpr.getBytes(StandardCharsets.UTF_8))); + expr.add(Pair.of(i, partExpr.getBytes(StandardCharsets.UTF_8))); } return expr; } diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java index b40d816..ea6dc88 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java @@ -66,6 +66,7 @@ import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.ArrayUtils; import org.apache.commons.lang.StringUtils; import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configurable; @@ -90,7 +91,6 @@ import org.apache.hadoop.hive.metastore.utils.FileUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreServerUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.thrift.TException; import org.datanucleus.store.rdbms.exceptions.MissingTableException; import org.slf4j.Logger; @@ -134,7 +134,6 @@ public class ObjectStore implements RawStore, Configurable { public static final String TRUSTSTORE_PASSWORD_KEY = "javax.net.ssl.trustStorePassword"; public static final String TRUSTSTORE_TYPE_KEY = "javax.net.ssl.trustStoreType"; - private static final Map<String, Class<?>> PINCLASSMAP; private static final String HOSTNAME; private static final String USER; private static final String JDO_PARAM = ":param"; @@ -148,7 +147,6 @@ public class ObjectStore implements RawStore, Configurable { map.put("type", MType.class); map.put("fieldschema", MFieldSchema.class); map.put("order", MOrder.class); - PINCLASSMAP = Collections.unmodifiableMap(map); String hostname = "UNKNOWN"; try { InetAddress clientAddr = InetAddress.getLocalHost(); @@ -1231,11 +1229,6 @@ public class ObjectStore implements RawStore, Configurable { return mConstraints; } - private static String getFullyQualifiedTableName(String dbName, String tblName) { - return ((dbName == null || dbName.isEmpty()) ? "" : "\"" + dbName + "\".\"") - + "\"" + tblName + "\""; - } - @Override public Table getTable(String catName, String dbName, String tableName) @@ -3339,7 +3332,6 @@ public class ObjectStore implements RawStore, Configurable { @Override protected List<Partition> getSqlResult(GetHelper<List<Partition>> ctx) throws MetaException { // If we have some sort of expression tree, try SQL filter pushdown. - List<Partition> result = null; if (exprTree != null) { SqlFilterForPushdown filter = new SqlFilterForPushdown(); if (directSql.generateSqlFilterForPushdown(ctx.getTable(), exprTree, defaultPartitionName, filter)) { @@ -3476,16 +3468,16 @@ public class ObjectStore implements RawStore, Configurable { return Batchable.runBatched(batchSize, partNames, new Batchable<String, Partition>() { @Override public List<Partition> run(List<String> input) throws MetaException { - ObjectPair<Query, Map<String, String>> queryWithParams = + Pair<Query, Map<String, String>> queryWithParams = getPartQueryWithParams(catName, dbName, tblName, input); - Query query = queryWithParams.getFirst(); + Query query = queryWithParams.getLeft(); query.setResultClass(MPartition.class); query.setClass(MPartition.class); query.setOrdering("partitionName ascending"); @SuppressWarnings("unchecked") - List<MPartition> mparts = (List<MPartition>) query.executeWithMap(queryWithParams.getSecond()); + List<MPartition> mparts = (List<MPartition>) query.executeWithMap(queryWithParams.getRight()); List<Partition> partitions = convertToParts(catName, dbName, tblName, mparts); query.closeAll(); @@ -3495,11 +3487,11 @@ public class ObjectStore implements RawStore, Configurable { } private void dropPartitionsNoTxn(String catName, String dbName, String tblName, List<String> partNames) { - ObjectPair<Query, Map<String, String>> queryWithParams = + Pair<Query, Map<String, String>> queryWithParams = getPartQueryWithParams(catName, dbName, tblName, partNames); - Query query = queryWithParams.getFirst(); + Query query = queryWithParams.getLeft(); query.setClass(MPartition.class); - long deleted = query.deletePersistentAll(queryWithParams.getSecond()); + long deleted = query.deletePersistentAll(queryWithParams.getRight()); LOG.debug("Deleted {} partition from store", deleted); query.closeAll(); } @@ -3510,16 +3502,16 @@ public class ObjectStore implements RawStore, Configurable { * SDs; so, we remove the links to delete SDs and then check the returned CDs to see if * they are referenced by other SDs. */ - private HashSet<MColumnDescriptor> detachCdsFromSdsNoTxn( + private Set<MColumnDescriptor> detachCdsFromSdsNoTxn( String catName, String dbName, String tblName, List<String> partNames) { - ObjectPair<Query, Map<String, String>> queryWithParams = + Pair<Query, Map<String, String>> queryWithParams = getPartQueryWithParams(catName, dbName, tblName, partNames); - Query query = queryWithParams.getFirst(); + Query query = queryWithParams.getLeft(); query.setClass(MPartition.class); query.setResult("sd"); @SuppressWarnings("unchecked") List<MStorageDescriptor> sds = (List<MStorageDescriptor>)query.executeWithMap( - queryWithParams.getSecond()); + queryWithParams.getRight()); HashSet<MColumnDescriptor> candidateCds = new HashSet<>(); for (MStorageDescriptor sd : sds) { if (sd != null && sd.getCD() != null) { @@ -3527,9 +3519,7 @@ public class ObjectStore implements RawStore, Configurable { sd.setCD(null); } } - if (query != null) { - query.closeAll(); - } + query.closeAll(); return candidateCds; } @@ -3568,7 +3558,7 @@ public class ObjectStore implements RawStore, Configurable { return queryFilter.toString(); } - private ObjectPair<Query, Map<String, String>> getPartQueryWithParams( + private Pair<Query, Map<String, String>> getPartQueryWithParams( String catName, String dbName, String tblName, List<String> partNames) { Query query = pm.newQuery(); Map<String, String> params = new HashMap<>(); @@ -3576,7 +3566,7 @@ public class ObjectStore implements RawStore, Configurable { query.setFilter(filterStr); LOG.debug(" JDOQL filter is {}", filterStr); query.declareParameters(makeParameterDeclarationString(params)); - return new ObjectPair<>(query, params); + return Pair.of(query, params); } @Override @@ -4204,7 +4194,6 @@ public class ObjectStore implements RawStore, Configurable { public Table alterTable(String catName, String dbname, String name, Table newTable, String queryValidWriteIds) throws InvalidObjectException, MetaException { boolean success = false; - boolean registerCreationSignature = false; try { openTransaction(); name = normalizeIdentifier(name); @@ -5154,7 +5143,6 @@ public class ObjectStore implements RawStore, Configurable { throws InvalidObjectException, MetaException { List<String> nnNames = new ArrayList<>(); List<MConstraint> cstrs = new ArrayList<>(); - String constraintName = null; for (int i = 0; i < cc.size(); i++) { final String catName = normalizeIdentifier(cc.get(i).getCatName()); @@ -5229,7 +5217,6 @@ public class ObjectStore implements RawStore, Configurable { throws InvalidObjectException, MetaException { List<String> nnNames = new ArrayList<>(); List<MConstraint> cstrs = new ArrayList<>(); - String constraintName = null; for (int i = 0; i < nns.size(); i++) { final String catName = normalizeIdentifier(nns.get(i).getCatName()); @@ -7052,11 +7039,11 @@ public class ObjectStore implements RawStore, Configurable { private void dropPartitionAllColumnGrantsNoTxn( String catName, String dbName, String tableName, List<String> partNames) { - ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, + Pair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames, MPartitionColumnPrivilege.class, "partition.table.tableName", "partition.table.database.name", "partition.partitionName", "partition.table.database.catalogName"); - queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); + queryWithParams.getLeft().deletePersistentAll(queryWithParams.getRight()); } @SuppressWarnings("unchecked") @@ -7121,27 +7108,27 @@ public class ObjectStore implements RawStore, Configurable { private void dropPartitionGrantsNoTxn(String catName, String dbName, String tableName, List<String> partNames) { - ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, + Pair<Query, Object[]> queryWithParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames,MPartitionPrivilege.class, "partition.table.tableName", "partition.table.database.name", "partition.partitionName", "partition.table.database.catalogName"); - queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); + queryWithParams.getLeft().deletePersistentAll(queryWithParams.getRight()); } @SuppressWarnings("unchecked") private <T> List<T> queryByPartitionNames(String catName, String dbName, String tableName, List<String> partNames, Class<T> clazz, String tbCol, String dbCol, String partCol, String catCol) { - ObjectPair<Query, Object[]> queryAndParams = makeQueryByPartitionNames(catName, + Pair<Query, Object[]> queryAndParams = makeQueryByPartitionNames(catName, dbName, tableName, partNames, clazz, tbCol, dbCol, partCol, catCol); - return (List<T>)queryAndParams.getFirst().executeWithArray(queryAndParams.getSecond()); + return (List<T>)queryAndParams.getLeft().executeWithArray(queryAndParams.getRight()); } - private ObjectPair<Query, Object[]> makeQueryByPartitionNames( + private Pair<Query, Object[]> makeQueryByPartitionNames( String catName, String dbName, String tableName, List<String> partNames, Class<?> clazz, String tbCol, String dbCol, String partCol, String catCol) { - String queryStr = tbCol + " == t1 && " + dbCol + " == t2 && " + catCol + " == t3"; - String paramStr = "java.lang.String t1, java.lang.String t2, java.lang.String t3"; + StringBuilder queryStr = new StringBuilder(tbCol + " == t1 && " + dbCol + " == t2 && " + catCol + " == t3"); + StringBuilder paramStr = new StringBuilder("java.lang.String t1, java.lang.String t2, java.lang.String t3"); Object[] params = new Object[3 + partNames.size()]; params[0] = normalizeIdentifier(tableName); params[1] = normalizeIdentifier(dbName); @@ -7149,14 +7136,14 @@ public class ObjectStore implements RawStore, Configurable { int index = 0; for (String partName : partNames) { params[index + 3] = partName; - queryStr += ((index == 0) ? " && (" : " || ") + partCol + " == p" + index; - paramStr += ", java.lang.String p" + index; + queryStr.append(((index == 0) ? " && (" : " || ") + partCol + " == p" + index); + paramStr.append(", java.lang.String p" + index); ++index; } - queryStr += ")"; - Query query = pm.newQuery(clazz, queryStr); - query.declareParameters(paramStr); - return new ObjectPair<>(query, params); + queryStr.append(")"); + Query query = pm.newQuery(clazz, queryStr.toString()); + query.declareParameters(paramStr.toString()); + return Pair.of(query, params); } private List<MTablePrivilege> listAllMTableGrants( @@ -9220,10 +9207,10 @@ public class ObjectStore implements RawStore, Configurable { private void dropPartitionColumnStatisticsNoTxn( String catName, String dbName, String tableName, List<String> partNames) throws MetaException { - ObjectPair<Query, Object[]> queryWithParams = makeQueryByPartitionNames( + Pair<Query, Object[]> queryWithParams = makeQueryByPartitionNames( catName, dbName, tableName, partNames, MPartitionColumnStatistics.class, "tableName", "dbName", "partition.partitionName", "catName"); - queryWithParams.getFirst().deletePersistentAll(queryWithParams.getSecond()); + queryWithParams.getLeft().deletePersistentAll(queryWithParams.getRight()); } @Override diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java index 6aea508..afe7bc6 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/HiveMetaStoreClientPreCatalog.java @@ -52,6 +52,7 @@ import javax.security.auth.login.LoginException; import com.google.common.base.Preconditions; +import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; @@ -67,7 +68,6 @@ import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge; import org.apache.hadoop.hive.metastore.txn.TxnCommonUtils; import org.apache.hadoop.hive.metastore.utils.JavaUtils; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; -import org.apache.hadoop.hive.metastore.utils.ObjectPair; import org.apache.hadoop.hive.metastore.utils.SecurityUtils; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -230,7 +230,6 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos List<URI> metastoreURIArray = new ArrayList<URI>(); try { - int i = 0; for (String s : metastoreUrisString) { URI tmpUri = new URI(s); if (tmpUri.getScheme() == null) { @@ -257,7 +256,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos } if (MetastoreConf.getVar(conf, ConfVars.THRIFT_URI_SELECTION).equalsIgnoreCase("RANDOM")) { - List uriList = Arrays.asList(metastoreUris); + List<URI> uriList = Arrays.asList(metastoreUris); Collections.shuffle(uriList); metastoreUris = (URI[]) uriList.toArray(); } @@ -1024,14 +1023,14 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, PartitionDropOptions options) + List<Pair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws TException { RequestPartsSpec rps = new RequestPartsSpec(); List<DropPartitionsExpr> exprs = new ArrayList<>(partExprs.size()); - for (ObjectPair<Integer, byte[]> partExpr : partExprs) { + for (Pair<Integer, byte[]> partExpr : partExprs) { DropPartitionsExpr dpe = new DropPartitionsExpr(); - dpe.setExpr(partExpr.getSecond()); - dpe.setPartArchiveLevel(partExpr.getFirst()); + dpe.setExpr(partExpr.getRight()); + dpe.setPartArchiveLevel(partExpr.getLeft()); exprs.add(dpe); } rps.setExprs(exprs); @@ -1048,7 +1047,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists, boolean needResult) throws NoSuchObjectException, MetaException, TException { return dropPartitions(dbName, tblName, partExprs, @@ -1061,7 +1060,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public List<Partition> dropPartitions(String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, boolean deleteData, + List<Pair<Integer, byte[]>> partExprs, boolean deleteData, boolean ifExists) throws NoSuchObjectException, MetaException, TException { // By default, we need the results from dropPartitions(); return dropPartitions(dbName, tblName, partExprs, @@ -3373,7 +3372,7 @@ public class HiveMetaStoreClientPreCatalog implements IMetaStoreClient, AutoClos @Override public List<Partition> dropPartitions(String catName, String dbName, String tblName, - List<ObjectPair<Integer, byte[]>> partExprs, + List<Pair<Integer, byte[]>> partExprs, PartitionDropOptions options) throws NoSuchObjectException, MetaException, TException { throw new UnsupportedOperationException();