hive git commit: HIVE-20705: Vectorization: Native Vector MapJoin doesn't support Complex Big Table values (Matt McCline, reviewed by Jason Dere)
Repository: hive
Updated Branches:
refs/heads/master 64bef36a3 -> 37c7fd783
HIVE-20705: Vectorization: Native Vector MapJoin doesn't support Complex Big
Table values (Matt McCline, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/37c7fd78
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/37c7fd78
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/37c7fd78
Branch: refs/heads/master
Commit: 37c7fd7833eba087eadd8048dbc63b403b272104
Parents: 64bef36
Author: Jason Dere
Authored: Wed Oct 10 18:11:02 2018 -0700
Committer: Jason Dere
Committed: Wed Oct 10 18:11:02 2018 -0700
--
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/physical/Vectorizer.java | 18 +-
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 10 +
.../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 14 +
.../vector_mapjoin_complex_values.q | 34 ++
.../llap/vector_mapjoin_complex_values.q.out| 355 +++
6 files changed, 430 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/37c7fd78/itests/src/test/resources/testconfiguration.properties
--
diff --git a/itests/src/test/resources/testconfiguration.properties
b/itests/src/test/resources/testconfiguration.properties
index 97609cf..9a87464 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -800,6 +800,7 @@ minillaplocal.query.files=\
vector_like_2.q,\
vector_llap_io_data_conversion.q,\
vector_llap_text_1.q,\
+ vector_mapjoin_complex_values.q,\
vector_mapjoin_reduce.q,\
vector_null_map.q,\
vector_number_compare_projection.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/37c7fd78/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index e93d666..7f48dc5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -3623,6 +3623,9 @@ public class Vectorizer implements PhysicalPlanResolver {
*
* Value expressions include keys? YES.
*/
+boolean supportsValueTypes = true; // Assume.
+HashSet notSupportedValueTypes = new HashSet();
+
int[] bigTableValueColumnMap = new int[allBigTableValueExpressions.length];
String[] bigTableValueColumnNames = new
String[allBigTableValueExpressions.length];
TypeInfo[] bigTableValueTypeInfos = new
TypeInfo[allBigTableValueExpressions.length];
@@ -3637,7 +3640,13 @@ public class Vectorizer implements PhysicalPlanResolver {
ExprNodeDesc exprNode = bigTableExprs.get(i);
bigTableValueColumnNames[i] = exprNode.toString();
- bigTableValueTypeInfos[i] = exprNode.getTypeInfo();
+ TypeInfo typeInfo = exprNode.getTypeInfo();
+ if (!(typeInfo instanceof PrimitiveTypeInfo)) {
+supportsValueTypes = false;
+Category category = typeInfo.getCategory();
+notSupportedValueTypes.add(category.toString());
+ }
+ bigTableValueTypeInfos[i] = typeInfo;
}
if (bigTableValueExpressionsList.size() == 0) {
slimmedBigTableValueExpressions = null;
@@ -3880,6 +3889,10 @@ public class Vectorizer implements PhysicalPlanResolver {
if (!supportsKeyTypes) {
vectorDesc.setNotSupportedKeyTypes(new ArrayList(notSupportedKeyTypes));
}
+vectorDesc.setSupportsValueTypes(supportsValueTypes);
+if (!supportsValueTypes) {
+ vectorDesc.setNotSupportedValueTypes(new
ArrayList(notSupportedValueTypes));
+}
// Check common conditions for both Optimized and Fast Hash Tables.
boolean result = true;// Assume.
@@ -3889,7 +3902,8 @@ public class Vectorizer implements PhysicalPlanResolver {
!oneMapJoinCondition ||
hasNullSafes ||
!smallTableExprVectorizes ||
-outerJoinHasNoKeys) {
+outerJoinHasNoKeys ||
+!supportsValueTypes) {
result = false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/37c7fd78/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
--
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
index 507114b..093a629 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@
[3/3] hive git commit: HIVE-20306 : Addendum to remove .orig files
HIVE-20306 : Addendum to remove .orig files Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/64bef36a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/64bef36a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/64bef36a Branch: refs/heads/master Commit: 64bef36a36bcbb3dd644ffdc394673fdad41eb93 Parents: 390afb5 Author: Vihang Karajgaonkar Authored: Wed Oct 10 10:28:31 2018 -0700 Committer: Vihang Karajgaonkar Committed: Wed Oct 10 10:40:13 2018 -0700 -- .../hive/metastore/MetaStoreDirectSql.java.orig | 2845 .../hadoop/hive/metastore/ObjectStore.java.orig | 12514 - 2 files changed, 15359 deletions(-) --
[1/3] hive git commit: HIVE-20306 : Addendum to remove .orig files
Repository: hive Updated Branches: refs/heads/master 390afb5ef -> 64bef36a3 http://git-wip-us.apache.org/repos/asf/hive/blob/64bef36a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java.orig -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java.orig b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java.orig deleted file mode 100644 index 5372714..000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java.orig +++ /dev/null @@ -1,12514 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import static org.apache.commons.lang.StringUtils.join; -import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; -import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; - -import java.io.IOException; -import java.lang.reflect.Field; -import java.net.InetAddress; -import java.net.URI; -import java.nio.ByteBuffer; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.SQLIntegrityConstraintViolationException; -import java.sql.Statement; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Properties; -import java.util.Set; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import java.util.regex.Pattern; - -import javax.jdo.JDOCanRetryException; -import javax.jdo.JDODataStoreException; -import javax.jdo.JDOException; -import javax.jdo.JDOHelper; -import javax.jdo.JDOObjectNotFoundException; -import javax.jdo.PersistenceManager; -import javax.jdo.PersistenceManagerFactory; -import javax.jdo.Query; -import javax.jdo.Transaction; -import javax.jdo.datastore.DataStoreCache; -import javax.jdo.datastore.JDOConnection; -import javax.jdo.identity.IntIdentity; -import javax.sql.DataSource; - -import com.google.common.base.Strings; - -import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.ArrayUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.commons.lang.exception.ExceptionUtils; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.classification.InterfaceStability; -import org.apache.hadoop.conf.Configurable; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.common.*; -import org.apache.hadoop.hive.metastore.MetaStoreDirectSql.SqlFilterForPushdown; -import org.apache.hadoop.hive.metastore.api.*; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; -import org.apache.hadoop.hive.metastore.datasource.DataSourceProvider; -import org.apache.hadoop.hive.metastore.datasource.DataSourceProviderFactory; -import org.apache.hadoop.hive.metastore.metrics.Metrics; -import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; -import org.apache.hadoop.hive.metastore.model.*; -import org.apache.hadoop.hive.metastore.model.MWMMapping.EntityType; -import org.apache.hadoop.hive.metastore.model.MWMResourcePlan.Status; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder; -import org.apache.hadoop.hive.metastore.partition.spec.PartitionSpecProxy; -import org.apache.hadoop.hive.metastore.tools.SQLGenerator; -import org.apache.hadoop.hive.metastore.txn.TxnUtils; -import
[2/3] hive git commit: HIVE-20306 : Addendum to remove .orig files
http://git-wip-us.apache.org/repos/asf/hive/blob/64bef36a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java.orig -- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java.orig b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java.orig deleted file mode 100644 index 0a25b77..000 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java.orig +++ /dev/null @@ -1,2845 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import static org.apache.commons.lang.StringUtils.join; -import static org.apache.commons.lang.StringUtils.normalizeSpace; -import static org.apache.commons.lang.StringUtils.repeat; -import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; - -import java.sql.Blob; -import java.sql.Clob; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import java.util.stream.Collectors; - -import javax.jdo.PersistenceManager; -import javax.jdo.Query; -import javax.jdo.Transaction; -import javax.jdo.datastore.JDOConnection; - -import org.apache.commons.lang.BooleanUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.AggregateStatsCache.AggrColStats; -import org.apache.hadoop.hive.metastore.api.AggrStats; -import org.apache.hadoop.hive.metastore.api.ColumnStatistics; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Order; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.SQLCheckConstraint; -import org.apache.hadoop.hive.metastore.api.SQLDefaultConstraint; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.SkewedInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; -import org.apache.hadoop.hive.metastore.model.MConstraint; -import org.apache.hadoop.hive.metastore.model.MCreationMetadata; -import org.apache.hadoop.hive.metastore.model.MDatabase; -import org.apache.hadoop.hive.metastore.model.MNotificationLog; -import org.apache.hadoop.hive.metastore.model.MNotificationNextId; -import org.apache.hadoop.hive.metastore.model.MPartitionColumnPrivilege; -import org.apache.hadoop.hive.metastore.model.MPartitionColumnStatistics; -import org.apache.hadoop.hive.metastore.model.MPartitionPrivilege; -import org.apache.hadoop.hive.metastore.model.MTableColumnStatistics; -import org.apache.hadoop.hive.metastore.model.MWMResourcePlan; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.FilterBuilder; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator;
hive git commit: HIVE-20710: Constant folding may not create null constants without types (Zoltan Haindrich reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 2ff9c5229 -> 90e12280c
HIVE-20710: Constant folding may not create null constants without types
(Zoltan Haindrich reviewed by Ashutosh Chauhan)
Signed-off-by: Zoltan Haindrich
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/90e12280
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/90e12280
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/90e12280
Branch: refs/heads/master
Commit: 90e12280ca06b3df0099a858f58a542f190cf9b3
Parents: 2ff9c52
Author: Zoltan Haindrich
Authored: Wed Oct 10 10:38:00 2018 +0200
Committer: Zoltan Haindrich
Committed: Wed Oct 10 10:38:00 2018 +0200
--
.../optimizer/calcite/HiveRexExecutorImpl.java | 3 +-
.../stats/FilterSelectivityEstimator.java | 33 +--
.../calcite/translator/RexNodeConverter.java| 13 +-
.../test/queries/clientpositive/fold_to_null.q | 14 ++
.../results/clientpositive/fold_to_null.q.out | 209 +++
.../clientpositive/literal_decimal.q.out| 6 +-
6 files changed, 253 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hive/blob/90e12280/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
index b4bd142..1dede0f 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRexExecutorImpl.java
@@ -28,7 +28,6 @@ import org.apache.calcite.rex.RexNode;
import org.apache.hadoop.hive.ql.optimizer.ConstantPropagateProcFactory;
import
org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.RexNodeConverter;
-import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.slf4j.Logger;
@@ -63,7 +62,7 @@ public class HiveRexExecutorImpl extends RexExecutorImpl {
if (constant != null) {
try {
// convert constant back to RexNode
-reducedValues.add(rexNodeConverter.convert((ExprNodeConstantDesc)
constant));
+reducedValues.add(rexNodeConverter.convert(constant));
} catch (Exception e) {
LOG.warn(e.getMessage());
reducedValues.add(rexNode);
http://git-wip-us.apache.org/repos/asf/hive/blob/90e12280/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
--
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
index 43f8508..d362e9b 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/stats/FilterSelectivityEstimator.java
@@ -31,6 +31,7 @@ import org.apache.calcite.rex.RexCall;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexUtil;
import org.apache.calcite.rex.RexVisitorImpl;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperator;
@@ -57,6 +58,7 @@ public class FilterSelectivityEstimator extends
RexVisitorImpl {
return predicate.accept(this);
}
+ @Override
public Double visitCall(RexCall call) {
if (!deep) {
return 1.0;
@@ -138,17 +140,18 @@ public class FilterSelectivityEstimator extends
RexVisitorImpl {
* NDV of "f1(x, y, z) != f2(p, q, r)" ->
* "(maxNDV(x,y,z,p,q,r) - 1)/maxNDV(x,y,z,p,q,r)".
*
- *
+ *
* @param call
* @return
*/
private Double computeNotEqualitySelectivity(RexCall call) {
double tmpNDV = getMaxNDV(call);
-if (tmpNDV > 1)
- return (tmpNDV - (double) 1) / tmpNDV;
-else
+if (tmpNDV > 1) {
+ return (tmpNDV - 1) / tmpNDV;
+} else {
return 1.0;
+}
}
/**
@@ -156,7 +159,7 @@ public class FilterSelectivityEstimator extends
RexVisitorImpl {
*
* Note that >, >=, <, <=, = ... are considered generic functions and uses
* this method to find their selectivity.
- *
+ *
* @param call
* @return
*/
@@ -171,7 +174,7 @@ public class FilterSelectivityEstimator extends
RexVisitorImpl {
*
* Note we compute m1.
