This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new c4791205d84 HIVE-28400: Refactor QueryProperties feature flags to use
QueryFeature enum (#6560)
c4791205d84 is described below
commit c4791205d84201b351a8353e9eacadbc3c972c3e
Author: MANYA MEHROTRA <[email protected]>
AuthorDate: Wed Jul 15 15:08:57 2026 +0530
HIVE-28400: Refactor QueryProperties feature flags to use QueryFeature enum
(#6560)
---
.../hive/ql/hooks/CheckQueryPropertiesHook.java | 18 +-
.../hadoop/hive/ql/HiveQueryLifeTimeHook.java | 3 +-
.../org/apache/hadoop/hive/ql/QueryProperties.java | 326 +++------------------
.../hadoop/hive/ql/optimizer/GenMRTableScan1.java | 6 +-
.../hadoop/hive/ql/optimizer/GroupByOptimizer.java | 5 +-
.../hive/ql/optimizer/SimpleFetchAggregation.java | 6 +-
.../hive/ql/optimizer/SimpleFetchOptimizer.java | 8 +-
.../ql/optimizer/SortedDynPartitionOptimizer.java | 3 +-
...SortedDynPartitionTimeGranularityOptimizer.java | 3 +-
.../hadoop/hive/ql/optimizer/StatsOptimizer.java | 6 +-
.../hive/ql/optimizer/lineage/Generator.java | 10 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 17 +-
.../hadoop/hive/ql/parse/ProcessAnalyzeTable.java | 8 +-
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 87 +++---
.../apache/hadoop/hive/ql/parse/TaskCompiler.java | 30 +-
.../plugin/DisallowTransformHook.java | 3 +-
.../hive/ql/optimizer/lineage/TestGenerator.java | 7 +-
.../hadoop/hive/ql/parse/TestQueryProperties.java | 11 +-
18 files changed, 180 insertions(+), 377 deletions(-)
diff --git
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java
index 916a8daa1d5..b3d083468e6 100644
---
a/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java
+++
b/itests/util/src/main/java/org/apache/hadoop/hive/ql/hooks/CheckQueryPropertiesHook.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.hooks;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.hive.ql.session.SessionState.LogHelper;
@@ -40,14 +41,15 @@ public void run(HookContext hookContext) {
QueryProperties queryProps =
hookContext.getQueryPlan().getQueryProperties();
if (queryProps != null) {
- console.printError("Has Join: " + queryProps.hasJoin());
- console.printError("Has Group By: " + queryProps.hasGroupBy());
- console.printError("Has Sort By: " + queryProps.hasSortBy());
- console.printError("Has Order By: " + queryProps.hasOrderBy());
- console.printError("Has Group By After Join: " +
queryProps.hasJoinFollowedByGroupBy());
- console.printError("Uses Script: " + queryProps.usesScript());
- console.printError("Has Distribute By: " + queryProps.hasDistributeBy());
- console.printError("Has Cluster By: " + queryProps.hasClusterBy());
+ console.printError("Has Join: " +
queryProps.hasFeature(QueryFeature.JOIN));
+ console.printError("Has Group By: " +
queryProps.hasFeature(QueryFeature.GROUP_BY));
+ console.printError("Has Sort By: " +
queryProps.hasFeature(QueryFeature.SORT_BY));
+ console.printError("Has Order By: " +
queryProps.hasFeature(QueryFeature.ORDER_BY));
+ console.printError("Has Group By After Join: "
+ + queryProps.hasFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY));
+ console.printError("Uses Script: " +
queryProps.hasFeature(QueryFeature.USES_SCRIPT));
+ console.printError("Has Distribute By: " +
queryProps.hasFeature(QueryFeature.DISTRIBUTE_BY));
+ console.printError("Has Cluster By: " +
queryProps.hasFeature(QueryFeature.CLUSTER_BY));
}
}
}
\ No newline at end of file
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/HiveQueryLifeTimeHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/HiveQueryLifeTimeHook.java
index 82a3f190000..406ea05ee00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/HiveQueryLifeTimeHook.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/HiveQueryLifeTimeHook.java
@@ -25,6 +25,7 @@
import org.apache.hadoop.hive.metastore.api.CompactionRequest;
import org.apache.hadoop.hive.metastore.api.CompactionType;
import org.apache.hadoop.hive.metastore.txn.TxnUtils;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.hooks.PrivateHookContext;
import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHook;
import org.apache.hadoop.hive.ql.hooks.QueryLifeTimeHookContext;
@@ -70,7 +71,7 @@ private void checkAndRollbackCTAS(QueryLifeTimeHookContext
ctx) {
HiveConf conf = ctx.getHiveConf();
QueryPlan queryPlan = ctx.getHookContext().getQueryPlan();
boolean isCTAS = Optional.ofNullable(queryPlan.getQueryProperties())
- .map(queryProps -> queryProps.isCTAS()).orElse(false);
+ .map(queryProps ->
queryProps.hasFeature(QueryFeature.CTAS)).orElse(false);
PrivateHookContext pCtx = (PrivateHookContext) ctx.getHookContext();
Path tblPath = pCtx.getContext().getLocation();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
index 8bbc38661b6..b95d2791c25 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/QueryProperties.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql;
+import java.util.EnumSet;
import java.util.HashSet;
import java.util.Set;
@@ -56,103 +57,49 @@ public String getName() {
}
}
- boolean query;
- boolean analyzeCommand;
- boolean noScanAnalyzeCommand;
- boolean analyzeRewrite;
- boolean ctas;
- int outerQueryLimit;
-
- boolean hasJoin = false;
- boolean hasGroupBy = false;
- boolean hasOrderBy = false;
- boolean hasOuterOrderBy = false;
- boolean hasSortBy = false;
- boolean hasLimit = false;
- boolean hasJoinFollowedByGroupBy = false;
- boolean hasPTF = false;
- boolean hasWindowing = false;
- boolean hasQualify = false;
- boolean hasExcept = false;
- boolean hasIntersect = false;
+ public enum QueryFeature {
+ QUERY,
+ JOIN,
+ GROUP_BY,
+ ORDER_BY,
+ OUTER_ORDER_BY,
+ SORT_BY,
+ LIMIT,
+ JOIN_FOLLOWED_BY_GROUP_BY,
+ PTF,
+ WINDOWING,
+ QUALIFY,
+ EXCEPT,
+ INTERSECT,
+ USES_SCRIPT,
+ DISTRIBUTE_BY,
+ CLUSTER_BY,
+ MAP_GROUP_BY,
+ LATERAL_VIEW,
+ MULTI_DEST_QUERY,
+ FILTER_WITH_SUBQUERY,
+ MATERIALIZED_VIEW,
+ VIEW,
+ CTAS,
+ ANALYZE,
+ NO_SCAN,
+ REWRITE
+ }
- // does the query have a using clause
- boolean usesScript = false;
+ int outerQueryLimit;
- boolean hasDistributeBy = false;
- boolean hasClusterBy = false;
- boolean mapJoinRemoved = false;
- boolean hasMapGroupBy = false;
+ private final EnumSet<QueryFeature> features =
EnumSet.noneOf(QueryFeature.class);
- private boolean hasLateralViews = false;
private boolean cboSupportedLateralViews = true;
private int noOfJoins = 0;
private int noOfOuterJoins = 0;
- private boolean multiDestQuery;
- private boolean filterWithSubQuery;
-
- // True if this statement creates or replaces a materialized view
- private boolean isMaterializedView;
- private boolean isView;
-
private QueryType queryType = null;
// set of used tables, aliases are resolved to real table names
private Set<String> usedTables = new HashSet<>();
- public boolean isQuery() {
- return query;
- }
-
- public void setQuery(boolean query) {
- this.query = query;
- }
-
- /**
- * The return value of either isAnalyzeCommand() or isAnalyzeRewrite() is
always true for analyze commands:
- * isAnalyzeCommand=true for "compute statistics",
- * isAnalyzeRewrite=true for "compute statistics for columns".
- *
- * @return whether the query is an ANALYZE TABLE query
- */
- public boolean isAnalyze() {
- return isAnalyzeCommand() || isAnalyzeRewrite();
- }
-
- public boolean isAnalyzeCommand() {
- return analyzeCommand;
- }
-
- public void setAnalyzeCommand(boolean analyzeCommand) {
- this.analyzeCommand = analyzeCommand;
- }
-
- public boolean isNoScanAnalyzeCommand() {
- return noScanAnalyzeCommand;
- }
-
- public void setNoScanAnalyzeCommand(boolean noScanAnalyzeCommand) {
- this.noScanAnalyzeCommand = noScanAnalyzeCommand;
- }
-
- public boolean isAnalyzeRewrite() {
- return analyzeRewrite;
- }
-
- public void setAnalyzeRewrite(boolean analyzeRewrite) {
- this.analyzeRewrite = analyzeRewrite;
- }
-
- public boolean isCTAS() {
- return ctas;
- }
-
- public void setCTAS(boolean ctas) {
- this.ctas = ctas;
- }
-
public int getOuterQueryLimit() {
return outerQueryLimit;
}
@@ -161,11 +108,8 @@ public void setOuterQueryLimit(int outerQueryLimit) {
this.outerQueryLimit = outerQueryLimit;
}
- public boolean hasJoin() {
- return (noOfJoins > 0);
- }
-
public void incrementJoinCount(boolean outerJoin) {
+ addFeature(QueryFeature.JOIN);
noOfJoins++;
if (outerJoin) {
noOfOuterJoins++;
@@ -180,184 +124,20 @@ public int getOuterJoinCount() {
return noOfOuterJoins;
}
- public void setHasLateralViews(boolean hasLateralViews) {
- this.hasLateralViews = hasLateralViews;
- }
-
- public boolean hasLateralViews() {
- return hasLateralViews;
- }
-
- public void setCBOSupportedLateralViews(boolean cboSupportedLateralViews) {
- this.cboSupportedLateralViews = cboSupportedLateralViews;
+ public void markUnsupportedLateralViewsForCBO() {
+ this.cboSupportedLateralViews = false;
}
public boolean isCBOSupportedLateralViews() {
return cboSupportedLateralViews;
}
- public boolean hasGroupBy() {
- return hasGroupBy;
- }
-
- public void setHasGroupBy(boolean hasGroupBy) {
- this.hasGroupBy = hasGroupBy;
- }
-
- public boolean hasOrderBy() {
- return hasOrderBy;
- }
-
- public void setHasOrderBy(boolean hasOrderBy) {
- this.hasOrderBy = hasOrderBy;
- }
-
- public boolean hasOuterOrderBy() {
- return hasOuterOrderBy;
- }
-
- public void setHasOuterOrderBy(boolean hasOuterOrderBy) {
- this.hasOuterOrderBy = hasOuterOrderBy;
- }
-
- public boolean hasSortBy() {
- return hasSortBy;
- }
-
- public void setHasSortBy(boolean hasSortBy) {
- this.hasSortBy = hasSortBy;
- }
-
- public void setHasLimit(boolean hasLimit) {
- this.hasLimit = hasLimit;
+ public void addFeature(QueryFeature feature) {
+ features.add(feature);
}
- public boolean hasLimit() {
- return hasLimit;
- }
-
- public boolean hasJoinFollowedByGroupBy() {
- return hasJoinFollowedByGroupBy;
- }
-
- public void setHasJoinFollowedByGroupBy(boolean hasJoinFollowedByGroupBy) {
- this.hasJoinFollowedByGroupBy = hasJoinFollowedByGroupBy;
- }
-
- public boolean usesScript() {
- return usesScript;
- }
-
- public void setUsesScript(boolean usesScript) {
- this.usesScript = usesScript;
- }
-
- public boolean hasDistributeBy() {
- return hasDistributeBy;
- }
-
- public void setHasDistributeBy(boolean hasDistributeBy) {
- this.hasDistributeBy = hasDistributeBy;
- }
-
- public boolean hasClusterBy() {
- return hasClusterBy;
- }
-
- public void setHasClusterBy(boolean hasClusterBy) {
- this.hasClusterBy = hasClusterBy;
- }
-
- public boolean hasPTF() {
- return hasPTF;
- }
-
- public void setHasPTF(boolean hasPTF) {
- this.hasPTF = hasPTF;
- }
-
- public boolean hasWindowing() {
- return hasWindowing;
- }
-
- public void setHasWindowing(boolean hasWindowing) {
- this.hasWindowing = hasWindowing;
- }
-
- public boolean hasQualify() {
- return hasQualify;
- }
-
- public void setHasQualify(boolean hasQualify) {
- this.hasQualify = hasQualify;
- }
-
- public boolean hasExcept() {
- return hasExcept;
- }
-
- public void setHasExcept(boolean hasExcept) {
- this.hasExcept = hasExcept;
- }
-
- public boolean hasIntersect() {
- return hasIntersect;
- }
-
- public void setHasIntersect(boolean hasIntersect) {
- this.hasIntersect = hasIntersect;
- }
-
- public boolean isMapJoinRemoved() {
- return mapJoinRemoved;
- }
-
- public void setMapJoinRemoved(boolean mapJoinRemoved) {
- this.mapJoinRemoved = mapJoinRemoved;
- }
-
- public boolean isHasMapGroupBy() {
- return hasMapGroupBy;
- }
-
- public void setHasMapGroupBy(boolean hasMapGroupBy) {
- this.hasMapGroupBy = hasMapGroupBy;
- }
-
- public boolean hasMultiDestQuery() {
- return this.multiDestQuery;
- }
-
- public void setMultiDestQuery(boolean multiDestQuery) {
- this.multiDestQuery = multiDestQuery;
- }
-
- public void setFilterWithSubQuery(boolean filterWithSubQuery) {
- this.filterWithSubQuery = filterWithSubQuery;
- }
-
- public boolean hasFilterWithSubQuery() {
- return this.filterWithSubQuery;
- }
-
- /**
- * True indicates this statement create or replaces a materialized view, not
that it is a query
- * against a materialized view.
- */
- public boolean isMaterializedView() {
- return isMaterializedView;
- }
-
- public void setMaterializedView(boolean isMaterializedView) {
- this.isMaterializedView = isMaterializedView;
- }
-
- public boolean isView() {
- return isView;
- }
-
- public void setView(boolean view) {
- isView = view;
+ public boolean hasFeature(QueryFeature feature) {
+ return features.contains(feature);
}
public QueryType getQueryType() {
@@ -377,40 +157,14 @@ public void setUsedTables(Set<String> usedTables) {
}
public void clear() {
- query = false;
- analyzeCommand = false;
- noScanAnalyzeCommand = false;
- analyzeRewrite = false;
- ctas = false;
outerQueryLimit = -1;
- isMaterializedView = false;
- hasJoin = false;
- hasGroupBy = false;
- hasOrderBy = false;
- hasOuterOrderBy = false;
- hasSortBy = false;
- hasJoinFollowedByGroupBy = false;
- hasPTF = false;
- hasWindowing = false;
- hasQualify = false;
- hasExcept = false;
- hasIntersect = false;
-
- // does the query have a using clause
- usesScript = false;
-
- hasDistributeBy = false;
- hasClusterBy = false;
- mapJoinRemoved = false;
- hasMapGroupBy = false;
+ features.clear();
+ cboSupportedLateralViews = true;
noOfJoins = 0;
noOfOuterJoins = 0;
- multiDestQuery = false;
- filterWithSubQuery = false;
-
usedTables.clear();
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
index 6c790071752..6058807fb19 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
@@ -23,6 +23,7 @@
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -83,8 +84,9 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx opProcCtx,
ctx.setCurrAliasId(currAliasId);
mapCurrCtx.put(op, new GenMapRedCtx(currTask, currAliasId));
- if (parseCtx.getQueryProperties().isAnalyzeCommand()) {
- boolean noScan =
parseCtx.getQueryProperties().isNoScanAnalyzeCommand();
+ if (parseCtx.getQueryProperties().hasFeature(QueryFeature.ANALYZE)
+ &&
!parseCtx.getQueryProperties().hasFeature(QueryFeature.REWRITE)) {
+ boolean noScan =
parseCtx.getQueryProperties().hasFeature(QueryFeature.NO_SCAN);
if (BasicStatsNoJobTask.canUseBasicStats(table, inputFormat)) {
// For ORC and Parquet, all the following statements are the same
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
index ec2a6ccb818..7ab5d7e89c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
@@ -32,6 +32,7 @@
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -212,7 +213,7 @@ else if (!HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_GROUPBY_SKEW)) {
convertGroupByMapSideSortedGroupBy(hiveConf, groupByOp, depth);
}
else if (optimizeDistincts && !HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
- pGraphContext.getQueryProperties().setHasMapGroupBy(true);
+
pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY);
ReduceSinkOperator reduceSinkOp =
(ReduceSinkOperator)groupByOp.getChildOperators().get(0);
GroupByDesc childGroupByDesc =
@@ -514,7 +515,7 @@ private GroupByOptimizerSortMatch matchBucketSortCols(
// The operators specified by depth and removed from the tree.
protected void convertGroupByMapSideSortedGroupBy(
HiveConf conf, GroupByOperator groupByOp, int depth) {
- pGraphContext.getQueryProperties().setHasMapGroupBy(true);
+ pGraphContext.getQueryProperties().addFeature(QueryFeature.MAP_GROUP_BY);
if (removeChildren(groupByOp, depth)) {
// Use bucketized hive input format - that makes sure that one mapper
reads the entire file
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java
index d14bafbcfa6..f3640e93c09 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchAggregation.java
@@ -25,6 +25,7 @@
import java.util.Stack;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -55,8 +56,9 @@ public class SimpleFetchAggregation extends Transform {
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
- if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery() ||
- pctx.getQueryProperties().isAnalyzeRewrite() ||
pctx.getQueryProperties().isCTAS() ||
+ if (pctx.getFetchTask() != null ||
!pctx.getQueryProperties().hasFeature(QueryFeature.QUERY) ||
+ pctx.getQueryProperties().hasFeature(QueryFeature.REWRITE) ||
+ pctx.getQueryProperties().hasFeature(QueryFeature.CTAS) ||
pctx.getLoadFileWork().size() > 1 ||
!pctx.getLoadTableWork().isEmpty()) {
return pctx;
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
index e469f679faa..dae7fa6eefa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
@@ -45,6 +45,7 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
@@ -105,8 +106,11 @@ public class SimpleFetchOptimizer extends Transform {
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
Map<String, TableScanOperator> topOps = pctx.getTopOps();
- if ((pctx.getQueryProperties().isQuery() ||
pctx.getQueryProperties().isView())
- && !pctx.getQueryProperties().isAnalyzeCommand()
+ boolean isAnalyzeCommand =
pctx.getQueryProperties().hasFeature(QueryFeature.ANALYZE)
+ && !pctx.getQueryProperties().hasFeature(QueryFeature.REWRITE);
+ if ((pctx.getQueryProperties().hasFeature(QueryFeature.QUERY)
+ || pctx.getQueryProperties().hasFeature(QueryFeature.VIEW))
+ && !isAnalyzeCommand
&& topOps.size() == 1) {
// no join, no groupby, no distinct, no lateral view, no subq,
// no CTAS or insert, not analyze command, and single sourced.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
index f5431fa3493..ec3917e56d8 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionOptimizer.java
@@ -38,6 +38,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
@@ -732,7 +733,7 @@ public ReduceSinkOperator getReduceSinkOp(List<Integer>
partitionPositions, List
// should honor the ordering of records provided by ORDER BY in SELECT
statement
ReduceSinkOperator parentRSOp =
OperatorUtils.findSingleOperatorUpstream(parent,
ReduceSinkOperator.class);
- if (parentRSOp != null &&
parseCtx.getQueryProperties().hasOuterOrderBy()) {
+ if (parentRSOp != null &&
parseCtx.getQueryProperties().hasFeature(QueryFeature.OUTER_ORDER_BY)) {
String parentRSOpOrder = parentRSOp.getConf().getOrder();
String parentRSOpNullOrder = parentRSOp.getConf().getNullOrder();
if (parentRSOpOrder != null && !parentRSOpOrder.isEmpty() &&
sortPositions.isEmpty()) {
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
index 0340a638648..39c8b0a49d5 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SortedDynPartitionTimeGranularityOptimizer.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.hive.conf.Constants;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -137,7 +138,7 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx procCtx,
// introduce RS and EX before FS
FileSinkOperator fsOp = (FileSinkOperator) nd;
final String sh =
fsOp.getConf().getTableInfo().getOutputFileFormatClassName();
- if (parseCtx.getQueryProperties().isQuery() || sh == null || !sh
+ if (parseCtx.getQueryProperties().hasFeature(QueryFeature.QUERY) || sh
== null || !sh
.equals(Constants.DRUID_HIVE_OUTPUT_FORMAT)) {
// Bail out, nothing to do
return null;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
index 444f56724cb..c3eacc59bd2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
@@ -27,6 +27,7 @@
import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData;
import org.apache.hadoop.hive.metastore.api.LongColumnStatsData;
import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
@@ -108,8 +109,9 @@ public class StatsOptimizer extends Transform {
@Override
public ParseContext transform(ParseContext pctx) throws SemanticException {
- if (pctx.getFetchTask() != null || !pctx.getQueryProperties().isQuery()
- || pctx.getQueryProperties().isAnalyzeRewrite() ||
pctx.getQueryProperties().isCTAS()
+ if (pctx.getFetchTask() != null ||
!pctx.getQueryProperties().hasFeature(QueryFeature.QUERY)
+ || pctx.getQueryProperties().hasFeature(QueryFeature.REWRITE)
+ || pctx.getQueryProperties().hasFeature(QueryFeature.CTAS)
|| pctx.getLoadFileWork().size() > 1 ||
!pctx.getLoadTableWork().isEmpty()
// If getNameToSplitSample is not empty, at least one of the source
// tables is being sampled and we can not optimize.
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
index b3d2793dfef..cefed587a03 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/Generator.java
@@ -28,6 +28,7 @@
import org.apache.commons.lang3.EnumUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.exec.CommonJoinOperator;
import org.apache.hadoop.hive.ql.exec.FilterOperator;
import org.apache.hadoop.hive.ql.exec.GroupByOperator;
@@ -68,11 +69,12 @@ public class Generator extends Transform {
enum LineageInfoFilter {
CREATE_TABLE(parseContext -> parseContext.getCreateTable() != null),
- CREATE_TABLE_AS_SELECT(parseContext ->
parseContext.getQueryProperties().isCTAS()),
- CREATE_VIEW(parseContext -> parseContext.getQueryProperties().isView()),
- CREATE_MATERIALIZED_VIEW(parseContext ->
parseContext.getQueryProperties().isMaterializedView()),
+ CREATE_TABLE_AS_SELECT(parseContext ->
parseContext.getQueryProperties().hasFeature(QueryFeature.CTAS)),
+ CREATE_VIEW(parseContext ->
parseContext.getQueryProperties().hasFeature(QueryFeature.VIEW)),
+ CREATE_MATERIALIZED_VIEW(parseContext ->
+
parseContext.getQueryProperties().hasFeature(QueryFeature.MATERIALIZED_VIEW)),
LOAD(parseContext -> !(parseContext.getLoadTableWork() == null ||
parseContext.getLoadTableWork().isEmpty())),
- QUERY(parseContext -> parseContext.getQueryProperties().isQuery()),
+ QUERY(parseContext ->
parseContext.getQueryProperties().hasFeature(QueryFeature.QUERY)),
ALL(parseContext -> true),
NONE(parseContext -> false);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 5e1640573db..a91a281eedf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -141,6 +141,7 @@
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
@@ -567,7 +568,7 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx)
throws SemanticExcept
}
Pair<Boolean, String> canCBOHandleReason = canCBOHandleAst(queryForCbo,
getQB(), cboCtx);
runCBO = canCBOHandleReason.left;
- if (queryProperties.hasMultiDestQuery()) {
+ if (queryProperties.hasFeature(QueryFeature.MULTI_DEST_QUERY)) {
handleMultiDestQuery(ast, cboCtx);
}
@@ -689,9 +690,9 @@ Operator genOPTree(ASTNode ast, PlannerContext plannerCtx)
throws SemanticExcept
this.ctx.setCboInfo(cboMsg);
// Determine if we should re-throw the exception OR if we try to
mark the query to retry as non-CBO.
- final boolean requiresCBO = queryProperties.hasQualify()
- || queryProperties.hasExcept()
- || queryProperties.hasIntersect();
+ final boolean requiresCBO =
queryProperties.hasFeature(QueryFeature.QUALIFY)
+ || queryProperties.hasFeature(QueryFeature.EXCEPT)
+ || queryProperties.hasFeature(QueryFeature.INTERSECT);
if (requiresCBO || fallbackStrategy.isFatal(e)) {
if (e instanceof RuntimeException || e instanceof
SemanticException) {
// These types of exceptions do not need wrapped
@@ -1000,13 +1001,13 @@ private static String canHandleQbForCbo(QueryProperties
queryProperties,
HiveConf conf, boolean topLevelQB) {
List<String> reasons = new ArrayList<>();
// Not ok to run CBO, build error message.
- if (queryProperties.hasSortBy() && queryProperties.hasLimit()) {
+ if (queryProperties.hasFeature(QueryFeature.SORT_BY) &&
queryProperties.hasFeature(QueryFeature.LIMIT)) {
reasons.add("has sort by with limit");
}
- if (queryProperties.hasPTF()) {
+ if (queryProperties.hasFeature(QueryFeature.PTF)) {
reasons.add("has PTF");
}
- if (queryProperties.usesScript()) {
+ if (queryProperties.hasFeature(QueryFeature.USES_SCRIPT)) {
reasons.add("uses scripts");
}
if (!queryProperties.isCBOSupportedLateralViews()) {
@@ -1024,7 +1025,7 @@ private static EnumSet<ExtendedCBOProfile>
obtainCBOProfiles(QueryProperties que
profilesCBO.add(ExtendedCBOProfile.JOIN_REORDERING);
}
// If the query contains windowing processing
- if (queryProperties.hasWindowing()) {
+ if (queryProperties.hasFeature(QueryFeature.WINDOWING)) {
profilesCBO.add(ExtendedCBOProfile.WINDOWING_POSTPROCESSING);
}
return profilesCBO;
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
index 5079fba22a0..53442fb4899 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ProcessAnalyzeTable.java
@@ -22,6 +22,7 @@
import java.util.Set;
import java.util.Stack;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.TableScanOperator;
@@ -72,7 +73,8 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx procContext,
Table table = tableScan.getConf().getTableMetadata();
Class<? extends InputFormat> inputFormat = table.getInputFormatClass();
- if (parseContext.getQueryProperties().isAnalyzeCommand()) {
+ if (parseContext.getQueryProperties().hasFeature(QueryFeature.ANALYZE)
+ &&
!parseContext.getQueryProperties().hasFeature(QueryFeature.REWRITE)) {
assert tableScan.getChildOperators() == null ||
tableScan.getChildOperators().size() == 0;
@@ -114,7 +116,7 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx procContext,
BasicStatsWork basicStatsWork = new
BasicStatsWork(table.getTableSpec());
basicStatsWork.setIsExplicitAnalyze(true);
-
basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().isNoScanAnalyzeCommand());
+
basicStatsWork.setNoScanAnalyzeCommand(parseContext.getQueryProperties().hasFeature(QueryFeature.NO_SCAN));
StatsWork columnStatsWork = new StatsWork(table, basicStatsWork,
parseContext.getConf());
columnStatsWork.collectStatsFromAggregator(tableScan.getConf());
@@ -124,7 +126,7 @@ public Object process(Node nd, Stack<Node> stack,
NodeProcessorCtx procContext,
// ANALYZE TABLE T [PARTITION (...)] COMPUTE STATISTICS noscan;
// The plan consists of a StatsTask only.
- if (parseContext.getQueryProperties().isNoScanAnalyzeCommand()) {
+ if
(parseContext.getQueryProperties().hasFeature(QueryFeature.NO_SCAN)) {
statsTask.setParentTasks(null);
context.rootTasks.remove(context.currentTask);
context.rootTasks.add(statsTask);
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index b754e677a47..b728bd85402 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -111,6 +111,7 @@
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.cache.results.CacheUsage;
import org.apache.hadoop.hive.ql.cache.results.QueryResultsCache;
@@ -669,19 +670,19 @@ void doPhase1QBExpr(ASTNode ast, QBExpr qbexpr, String
id, String alias, boolean
qbexpr.setOpcode(QBExpr.Opcode.UNION);
break;
case HiveParser.TOK_INTERSECTALL:
- queryProperties.setHasIntersect(true);
+ queryProperties.addFeature(QueryFeature.INTERSECT);
qbexpr.setOpcode(QBExpr.Opcode.INTERSECTALL);
break;
case HiveParser.TOK_INTERSECTDISTINCT:
- queryProperties.setHasIntersect(true);
+ queryProperties.addFeature(QueryFeature.INTERSECT);
qbexpr.setOpcode(QBExpr.Opcode.INTERSECT);
break;
case HiveParser.TOK_EXCEPTALL:
- queryProperties.setHasExcept(true);
+ queryProperties.addFeature(QueryFeature.EXCEPT);
qbexpr.setOpcode(QBExpr.Opcode.EXCEPTALL);
break;
case HiveParser.TOK_EXCEPTDISTINCT:
- queryProperties.setHasExcept(true);
+ queryProperties.addFeature(QueryFeature.EXCEPT);
qbexpr.setOpcode(QBExpr.Opcode.EXCEPT);
break;
default:
@@ -724,7 +725,7 @@ private Map<String, ASTNode>
doPhase1GetAggregationsFromSelect(
for (ASTNode wdwFn : wdwFns) {
WindowingSpec spec = qb.getWindowingSpec(dest);
if(spec == null) {
- queryProperties.setHasWindowing(true);
+ queryProperties.addFeature(QueryFeature.WINDOWING);
spec = new WindowingSpec();
qb.addDestToWindowingSpec(dest, spec);
}
@@ -1674,7 +1675,7 @@ private void processJoin(QB qb, ASTNode join) throws
SemanticException {
} else if (child.getToken().getType() == HiveParser.TOK_SUBQUERY) {
processSubQuery(qb, child);
} else if (child.getToken().getType() == HiveParser.TOK_PTBLFUNCTION) {
- queryProperties.setHasPTF(true);
+ queryProperties.addFeature(QueryFeature.PTF);
processPTF(qb, child);
PTFInvocationSpec ptfInvocationSpec = qb.getPTFInvocationSpec(child);
String inputAlias = ptfInvocationSpec == null ? null :
@@ -1713,7 +1714,9 @@ private String processLateralView(QB qb, ASTNode
lateralView)
int numChildren = lateralView.getChildCount();
assert (numChildren == 2);
- queryProperties.setCBOSupportedLateralViews(isCBOSupportedLateralView());
+ if (!isCBOSupportedLateralView()) {
+ queryProperties.markUnsupportedLateralViewsForCBO();
+ }
ASTNode next = (ASTNode) lateralView.getChild(1);
String alias = null;
@@ -1785,7 +1788,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
}
if ((ast.getChild(posn).getChild(0).getType() ==
HiveParser.TOK_TRANSFORM)) {
- queryProperties.setUsesScript(true);
+ queryProperties.addFeature(QueryFeature.USES_SCRIPT);
}
Map<String, ASTNode> aggregations =
doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest);
@@ -1798,7 +1801,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
case HiveParser.TOK_WHERE:
qbp.setWhrExprForClause(ctx_1.dest, ast);
if (!SubQueryUtils.findSubQueries((ASTNode)
ast.getChild(0)).isEmpty()) {
- queryProperties.setFilterWithSubQuery(true);
+ queryProperties.addFeature(QueryFeature.FILTER_WITH_SUBQUERY);
}
doPhase1WhereClause(ast, qb);
break;
@@ -1853,10 +1856,10 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
// Using qbp.getClauseNamesForDest().size() >= 2 would be
// equivalent, but we use == to avoid setting the property
// multiple times
- queryProperties.setMultiDestQuery(true);
+ queryProperties.addFeature(QueryFeature.MULTI_DEST_QUERY);
}
- if (plannerCtx != null && !queryProperties.hasMultiDestQuery()) {
+ if (plannerCtx != null &&
!queryProperties.hasFeature(QueryFeature.MULTI_DEST_QUERY)) {
plannerCtx.setInsertToken(ast, isTmpFileDest);
} else if (plannerCtx != null && qbp.getClauseNamesForDest().size() ==
2) {
// For multi-insert query, currently we only optimize the FROM
clause.
@@ -1888,13 +1891,13 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
} else if (frm.getToken().getType() == HiveParser.TOK_SUBQUERY) {
processSubQuery(qb, frm);
} else if (isASTNodeLateralView(frm)) {
- queryProperties.setHasLateralViews(true);
+ queryProperties.addFeature(QueryFeature.LATERAL_VIEW);
processLateralView(qb, frm);
} else if (isJoinToken(frm)) {
processJoin(qb, frm);
qbp.setJoinExpr(frm);
}else if(frm.getToken().getType() == HiveParser.TOK_PTBLFUNCTION){
- queryProperties.setHasPTF(true);
+ queryProperties.addFeature(QueryFeature.PTF);
processPTF(qb, frm);
}
break;
@@ -1902,14 +1905,14 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
case HiveParser.TOK_CLUSTERBY:
// Get the clusterby aliases - these are aliased to the entries in the
// select list
- queryProperties.setHasClusterBy(true);
+ queryProperties.addFeature(QueryFeature.CLUSTER_BY);
qbp.setClusterByExprForClause(ctx_1.dest, ast);
break;
case HiveParser.TOK_DISTRIBUTEBY:
// Get the distribute by aliases - these are aliased to the entries in
// the select list
- queryProperties.setHasDistributeBy(true);
+ queryProperties.addFeature(QueryFeature.DISTRIBUTE_BY);
qbp.setDistributeByExprForClause(ctx_1.dest, ast);
if (qbp.getClusterByForClause(ctx_1.dest) != null) {
throw new SemanticException(generateErrorMessage(ast,
@@ -1923,7 +1926,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
case HiveParser.TOK_SORTBY:
// Get the sort by aliases - these are aliased to the entries in the
// select list
- queryProperties.setHasSortBy(true);
+ queryProperties.addFeature(QueryFeature.SORT_BY);
qbp.setSortByExprForClause(ctx_1.dest, ast);
if (qbp.getClusterByForClause(ctx_1.dest) != null) {
throw new SemanticException(generateErrorMessage(ast,
@@ -1938,7 +1941,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
case HiveParser.TOK_ORDERBY:
// Get the order by aliases - these are aliased to the entries in the
// select list
- queryProperties.setHasOrderBy(true);
+ queryProperties.addFeature(QueryFeature.ORDER_BY);
qbp.setOrderByExprForClause(ctx_1.dest, ast);
if (qbp.getClusterByForClause(ctx_1.dest) != null) {
throw new SemanticException(generateErrorMessage(ast,
@@ -1955,9 +1958,9 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
case HiveParser.TOK_GROUPING_SETS:
// Get the groupby aliases - these are aliased to the entries in the
// select list
- queryProperties.setHasGroupBy(true);
+ queryProperties.addFeature(QueryFeature.GROUP_BY);
if (qbp.getJoinExpr() != null) {
- queryProperties.setHasJoinFollowedByGroupBy(true);
+ queryProperties.addFeature(QueryFeature.JOIN_FOLLOWED_BY_GROUP_BY);
}
qbp.setGroupByExprForClause(ctx_1.dest, ast);
skipRecursion = true;
@@ -1982,7 +1985,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
break;
case HiveParser.TOK_QUALIFY:
- queryProperties.setHasQualify(true);
+ queryProperties.addFeature(QueryFeature.QUALIFY);
qbp.setQualifyExprForClause(ctx_1.dest, ast);
qbp.addAggregationExprsForClause(ctx_1.dest,
doPhase1GetAggregationsFromSelect(ast, qb, ctx_1.dest));
@@ -1997,7 +2000,7 @@ boolean doPhase1(ASTNode ast, QB qb, Phase1Ctx ctx_1,
PlannerContext plannerCtx,
break;
case HiveParser.TOK_LIMIT:
- queryProperties.setHasLimit(true);
+ queryProperties.addFeature(QueryFeature.LIMIT);
if (ast.getChildCount() == 2) {
qbp.setDestLimit(ctx_1.dest,
Integer.valueOf(ast.getChild(0).getText()),
Integer.valueOf(ast.getChild(1).getText()));
@@ -4767,7 +4770,7 @@ private Operator<?> genSelectPlan(String dest, ASTNode
selExprList, QB qb, Opera
boolean isInTransform = (selExprList.getChild(posn).getChild(0).getType()
==
HiveParser.TOK_TRANSFORM);
if (isInTransform) {
- queryProperties.setUsesScript(true);
+ queryProperties.addFeature(QueryFeature.USES_SCRIPT);
globalLimitCtx.setHasTransformOrUDTF(true);
trfm = (ASTNode) selExprList.getChild(posn).getChild(0);
}
@@ -10333,7 +10336,6 @@ private List<String> getMapSideJoinTables(QB qb) {
}
}
else {
- queryProperties.setMapJoinRemoved(true);
}
}
}
@@ -11602,7 +11604,7 @@ private Operator genPostGroupByBodyPlan(Operator curr,
String dest, QB qb,
curr = genHavingPlan(dest, qb, curr, aliasToOpInfo);
}
- if(queryProperties.hasWindowing() && qb.getWindowingSpec(dest) != null) {
+ if(queryProperties.hasFeature(QueryFeature.WINDOWING) &&
qb.getWindowingSpec(dest) != null) {
curr = genWindowingPlan(qb, qb.getWindowingSpec(dest), curr);
// GBy for DISTINCT after windowing
if ((qbp.getAggregationExprsForClause(dest).size() != 0
@@ -12424,7 +12426,7 @@ private Operator genPlan(QB qb, boolean
skipAmbiguityCheck)
Operator srcOpInfo = null;
Operator lastPTFOp = null;
- if(queryProperties.hasPTF()){
+ if(queryProperties.hasFeature(QueryFeature.PTF)){
//After processing subqueries and source tables, process
// partitioned table functions
@@ -15115,19 +15117,36 @@ private ASTNode buildSelExprSubTree(String
tableAlias, String col) {
private void copyInfoToQueryProperties(QueryProperties queryProperties) {
if (qb != null) {
- queryProperties.setQuery(qb.getIsQuery() && !forViewCreation);
- queryProperties.setAnalyzeCommand(qb.getParseInfo().isAnalyzeCommand());
-
queryProperties.setNoScanAnalyzeCommand(qb.getParseInfo().isNoScanAnalyzeCommand());
- queryProperties.setAnalyzeRewrite(qb.isAnalyzeRewrite());
- queryProperties.setCTAS(qb.getTableDesc() != null);
+ if (qb.getIsQuery() && !forViewCreation) {
+ queryProperties.addFeature(QueryFeature.QUERY);
+ }
+ if (qb.getParseInfo().isAnalyzeCommand()) {
+ queryProperties.addFeature(QueryFeature.ANALYZE);
+ }
+ if (qb.getParseInfo().isNoScanAnalyzeCommand()) {
+ queryProperties.addFeature(QueryFeature.ANALYZE);
+ queryProperties.addFeature(QueryFeature.NO_SCAN);
+ }
+ if (qb.isAnalyzeRewrite()) {
+ queryProperties.addFeature(QueryFeature.ANALYZE);
+ queryProperties.addFeature(QueryFeature.REWRITE);
+ }
+ if (qb.getTableDesc() != null) {
+ queryProperties.addFeature(QueryFeature.CTAS);
+ }
if (qb.getParseInfo().hasInsertTables()) {
queryProperties.setQueryType(QueryProperties.QueryType.DML);
}
- queryProperties.setHasOuterOrderBy(!qb.getParseInfo().getIsSubQ() &&
- !qb.getParseInfo().getDestToOrderBy().isEmpty());
+ if (!qb.getParseInfo().getIsSubQ() &&
!qb.getParseInfo().getDestToOrderBy().isEmpty()) {
+ queryProperties.addFeature(QueryFeature.OUTER_ORDER_BY);
+ }
queryProperties.setOuterQueryLimit(qb.getParseInfo().getOuterQueryLimit());
- queryProperties.setView(forViewCreation);
- queryProperties.setMaterializedView(qb.isMaterializedView());
+ if (forViewCreation) {
+ queryProperties.addFeature(QueryFeature.VIEW);
+ }
+ if (qb.isMaterializedView()) {
+ queryProperties.addFeature(QueryFeature.MATERIALIZED_VIEW);
+ }
}
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
index d661102a104..aed8ebdebb9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java
@@ -33,6 +33,7 @@
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.ddl.DDLDesc;
import org.apache.hadoop.hive.ql.ddl.DDLDescWithTableProperties;
@@ -135,7 +136,8 @@ public void compile(final ParseContext pCtx,
List<LoadTableDesc> loadTableWork = pCtx.getLoadTableWork();
List<LoadFileDesc> loadFileWork = pCtx.getLoadFileWork();
- boolean isCStats = pCtx.getQueryProperties().isAnalyzeRewrite();
+ boolean isCStats =
pCtx.getQueryProperties().hasFeature(QueryFeature.REWRITE);
+ boolean isAnalyzeCommand =
pCtx.getQueryProperties().hasFeature(QueryFeature.ANALYZE) && !isCStats;
int outerQueryLimit = pCtx.getQueryProperties().getOuterQueryLimit();
boolean directInsert = false;
@@ -178,7 +180,7 @@ public void compile(final ParseContext pCtx,
return;
}
- if (!pCtx.getQueryProperties().isAnalyzeCommand()) {
+ if (!isAnalyzeCommand) {
LOG.debug("Skipping optimize operator plan for analyze command.");
optimizeOperatorPlan(pCtx);
}
@@ -188,7 +190,7 @@ public void compile(final ParseContext pCtx,
* If the select is from analyze table column rewrite, don't create a
fetch task. Instead create
* a column stats task later.
*/
- if (pCtx.getQueryProperties().isQuery() && !isCStats) {
+ if (pCtx.getQueryProperties().hasFeature(QueryFeature.QUERY) && !isCStats)
{
if ((!loadTableWork.isEmpty()) || (loadFileWork.size() != 1)) {
throw new
SemanticException(ErrorMsg.INVALID_LOAD_TABLE_FILE_WORK.getMsg());
}
@@ -279,14 +281,16 @@ public void compile(final ParseContext pCtx,
} else if (!isCStats) {
for (LoadTableDesc ltd : loadTableWork) {
Task<MoveWork> tsk = TaskFactory
- .get(new MoveWork(pCtx.getQueryProperties().isCTAS() &&
pCtx.getCreateTable().isExternal(),
+ .get(new
MoveWork(pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS)
+ && pCtx.getCreateTable().isExternal(),
null, null, ltd, null, false));
mvTask.add(tsk);
}
boolean oneLoadFileForCtas = true;
for (LoadFileDesc lfd : loadFileWork) {
- if (pCtx.getQueryProperties().isCTAS() ||
pCtx.getQueryProperties().isMaterializedView()) {
+ if (pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS)
+ ||
pCtx.getQueryProperties().hasFeature(QueryFeature.MATERIALIZED_VIEW)) {
if (!oneLoadFileForCtas) { // should not have more than 1 load file
for CTAS.
throw new SemanticException(
"One query is not expected to contain multiple CTAS loads
statements");
@@ -295,7 +299,8 @@ public void compile(final ParseContext pCtx,
oneLoadFileForCtas = false;
}
mvTask.add(TaskFactory.get(
- new MoveWork(pCtx.getQueryProperties().isCTAS() &&
pCtx.getCreateTable().isExternal(), null, null, null,
+ new
MoveWork(pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS)
+ && pCtx.getCreateTable().isExternal(), null, null, null,
lfd, false)));
}
}
@@ -408,14 +413,15 @@ public void compile(final ParseContext pCtx,
// for direct insert CTAS, we don't need this table creation DDL task,
since the table will be created
// ahead of time by the non-native table
- if (pCtx.getQueryProperties().isCTAS() &&
!pCtx.getCreateTable().isMaterialization() && !directInsert) {
+ if (pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS)
+ && !pCtx.getCreateTable().isMaterialization() && !directInsert) {
// generate a DDL task and make it a dependent task of the leaf
CreateTableDesc crtTblDesc = pCtx.getCreateTable();
crtTblDesc.validate(conf);
Task<?> crtTblTask = TaskFactory.get(new DDLWork(inputs, outputs,
crtTblDesc));
patchUpAfterCTASorMaterializedView(rootTasks, inputs, outputs,
crtTblTask,
CollectionUtils.isEmpty(crtTblDesc.getPartColNames()));
- } else if (pCtx.getQueryProperties().isMaterializedView() &&
!directInsert) {
+ } else if
(pCtx.getQueryProperties().hasFeature(QueryFeature.MATERIALIZED_VIEW) &&
!directInsert) {
// generate a DDL task and make it a dependent task of the leaf
CreateMaterializedViewDesc viewDesc = pCtx.getCreateViewDesc();
Task<?> crtViewTask = TaskFactory.get(new DDLWork(
@@ -495,7 +501,7 @@ private Task<?> genTableStats(ParseContext parseContext,
TableScanOperator table
private void setLoadFileLocation(
final ParseContext pCtx, LoadFileDesc lfd) throws SemanticException {
// CTAS; make the move task's destination directory the table's
destination.
- DDLDescWithTableProperties ddlDesc = pCtx.getQueryProperties().isCTAS() ?
+ DDLDescWithTableProperties ddlDesc =
pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS) ?
pCtx.getCreateTable() : pCtx.getCreateViewDesc();
FileSinkDesc dataSink = ddlDesc.getAndUnsetWriter();
@@ -504,7 +510,7 @@ private void setLoadFileLocation(
int stmtId = 0; // CTAS or CMV cannot be part of multi-txn stmt
Path location = (loc == null) ? getDefaultCtasOrCMVLocation(pCtx) : new
Path(loc);
- if (pCtx.getQueryProperties().isCTAS()) {
+ if (pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS)) {
CreateTableDesc ctd = pCtx.getCreateTable();
if (HiveConf.getBoolVar(conf,
HiveConf.ConfVars.CREATE_TABLE_AS_EXTERNAL)) {
ctd.getTblProps().put(hive_metastoreConstants.CTAS_LEGACY_CONFIG,
"true"); // create as external table
@@ -547,12 +553,12 @@ private Path getDefaultCtasOrCMVLocation(final
ParseContext pCtx) throws Semanti
boolean createTableOrMVUseSuffix = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_CREATE_TABLE_USE_SUFFIX)
|| HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_ACID_LOCKLESS_READS_ENABLED);
- if (pCtx.getQueryProperties().isCTAS()) {
+ if (pCtx.getQueryProperties().hasFeature(QueryFeature.CTAS)) {
protoName = pCtx.getCreateTable().getDbTableName();
isExternal = pCtx.getCreateTable().isExternal();
createTableOrMVUseSuffix &=
AcidUtils.isTransactionalTable(pCtx.getCreateTable());
suffix = Utilities.getTableOrMVSuffix(pCtx.getContext(),
createTableOrMVUseSuffix);
- } else if (pCtx.getQueryProperties().isMaterializedView()) {
+ } else if
(pCtx.getQueryProperties().hasFeature(QueryFeature.MATERIALIZED_VIEW)) {
protoName = pCtx.getCreateViewDesc().getViewName();
createTableOrMVUseSuffix &=
AcidUtils.isTransactionalView(pCtx.getCreateViewDesc());
suffix = Utilities.getTableOrMVSuffix(pCtx.getContext(),
createTableOrMVUseSuffix);
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java
index 59a57ca9d30..b0571c47dd5 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/DisallowTransformHook.java
@@ -21,6 +21,7 @@
import org.apache.hadoop.hive.ql.hooks.ExecuteWithHookContext;
import org.apache.hadoop.hive.ql.hooks.HookContext;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
public class DisallowTransformHook implements ExecuteWithHookContext {
@@ -30,7 +31,7 @@ public void run(HookContext hookContext) throws Exception {
if (null == qProps) {
return; // its a ddl query.
}
- if (qProps.usesScript()) {
+ if (qProps.hasFeature(QueryFeature.USES_SCRIPT)) {
throw new HiveAccessControlException("Query with transform clause is
disallowed in"
+ " current configuration.");
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
index 38eff316d0d..d2afbbf0b91 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/lineage/TestGenerator.java
@@ -20,6 +20,7 @@
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.junit.Test;
@@ -41,19 +42,19 @@ public void testCreateFilterPredicateFromConf() {
ParseContext parseContext = new ParseContext();
QueryProperties queryProperties = new QueryProperties();
- queryProperties.setCTAS(true);
+ queryProperties.addFeature(QueryFeature.CTAS);
parseContext.setQueryProperties(queryProperties);
assertThat(predicate.test(parseContext), is(true));
parseContext = new ParseContext();
queryProperties = new QueryProperties();
- queryProperties.setQuery(true);
+ queryProperties.addFeature(QueryFeature.QUERY);
parseContext.setQueryProperties(queryProperties);
assertThat(predicate.test(parseContext), is(true));
parseContext = new ParseContext();
queryProperties = new QueryProperties();
- queryProperties.setView(true);
+ queryProperties.addFeature(QueryFeature.VIEW);
parseContext.setQueryProperties(queryProperties);
assertThat(predicate.test(parseContext), is(false));
}
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
index f41ed705afb..fa062620bb9 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/parse/TestQueryProperties.java
@@ -29,6 +29,7 @@
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.QueryProperties;
+import org.apache.hadoop.hive.ql.QueryProperties.QueryFeature;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
import org.apache.hadoop.hive.ql.lockmgr.HiveTxnManager;
@@ -129,7 +130,7 @@ public void testExceptAll() throws Exception {
}
QueryProperties properties = analyze(
"SELECT t1 FROM test_db.test_table EXCEPT ALL SELECT t2 FROM
test_db.test_table", false);
- Assert.assertTrue(properties.hasExcept());
+ Assert.assertTrue(properties.hasFeature(QueryFeature.EXCEPT));
}
@Test
@@ -140,7 +141,7 @@ public void testExceptDistinct() throws Exception {
}
QueryProperties properties = analyze(
"SELECT t1 FROM test_db.test_table EXCEPT DISTINCT SELECT t2 FROM
test_db.test_table", false);
- Assert.assertTrue(properties.hasExcept());
+ Assert.assertTrue(properties.hasFeature(QueryFeature.EXCEPT));
}
@Test
@@ -151,7 +152,7 @@ public void testIntersectAll() throws Exception {
}
QueryProperties properties = analyze(
"SELECT t1 FROM test_db.test_table INTERSECT ALL SELECT t2 FROM
test_db.test_table", false);
- Assert.assertTrue(properties.hasIntersect());
+ Assert.assertTrue(properties.hasFeature(QueryFeature.INTERSECT));
}
@Test
@@ -162,7 +163,7 @@ public void testIntersectDistinct() throws Exception {
}
QueryProperties properties = analyze(
"SELECT t1 FROM test_db.test_table INTERSECT DISTINCT SELECT t2 FROM
test_db.test_table", false);
- Assert.assertTrue(properties.hasIntersect());
+ Assert.assertTrue(properties.hasFeature(QueryFeature.INTERSECT));
}
@Test
@@ -173,7 +174,7 @@ public void testQualify() throws Exception {
}
QueryProperties properties = analyze(
"SELECT t1 FROM test_db.test_table QUALIFY row_number() OVER
(PARTITION BY t1 ORDER BY t2) = 1", false);
- Assert.assertTrue(properties.hasQualify());
+ Assert.assertTrue(properties.hasFeature(QueryFeature.QUALIFY));
}
@Test
