Author: hashutosh
Date: Fri Sep 19 15:10:37 2014
New Revision: 1626244
URL: http://svn.apache.org/r1626244
Log:
HIVE-7359 : Stats based compute query replies fail to do simple column
transforms (Ashutosh Chauhan via Gopal V)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java?rev=1626244&r1=1626243&r2=1626244&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java
Fri Sep 19 15:10:37 2014
@@ -193,11 +193,12 @@ public class StatsOptimizer implements T
}
SelectOperator selOp = (SelectOperator)tsOp.getChildren().get(0);
for(ExprNodeDesc desc : selOp.getConf().getColList()) {
- if (!(desc instanceof ExprNodeColumnDesc)) {
+ if (!((desc instanceof ExprNodeColumnDesc) || (desc instanceof
ExprNodeConstantDesc))) {
// Probably an expression, cant handle that
return null;
}
}
+ Map<String, ExprNodeDesc> exprMap = selOp.getColumnExprMap();
// Since we have done an exact match on TS-SEL-GBY-RS-GBY-SEL-FS
// we need not to do any instanceof checks for following.
GroupByOperator gbyOp = (GroupByOperator)selOp.getChildren().get(0);
@@ -215,6 +216,12 @@ public class StatsOptimizer implements T
return null;
}
+ for(ExprNodeDesc desc : selOp.getConf().getColList()) {
+ if (!(desc instanceof ExprNodeColumnDesc)) {
+ // Probably an expression, cant handle that
+ return null;
+ }
+ }
FileSinkOperator fsOp = (FileSinkOperator)(selOp.getChildren().get(0));
if (fsOp.getChildOperators() != null &&
fsOp.getChildOperators().size() > 0) {
// looks like a subq plan.
@@ -236,22 +243,28 @@ public class StatsOptimizer implements T
GenericUDAFResolver udaf =
FunctionRegistry.getGenericUDAFResolver(aggr.getGenericUDAFName());
if (udaf instanceof GenericUDAFSum) {
- if(!(aggr.getParameters().get(0) instanceof ExprNodeConstantDesc)){
+ ExprNodeDesc desc = aggr.getParameters().get(0);
+ String constant;
+ if (desc instanceof ExprNodeConstantDesc) {
+ constant = ((ExprNodeConstantDesc) desc).getValue().toString();
+ } else if (desc instanceof ExprNodeColumnDesc &&
exprMap.get(((ExprNodeColumnDesc)desc).getColumn()) instanceof
ExprNodeConstantDesc) {
+ constant =
((ExprNodeConstantDesc)exprMap.get(((ExprNodeColumnDesc)desc).getColumn())).getValue().toString();
+ } else {
return null;
}
Long rowCnt = getRowCnt(pctx, tsOp, tbl);
if(rowCnt == null) {
return null;
}
- oneRow.add(HiveDecimal.create(((ExprNodeConstantDesc)
aggr.getParameters().get(0))
- .getValue().toString()).multiply(HiveDecimal.create(rowCnt)));
+
oneRow.add(HiveDecimal.create(constant).multiply(HiveDecimal.create(rowCnt)));
ois.add(PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
PrimitiveCategory.DECIMAL));
}
else if (udaf instanceof GenericUDAFCount) {
Long rowCnt = 0L;
- if ((aggr.getParameters().isEmpty() || aggr.getParameters().get(0)
instanceof
- ExprNodeConstantDesc)) {
+ if (aggr.getParameters().isEmpty() || aggr.getParameters().get(0)
instanceof
+ ExprNodeConstantDesc || ((aggr.getParameters().get(0)
instanceof ExprNodeColumnDesc) &&
+
exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn())
instanceof ExprNodeConstantDesc)) {
// Its either count (*) or count(1) case
rowCnt = getRowCnt(pctx, tsOp, tbl);
if(rowCnt == null) {
@@ -259,12 +272,7 @@ public class StatsOptimizer implements T
}
} else {
// Its count(col) case
- if (!(aggr.getParameters().get(0) instanceof
ExprNodeColumnDesc)) {
- // this is weird, we got expr or something in there, bail out
- Log.debug("Unexpected expression : " +
aggr.getParameters().get(0));
- return null;
- }
- ExprNodeColumnDesc desc =
(ExprNodeColumnDesc)aggr.getParameters().get(0);
+ ExprNodeColumnDesc desc =
(ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
String colName = desc.getColumn();
StatType type = getType(desc.getTypeString());
if(!tbl.isPartitioned()) {
@@ -330,7 +338,7 @@ public class StatsOptimizer implements T
ois.add(PrimitiveObjectInspectorFactory.
getPrimitiveJavaObjectInspector(PrimitiveCategory.LONG));
} else if (udaf instanceof GenericUDAFMax) {
- ExprNodeColumnDesc colDesc =
(ExprNodeColumnDesc)aggr.getParameters().get(0);
+ ExprNodeColumnDesc colDesc =
(ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
String colName = colDesc.getColumn();
StatType type = getType(colDesc.getTypeString());
if(!tbl.isPartitioned()) {
@@ -419,7 +427,7 @@ public class StatsOptimizer implements T
}
}
} else if (udaf instanceof GenericUDAFMin) {
- ExprNodeColumnDesc colDesc =
(ExprNodeColumnDesc)aggr.getParameters().get(0);
+ ExprNodeColumnDesc colDesc =
(ExprNodeColumnDesc)exprMap.get(((ExprNodeColumnDesc)aggr.getParameters().get(0)).getColumn());
String colName = colDesc.getColumn();
StatType type = getType(colDesc.getTypeString());
if (!tbl.isPartitioned()) {
Modified:
hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q?rev=1626244&r1=1626243&r2=1626244&view=diff
==============================================================================
---
hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
(original)
+++
hive/trunk/ql/src/test/queries/clientpositive/metadata_only_queries_with_filters.q
Fri Sep 19 15:10:37 2014
@@ -46,6 +46,8 @@ explain
select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin),
count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt > 2010;
select count(*), count(1), sum(1), sum(2), count(s), count(bo), count(bin),
count(si), max(i), min(b), max(f), min(d) from stats_tbl_part where dt > 2010;
+select count(*) from stats_tbl_part;
+select count(*)/2 from stats_tbl_part;
drop table stats_tbl_part;
set hive.compute.query.using.stats=false;
set hive.stats.dbclass=jdbc:derby;
Modified:
hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out?rev=1626244&r1=1626243&r2=1626244&view=diff
==============================================================================
---
hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out
(original)
+++
hive/trunk/ql/src/test/results/clientpositive/metadata_only_queries_with_filters.q.out
Fri Sep 19 15:10:37 2014
@@ -188,6 +188,26 @@ POSTHOOK: query: select count(*), count(
POSTHOOK: type: QUERY
#### A masked pattern was here ####
2219 2219 2219 4438 2219 2219 2219 2219 65791
4294967296 99.95999908447266 0.04
+PREHOOK: query: select count(*) from stats_tbl_part
+PREHOOK: type: QUERY
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from stats_tbl_part
+POSTHOOK: type: QUERY
+#### A masked pattern was here ####
+4541
+PREHOOK: query: select count(*)/2 from stats_tbl_part
+PREHOOK: type: QUERY
+PREHOOK: Input: default@stats_tbl_part
+PREHOOK: Input: default@stats_tbl_part@dt=2010
+PREHOOK: Input: default@stats_tbl_part@dt=2014
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)/2 from stats_tbl_part
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@stats_tbl_part
+POSTHOOK: Input: default@stats_tbl_part@dt=2010
+POSTHOOK: Input: default@stats_tbl_part@dt=2014
+#### A masked pattern was here ####
+2270.5
PREHOOK: query: drop table stats_tbl_part
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@stats_tbl_part
