Author: prasanthj Date: Tue Sep 30 18:05:06 2014 New Revision: 1628504 URL: http://svn.apache.org/r1628504 Log: HIVE-8196: Joining on partition columns with fetch column stats enabled results it very small CE which negatively affects query performance (Prasanth J reviewed by Gunther Hagleitner)
Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java hive/branches/branch-0.14/ql/src/test/queries/clientpositive/annotate_stats_part.q hive/branches/branch-0.14/ql/src/test/results/clientnegative/limit_partition_stats.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_partition_coltype.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/combine2.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/groupby_sort_11.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/input24.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/metadataonly1.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/nullgroup3.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/ppd_constant_where.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/regex_col.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats10.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats12.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats13.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats2.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats7.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats8.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/dynpart_sort_opt_vectorization.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/metadataonly1.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/vectorized_dynamic_partition_pruning.q.out hive/branches/branch-0.14/ql/src/test/results/clientpositive/union_view.q.out Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java (original) +++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/optimizer/optiq/RelOptHiveTable.java Tue Sep 30 18:05:06 2014 @@ -257,19 +257,20 @@ public class RelOptHiveTable extends Rel } colNamesFailedStats.clear(); } else { - Statistics stats = StatsUtils.collectStatistics(hiveConf, partitionList, - hiveTblMetadata, hiveNonPartitionCols, nonPartColNamesThatRqrStats, true, true); - rowCount = stats.getNumRows(); - hiveColStats = new ArrayList<ColStatistics>(); - for (String c : nonPartColNamesThatRqrStats) { - ColStatistics cs = stats.getColumnStatisticsFromColName(c); - if (cs != null) { - hiveColStats.add(cs); - } else { - colNamesFailedStats.add(c); + Statistics stats = StatsUtils.collectStatistics(hiveConf, partitionList, + hiveTblMetadata, hiveNonPartitionCols, nonPartColNamesThatRqrStats, + nonPartColNamesThatRqrStats, true, true); + rowCount = stats.getNumRows(); + hiveColStats = new ArrayList<ColStatistics>(); + for (String c : nonPartColNamesThatRqrStats) { + ColStatistics cs = stats.getColumnStatisticsFromColName(c); + if (cs != null) { + hiveColStats.add(cs); + } else { + colNamesFailedStats.add(c); + } } } - } } catch (HiveException e) { String logMsg = "Collecting stats failed."; LOG.error(logMsg); Modified: hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java (original) +++ hive/branches/branch-0.14/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java Tue Sep 30 18:05:06 2014 @@ -18,8 +18,15 @@ package org.apache.hadoop.hive.ql.stats; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.FileSystem; @@ -80,14 +87,8 @@ import org.apache.hadoop.hive.serde2.obj import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableTimestampObjectInspector; import org.apache.hadoop.io.BytesWritable; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; public class StatsUtils { @@ -113,24 +114,28 @@ public class StatsUtils { // column level statistics are required only for the columns that are needed List<ColumnInfo> schema = tableScanOperator.getSchema().getSignature(); List<String> neededColumns = tableScanOperator.getNeededColumns(); + List<String> referencedColumns = tableScanOperator.getReferencedColumns(); - return collectStatistics(conf, partList, table, schema, neededColumns); + return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns); } private static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, - Table table, List<ColumnInfo> schema, List<String> neededColumns) throws HiveException { + Table table, List<ColumnInfo> schema, List<String> neededColumns, + List<String> referencedColumns) throws HiveException { boolean fetchColStats = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_COLUMN_STATS); boolean fetchPartStats = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_STATS_FETCH_PARTITION_STATS); - return collectStatistics(conf, partList, table, schema, neededColumns, fetchColStats, fetchPartStats); + return collectStatistics(conf, partList, table, schema, neededColumns, referencedColumns, + fetchColStats, fetchPartStats); } public static Statistics collectStatistics(HiveConf conf, PrunedPartitionList partList, Table table, List<ColumnInfo> schema, List<String> neededColumns, - boolean fetchColStats, boolean fetchPartStats) throws HiveException { + List<String> referencedColumns, boolean fetchColStats, boolean fetchPartStats) + throws HiveException { Statistics stats = new Statistics(); @@ -223,7 +228,6 @@ public class StatsUtils { stats.getBasicStatsState().equals(State.COMPLETE)) { stats.setBasicStatsState(State.PARTIAL); } - boolean haveFullStats = fetchColStats; if (fetchColStats) { List<String> partNames = new ArrayList<String>(partList.getNotDeniedPartns().size()); for (Partition part : partList.getNotDeniedPartns()) { @@ -231,37 +235,84 @@ public class StatsUtils { } Map<String, String> colToTabAlias = new HashMap<String, String>(); neededColumns = processNeededColumns(schema, neededColumns, colToTabAlias); - AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(), neededColumns, partNames); + AggrStats aggrStats = Hive.get().getAggrColStatsFor(table.getDbName(), table.getTableName(), + neededColumns, partNames); if (null == aggrStats) { - haveFullStats = false; + // There are some partitions with no state (or we didn't fetch any state). + // Update the stats with empty list to reflect that in the + // state/initialize structures. + List<ColStatistics> emptyStats = Lists.newArrayList(); + + // add partition column stats + addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList, + emptyStats); + + stats.addToColumnStats(emptyStats); + stats.updateColumnStatsState(deriveStatType(emptyStats, referencedColumns)); } else { List<ColumnStatisticsObj> colStats = aggrStats.getColStats(); if (colStats.size() != neededColumns.size()) { - LOG.debug("Column stats requested for : " + neededColumns.size() + " columns. Able to retrieve" - + " for " + colStats.size() + " columns"); + LOG.debug("Column stats requested for : " + neededColumns.size() + " columns. Able to" + + " retrieve for " + colStats.size() + " columns"); } - List<ColStatistics> columnStats = convertColStats(colStats, table.getTableName(), colToTabAlias); + List<ColStatistics> columnStats = convertColStats(colStats, table.getTableName(), + colToTabAlias); + + addParitionColumnStats(neededColumns, referencedColumns, schema, table, partList, + columnStats); + stats.addToColumnStats(columnStats); - State colState = deriveStatType(columnStats, neededColumns); + State colState = deriveStatType(columnStats, referencedColumns); if (aggrStats.getPartsFound() != partNames.size() && colState != State.NONE) { - LOG.debug("Column stats requested for : " + partNames.size() +" partitions. " - + "Able to retrieve for " + aggrStats.getPartsFound() + " partitions"); + LOG.debug("Column stats requested for : " + partNames.size() + " partitions. " + + "Able to retrieve for " + aggrStats.getPartsFound() + " partitions"); colState = State.PARTIAL; } stats.setColumnStatsState(colState); } } - // There are some partitions with no state (or we didn't fetch any state). - // Update the stats with empty list to reflect that in the state/initialize structures. - if (!haveFullStats) { - List<ColStatistics> emptyStats = Lists.<ColStatistics>newArrayList(); - stats.addToColumnStats(emptyStats); - stats.updateColumnStatsState(deriveStatType(emptyStats, neededColumns)); - } } return stats; } + private static void addParitionColumnStats(List<String> neededColumns, + List<String> referencedColumns, List<ColumnInfo> schema, Table table, + PrunedPartitionList partList, List<ColStatistics> colStats) + throws HiveException { + + // extra columns is difference between referenced columns vs needed + // columns. The difference could be partition columns. + List<String> extraCols = Lists.newArrayList(referencedColumns); + if (referencedColumns.size() > neededColumns.size()) { + extraCols.removeAll(neededColumns); + for (String col : extraCols) { + for (ColumnInfo ci : schema) { + // conditions for being partition column + if (col.equals(ci.getInternalName()) && ci.getIsVirtualCol() && + !ci.isHiddenVirtualCol()) { + // currently metastore does not store column stats for + // partition column, so we calculate the NDV from pruned + // partition list + ColStatistics partCS = new ColStatistics(table.getTableName(), + ci.getInternalName(), ci.getType().getTypeName()); + long numPartitions = getNDVPartitionColumn(partList.getPartitions(), + ci.getInternalName()); + partCS.setCountDistint(numPartitions); + colStats.add(partCS); + } + } + } + } + } + + public static int getNDVPartitionColumn(Set<Partition> partitions, String partColName) { + Set<String> distinctVals = new HashSet<String>(partitions.size()); + for (Partition partition : partitions) { + distinctVals.add(partition.getSpec().get(partColName)); + } + return distinctVals.size(); + } + private static void setUnknownRcDsToAverage( List<Long> rowCounts, List<Long> dataSizes, int avgRowSize) { if (LOG.isDebugEnabled()) { @@ -1060,10 +1111,8 @@ public class StatsUtils { /** * Get basic stats of table - * @param dbName - * - database name - * @param tabName - * - table name + * @param table + * - table * @param statType * - type of stats * @return value of stats Modified: hive/branches/branch-0.14/ql/src/test/queries/clientpositive/annotate_stats_part.q URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/queries/clientpositive/annotate_stats_part.q?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/queries/clientpositive/annotate_stats_part.q (original) +++ hive/branches/branch-0.14/ql/src/test/queries/clientpositive/annotate_stats_part.q Tue Sep 30 18:05:06 2014 @@ -65,6 +65,9 @@ explain select zip from loc_orc; -- basicStatState: COMPLETE colStatState: PARTIAL explain select state from loc_orc; +-- basicStatState: COMPLETE colStatState: COMPLETE +explain select year from loc_orc; + -- column statistics for __HIVE_DEFAULT_PARTITION__ is not supported yet. Hence colStatState reports PARTIAL -- basicStatState: COMPLETE colStatState: PARTIAL explain select state,locid from loc_orc; Modified: hive/branches/branch-0.14/ql/src/test/results/clientnegative/limit_partition_stats.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientnegative/limit_partition_stats.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientnegative/limit_partition_stats.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientnegative/limit_partition_stats.q.out Tue Sep 30 18:05:06 2014 @@ -64,31 +64,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: part - Statistics: Num rows: 2000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 4000 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 2000 Data size: 4000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 4000 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_partition_coltype.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_partition_coltype.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_partition_coltype.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/alter_partition_coltype.q.out Tue Sep 30 18:05:06 2014 differ Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/annotate_stats_part.q.out Tue Sep 30 18:05:06 2014 @@ -56,11 +56,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL ListSink PREHOOK: query: insert overwrite table loc_orc partition(year) select * from loc_staging @@ -98,11 +98,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 5 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 5 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 5 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- partition level analyze statistics for specific parition @@ -135,11 +135,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 2 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- basicStatState: PARTIAL colStatState: NONE @@ -158,11 +158,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 9 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 9 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 9 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -181,11 +181,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 7 Data size: 399 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 399 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 7 Data size: 399 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 7 Data size: 399 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- partition level analyze statistics for all partitions @@ -222,11 +222,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 1 Data size: 323 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -245,11 +245,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- basicStatState: COMPLETE colStatState: NONE @@ -268,11 +268,11 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), year (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: NONE + Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: PARTIAL ListSink PREHOOK: query: -- both partitions will be pruned @@ -293,14 +293,14 @@ STAGE PLANS: Processor Tree: TableScan alias: loc_orc - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Filter Operator predicate: ((year = '2001') and (year = '__HIVE_DEFAULT_PARTITION__')) (type: boolean) - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL Select Operator expressions: state (type: string), locid (type: int), zip (type: bigint), '__HIVE_DEFAULT_PARTITION__' (type: string) outputColumnNames: _col0, _col1, _col2, _col3 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: PARTIAL ListSink PREHOOK: query: -- partition level partial column statistics @@ -361,6 +361,29 @@ STAGE PLANS: Statistics: Num rows: 8 Data size: 688 Basic stats: COMPLETE Column stats: PARTIAL ListSink +PREHOOK: query: -- basicStatState: COMPLETE colStatState: COMPLETE +explain select year from loc_orc +PREHOOK: type: QUERY +POSTHOOK: query: -- basicStatState: COMPLETE colStatState: COMPLETE +explain select year from loc_orc +POSTHOOK: type: QUERY +STAGE DEPENDENCIES: + Stage-0 is a root stage + +STAGE PLANS: + Stage: Stage-0 + Fetch Operator + limit: -1 + Processor Tree: + TableScan + alias: loc_orc + Statistics: Num rows: 8 Data size: 722 Basic stats: COMPLETE Column stats: COMPLETE + Select Operator + expressions: year (type: string) + outputColumnNames: _col0 + Statistics: Num rows: 8 Data size: 1472 Basic stats: COMPLETE Column stats: COMPLETE + ListSink + PREHOOK: query: -- column statistics for __HIVE_DEFAULT_PARTITION__ is not supported yet. Hence colStatState reports PARTIAL -- basicStatState: COMPLETE colStatState: PARTIAL explain select state,locid from loc_orc @@ -512,7 +535,7 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 399 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: locid (type: int), year (type: string) outputColumnNames: _col0, _col1 @@ -548,14 +571,14 @@ STAGE PLANS: Statistics: Num rows: 7 Data size: 399 Basic stats: COMPLETE Column stats: COMPLETE Filter Operator predicate: (locid > 0) (type: boolean) - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE Select Operator expressions: locid (type: int), year (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE File Output Operator compressed: false - Statistics: Num rows: 2 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2 Data size: 176 Basic stats: COMPLETE Column stats: COMPLETE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/auto_sortmerge_join_12.q.out Tue Sep 30 18:05:06 2014 differ Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/combine2.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/combine2.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/combine2.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/combine2.q.out Tue Sep 30 18:05:06 2014 @@ -185,18 +185,18 @@ STAGE PLANS: Map Operator Tree: TableScan alias: combine2 - Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE GatherStats: false Select Operator - Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 12 Data size: 14 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE tag: -1 value expressions: _col0 (type: bigint) auto parallelism: false @@ -578,17 +578,17 @@ STAGE PLANS: aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false GlobalTableId: 0 #### A masked pattern was here #### NumFilesPerFileSink: 1 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE #### A masked pattern was here #### table: input format: org.apache.hadoop.mapred.TextInputFormat @@ -652,22 +652,22 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: ds - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(1) keys: ds (type: string) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE value expressions: _col1 (type: bigint) Reduce Operator Tree: Group By Operator @@ -675,14 +675,14 @@ STAGE PLANS: keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string), _col1 (type: bigint) outputColumnNames: _col0, _col1 - Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1000 Data size: 192000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/dynpart_sort_opt_vectorization.q.out Tue Sep 30 18:05:06 2014 @@ -2265,31 +2265,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc - Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -2483,31 +2483,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: over1k_part_buck_sort2_orc - Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 19 Data size: 493 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/groupby_sort_11.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/groupby_sort_11.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/groupby_sort_11.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/groupby_sort_11.q.out Tue Sep 30 18:05:06 2014 @@ -388,32 +388,32 @@ STAGE PLANS: Map Operator Tree: TableScan alias: t1 - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count(DISTINCT 1) keys: 1 (type: int) mode: hash outputColumnNames: _col0, _col1 - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: int) sort order: + - Statistics: Num rows: 1 Data size: 12 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 10 Data size: 70 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator aggregations: count(DISTINCT KEY._col0:0._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 16 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/input24.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/input24.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/input24.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/input24.q.out Tue Sep 30 18:05:06 2014 @@ -29,31 +29,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: x - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/limit_partition_metadataonly.q.out Tue Sep 30 18:05:06 2014 @@ -12,11 +12,11 @@ STAGE PLANS: Processor Tree: TableScan alias: srcpart - Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 500 Data size: 92000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: select ds from srcpart where hr=11 and ds='2008-04-08' @@ -543,34 +543,34 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: hr (type: string) outputColumnNames: hr - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Group By Operator keys: hr (type: string) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator key expressions: _col0 (type: string) sort order: + Map-reduce partition columns: _col0 (type: string) - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Reduce Operator Tree: Group By Operator keys: KEY._col0 (type: string) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: string) outputColumnNames: _col0 - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1000 Data size: 184000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/list_bucket_query_multiskew_3.q.out Tue Sep 30 18:05:06 2014 differ Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/metadataonly1.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/metadataonly1.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/metadataonly1.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/metadataonly1.q.out Tue Sep 30 18:05:06 2014 differ Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/nullgroup3.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/nullgroup3.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/nullgroup3.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/nullgroup3.q.out Tue Sep 30 18:05:06 2014 @@ -40,31 +40,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tstparttbl - Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -131,31 +131,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tstparttbl2 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -230,31 +230,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tstparttbl - Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 5812 Basic stats: PARTIAL Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat @@ -329,31 +329,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: tstparttbl2 - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Select Operator - Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 0 Basic stats: NONE Column stats: NONE Group By Operator aggregations: count(1) mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/ppd_constant_where.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/ppd_constant_where.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/ppd_constant_where.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/ppd_constant_where.q.out Tue Sep 30 18:05:06 2014 @@ -16,31 +16,31 @@ STAGE PLANS: Map Operator Tree: TableScan alias: srcpart - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Select Operator - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Group By Operator aggregations: count() mode: hash outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Reduce Output Operator sort order: - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE value expressions: _col0 (type: bigint) Reduce Operator Tree: Group By Operator aggregations: count(VALUE._col0) mode: mergepartial outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: _col0 (type: bigint) outputColumnNames: _col0 - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE File Output Operator compressed: false - Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE table: input format: org.apache.hadoop.mapred.TextInputFormat output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/regex_col.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/regex_col.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/regex_col.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/regex_col.q.out Tue Sep 30 18:05:06 2014 @@ -37,11 +37,11 @@ STAGE PLANS: Processor Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: EXPLAIN @@ -60,11 +60,11 @@ STAGE PLANS: Processor Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string), hr (type: string) outputColumnNames: _col0, _col1 - Statistics: Num rows: 2000 Data size: 736000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: EXPLAIN @@ -297,11 +297,11 @@ STAGE PLANS: Processor Tree: TableScan alias: srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Select Operator expressions: ds (type: string) outputColumnNames: _col0 - Statistics: Num rows: 2000 Data size: 368000 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE ListSink PREHOOK: query: EXPLAIN Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats10.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats10.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats10.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats10.q.out Tue Sep 30 18:05:06 2014 @@ -373,7 +373,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: bucket3_1 - Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 Stats-Aggr Operator Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats12.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats12.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats12.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats12.q.out Tue Sep 30 18:05:06 2014 differ Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats13.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats13.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats13.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats13.q.out Tue Sep 30 18:05:06 2014 differ Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats2.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats2.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats2.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats2.q.out Tue Sep 30 18:05:06 2014 @@ -136,7 +136,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_t1 - Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 Stats-Aggr Operator Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats7.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats7.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats7.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats7.q.out Tue Sep 30 18:05:06 2014 @@ -47,7 +47,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcpart - Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 Stats-Aggr Operator Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats8.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats8.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== --- hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats8.q.out (original) +++ hive/branches/branch-0.14/ql/src/test/results/clientpositive/stats8.q.out Tue Sep 30 18:05:06 2014 @@ -47,7 +47,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcpart - Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 0 Data size: 23248 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 Stats-Aggr Operator @@ -157,7 +157,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcpart - Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 500 Data size: 5312 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 Stats-Aggr Operator @@ -230,7 +230,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcpart - Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1000 Data size: 10624 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 Stats-Aggr Operator @@ -303,7 +303,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcpart - Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: COMPLETE + Statistics: Num rows: 1500 Data size: 15936 Basic stats: PARTIAL Column stats: NONE Stage: Stage-1 Stats-Aggr Operator @@ -376,7 +376,7 @@ STAGE PLANS: Map Operator Tree: TableScan alias: analyze_srcpart - Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: COMPLETE + Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE Stage: Stage-1 Stats-Aggr Operator Modified: hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out URL: http://svn.apache.org/viewvc/hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out?rev=1628504&r1=1628503&r2=1628504&view=diff ============================================================================== Files hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out (original) and hive/branches/branch-0.14/ql/src/test/results/clientpositive/tez/auto_sortmerge_join_12.q.out Tue Sep 30 18:05:06 2014 differ