deniskuzZ commented on code in PR #6716:
URL: https://github.com/apache/hive/pull/6716#discussion_r4020302258
##########
ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java:
##########
@@ -491,6 +491,9 @@ private static Statistics collectStatistics(HiveConf conf,
PrunedPartitionList p
// Change if we could not retrieve for all partitions
if (aggrStats != null && aggrStats.getPartsFound() !=
partNames.size() && stats.getColumnStatsState() != State.NONE) {
stats.updateColumnStatsState(State.PARTIAL);
+ // values aggregated from a subset of the scanned partitions
estimate, but never
+ // answer; a partition column's stats come from the pruned values
and stay exact
+ aggregatedStats.forEach(colStats ->
colStats.setPartialAggregate(true));
Review Comment:
candidates are a superset of the scanned rows, so the bounds are wider
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java:
##########
@@ -899,39 +590,198 @@ else if (udaf instanceof GenericUDAFCount) {
}
}
- private ColumnStatisticsData
validateSingleColStat(List<ColumnStatisticsObj> statObj) {
- if (statObj.size() > 1) {
- Logger.error("More than one stat for a single column!");
- return null;
- } else if (statObj.isEmpty()) {
- Logger.debug("No stats for some partition and column");
- return null;
- }
- return statObj.get(0).getStatsData();
+ /** The columns the aggregates read, which are the ones statistics have to
be fetched for. */
+ private static List<String> aggregateColumns(GroupByOperator pgbyOp,
Map<String, ExprNodeDesc> exprMap) {
+ return pgbyOp.getConf().getAggregators().stream()
+ .filter(aggr -> !aggr.getParameters().isEmpty())
+ .map(aggr -> aggr.getParameters().get(0))
+ .filter(ExprNodeColumnDesc.class::isInstance)
+ .map(desc -> exprMap.get(((ExprNodeColumnDesc) desc).getColumn()))
+ .filter(ExprNodeColumnDesc.class::isInstance)
+ .map(desc -> ((ExprNodeColumnDesc) desc).getColumn())
+ .distinct()
+ .collect(Collectors.toList());
}
- private Collection<List<ColumnStatisticsObj>> verifyAndGetPartColumnStats(
- Hive hive, Table tbl, String colName, Set<Partition> parts) throws
TException, LockException {
- List<String> partNames = new ArrayList<String>(parts.size());
- for (Partition part : parts) {
- if
(!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(),
part.getParameters(), colName)) {
- Logger.debug("Stats for part : " + part.getSpec() + " column " +
colName
+ /**
+ * The statistics of the columns a scan's aggregates read, fetched once
when the first
+ * aggregate needs them and shared by the rest. An aggregate this rewrite
cannot answer leaves
+ * the query for execution, whole or not at all. Answers for a scan of a
partitioned table.
+ */
+ private static final class ScanColStats {
+ private final Hive hive;
+ private final Table tbl;
+ private final List<String> colNames;
+ private final PrunedPartitionList prunedList;
+ private Map<String, ColumnStatisticsObj> colStatsByName;
+ private boolean fetched;
+
+ ScanColStats(Hive hive, Table tbl, List<String> colNames,
PrunedPartitionList prunedList) {
+ this.hive = hive;
+ this.tbl = tbl;
+ this.colNames = colNames;
+ this.prunedList = prunedList;
+ }
+
+ /**
+ * One column's statistics. A scan pruned to no partitions reads no
rows, and the statistics
+ * of no rows are the empty ones: nothing counted, and no least or
greatest to name.
+ */
+ ColumnStatisticsData statsFor(String colName, StatType type) throws
HiveException {
+ if (prunedList != null && prunedList.getPartitions().isEmpty()) {
+ return emptyColStats(type);
+ }
+ if (!fetched) {
+ fetched = true;
+ colStatsByName = prunedList == null ? tableColStats() :
partitionColStats();
+ }
+ ColumnStatisticsObj stat = colStatsByName == null ? null :
colStatsByName.get(colName);
+ if (stat == null) {
+ Logger.debug("No stats for " + tbl.getTableName() + " column " +
colName);
+ return null;
+ }
+ return stat.getStatsData();
+ }
+
+ /**
+ * Whether the table's own statistics answer for this scan: it keeps
them for the table as
+ * a whole, and the scan reads every partition. They then describe
exactly the rows read.
+ */
+ private boolean answeredByTableStats() {
+ return !StatsUtils.isPartitionStats(tbl, hive.getConf()) &&
+ prunedList.getReferredPartCols().isEmpty() &&
!prunedList.hasUnknownPartitions();
+ }
+
+ /** The table's own statistics, taken only while they still describe it.
*/
+ private Map<String, ColumnStatisticsObj> tableColStats() throws
HiveException {
+ if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl,
tbl.getParameters(), colNames)) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " columns "
+ colNames
+ " are not up to date.");
return null;
}
- partNames.add(part.getName());
+ return indexByColumnName(hive.getTableColumnStatistics(tbl, colNames,
true));
+ }
+
+ /** What the scan's partitions hold for every column asked about, or
null to decline. */
+ private Map<String, ColumnStatisticsObj> partitionColStats() throws
HiveException {
+ Set<Partition> parts = prunedList.getPartitions();
Review Comment:
pruned list is exactly what the scan read
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
