difin commented on code in PR #6716:
URL: https://github.com/apache/hive/pull/6716#discussion_r4041410622
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java:
##########
@@ -899,39 +592,202 @@ else if (udaf instanceof GenericUDAFCount) {
}
}
- private ColumnStatisticsData
validateSingleColStat(List<ColumnStatisticsObj> statObj) {
- if (statObj.size() > 1) {
- Logger.error("More than one stat for a single column!");
- return null;
- } else if (statObj.isEmpty()) {
- Logger.debug("No stats for some partition and column");
- return null;
+ /** The columns the aggregates read, which are the ones statistics have to
be fetched for. */
+ private static List<String> aggregateColumns(GroupByOperator pgbyOp,
Map<String, ExprNodeDesc> exprMap) {
+ Set<String> columns = new LinkedHashSet<>();
+ for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) {
+ List<ExprNodeDesc> params = aggr.getParameters();
+ if (!params.isEmpty()
+ && params.getFirst() instanceof ExprNodeColumnDesc param
+ && exprMap.get(param.getColumn()) instanceof ExprNodeColumnDesc
column) {
+ columns.add(column.getColumn());
+ }
}
- return statObj.get(0).getStatsData();
+ return List.copyOf(columns);
}
- private Collection<List<ColumnStatisticsObj>> verifyAndGetPartColumnStats(
- Hive hive, Table tbl, String colName, Set<Partition> parts) throws
TException, LockException {
- List<String> partNames = new ArrayList<String>(parts.size());
- for (Partition part : parts) {
- if
(!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(),
part.getParameters(), colName)) {
- Logger.debug("Stats for part : " + part.getSpec() + " column " +
colName
+ /**
+ * The statistics of the columns a scan's aggregates read, fetched once
when the first
+ * aggregate needs them and shared by the rest. An aggregate this rewrite
cannot answer leaves
+ * the query for execution, whole or not at all. Answers for a scan of a
partitioned table.
+ */
+ private static final class ScanColStats {
+ private final Hive hive;
+ private final Table tbl;
+ private final List<String> colNames;
+ private final PrunedPartitionList prunedList;
+ private Map<String, ColumnStatisticsObj> colStatsByName;
+ private boolean fetched;
+
+ ScanColStats(Hive hive, Table tbl, List<String> colNames,
PrunedPartitionList prunedList) {
+ this.hive = hive;
+ this.tbl = tbl;
+ this.colNames = colNames;
+ this.prunedList = prunedList;
+ }
+
+ /**
+ * One column's statistics. A scan pruned to no partitions reads no
rows, and the statistics
+ * of no rows are the empty ones: nothing counted, and no least or
greatest to name.
+ */
+ ColumnStatisticsData statsFor(String colName, StatType type) throws
HiveException {
+ if (prunedList != null && prunedList.getPartitions().isEmpty()) {
+ return emptyColStats(type);
+ }
+ if (!fetched) {
+ fetched = true;
+ colStatsByName = prunedList == null ? tableColStats() :
partitionColStats();
+ }
+ ColumnStatisticsObj stat = colStatsByName == null ? null :
colStatsByName.get(colName);
+ if (stat == null) {
+ Logger.debug("No stats for " + tbl.getTableName() + " column " +
colName);
+ return null;
+ }
+ return stat.getStatsData();
+ }
+
+ /**
+ * Whether the table's own statistics answer for this scan: it keeps
them for the table as
+ * a whole, and the scan reads every partition. They then describe
exactly the rows read.
+ */
+ private boolean answeredByTableStats() {
+ return !StatsUtils.isPartitionStats(tbl, hive.getConf()) &&
+ prunedList.getReferredPartCols().isEmpty() &&
!prunedList.hasUnknownPartitions();
+ }
+
+ /** The table's own statistics, taken only while they still describe it.
*/
+ private Map<String, ColumnStatisticsObj> tableColStats() throws
HiveException {
+ if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl,
tbl.getParameters(), colNames)) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " columns "
+ colNames
+ " are not up to date.");
return null;
}
- partNames.add(part.getName());
+ return indexByColumnName(hive.getTableColumnStatistics(tbl, colNames,
true));
+ }
+
+ /** What the scan's partitions hold for every column asked about, or
null to decline. */
+ private Map<String, ColumnStatisticsObj> partitionColStats() throws
HiveException {
+ Set<Partition> parts = prunedList.getPartitions();
+ List<String> partNames = new ArrayList<>(parts.size());
+ // a storage handler holds no partition parameters, and one kept per
partition describes no
+ // partition in particular: whether each still describes itself is
answered by the aggregate
+ // below, which is told the partitions this query pruned to
+ if (tbl.isNonNative()) {
+ if (!StatsUtils.checkCanProvideColumnStats(tbl)) {
+ Logger.debug("Table : " + tbl.getTableName() + " provides no
column statistics.");
+ return null;
+ }
+ if (answeredByTableStats()) {
+ return tableColStats();
+ }
+ parts.forEach(part -> partNames.add(part.getName()));
+ } else {
+ for (Partition part : parts) {
+ if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(
+ part.getTable(), part.getParameters(), colNames)) {
+ Logger.debug("Stats for part : " + part.getSpec() + " columns "
+ colNames
+ + " are not up to date.");
+ return null;
+ }
+ partNames.add(part.getName());
+ }
+ }
+ // Aggregated rather than per partition: the callers fold these with
min, max or a sum, so
+ // merging first gives the same answer. A handler aggregates its own
statistics, which
+ // the metastore cannot hold: PART_COL_STATS rows need a partition
Iceberg never creates.
+ AggrStats aggrStats;
+ try {
+ aggrStats = tbl.isNonNative()
+ ? tbl.getStorageHandler().getAggrColStatsFor(tbl, colNames,
partNames)
+ : exactAggrColStats(partNames);
+ } catch (MetaException e) {
+ throw new HiveException(e);
+ }
+ if (aggrStats == null || aggrStats.getColStats() == null) {
+ Logger.debug("No stats for " + tbl.getTableName() + " columns " +
colNames);
+ return null;
+ }
+ if (aggrStats.getPartsFound() != parts.size()) {
+ // a partition whose statistics are missing would leave the answer
describing a subset
+ Logger.debug("Received " + aggrStats.getPartsFound() + " stats for "
+ parts.size() + " partitions");
Review Comment:
same
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
