difin commented on code in PR #6716:
URL: https://github.com/apache/hive/pull/6716#discussion_r4041403872
##########
ql/src/java/org/apache/hadoop/hive/ql/optimizer/StatsOptimizer.java:
##########
@@ -899,39 +592,202 @@ else if (udaf instanceof GenericUDAFCount) {
}
}
- private ColumnStatisticsData
validateSingleColStat(List<ColumnStatisticsObj> statObj) {
- if (statObj.size() > 1) {
- Logger.error("More than one stat for a single column!");
- return null;
- } else if (statObj.isEmpty()) {
- Logger.debug("No stats for some partition and column");
- return null;
+ /** The columns the aggregates read, which are the ones statistics have to
be fetched for. */
+ private static List<String> aggregateColumns(GroupByOperator pgbyOp,
Map<String, ExprNodeDesc> exprMap) {
+ Set<String> columns = new LinkedHashSet<>();
+ for (AggregationDesc aggr : pgbyOp.getConf().getAggregators()) {
+ List<ExprNodeDesc> params = aggr.getParameters();
+ if (!params.isEmpty()
+ && params.getFirst() instanceof ExprNodeColumnDesc param
+ && exprMap.get(param.getColumn()) instanceof ExprNodeColumnDesc
column) {
+ columns.add(column.getColumn());
+ }
}
- return statObj.get(0).getStatsData();
+ return List.copyOf(columns);
}
- private Collection<List<ColumnStatisticsObj>> verifyAndGetPartColumnStats(
- Hive hive, Table tbl, String colName, Set<Partition> parts) throws
TException, LockException {
- List<String> partNames = new ArrayList<String>(parts.size());
- for (Partition part : parts) {
- if
(!StatsUtils.areColumnStatsUptoDateForQueryAnswering(part.getTable(),
part.getParameters(), colName)) {
- Logger.debug("Stats for part : " + part.getSpec() + " column " +
colName
+ /**
+ * The statistics of the columns a scan's aggregates read, fetched once
when the first
+ * aggregate needs them and shared by the rest. An aggregate this rewrite
cannot answer leaves
+ * the query for execution, whole or not at all. Answers for a scan of a
partitioned table.
+ */
+ private static final class ScanColStats {
+ private final Hive hive;
+ private final Table tbl;
+ private final List<String> colNames;
+ private final PrunedPartitionList prunedList;
+ private Map<String, ColumnStatisticsObj> colStatsByName;
+ private boolean fetched;
+
+ ScanColStats(Hive hive, Table tbl, List<String> colNames,
PrunedPartitionList prunedList) {
+ this.hive = hive;
+ this.tbl = tbl;
+ this.colNames = colNames;
+ this.prunedList = prunedList;
+ }
+
+ /**
+ * One column's statistics. A scan pruned to no partitions reads no
rows, and the statistics
+ * of no rows are the empty ones: nothing counted, and no least or
greatest to name.
+ */
+ ColumnStatisticsData statsFor(String colName, StatType type) throws
HiveException {
+ if (prunedList != null && prunedList.getPartitions().isEmpty()) {
+ return emptyColStats(type);
+ }
+ if (!fetched) {
+ fetched = true;
+ colStatsByName = prunedList == null ? tableColStats() :
partitionColStats();
+ }
+ ColumnStatisticsObj stat = colStatsByName == null ? null :
colStatsByName.get(colName);
+ if (stat == null) {
+ Logger.debug("No stats for " + tbl.getTableName() + " column " +
colName);
+ return null;
+ }
+ return stat.getStatsData();
+ }
+
+ /**
+ * Whether the table's own statistics answer for this scan: it keeps
them for the table as
+ * a whole, and the scan reads every partition. They then describe
exactly the rows read.
+ */
+ private boolean answeredByTableStats() {
+ return !StatsUtils.isPartitionStats(tbl, hive.getConf()) &&
+ prunedList.getReferredPartCols().isEmpty() &&
!prunedList.hasUnknownPartitions();
+ }
+
+ /** The table's own statistics, taken only while they still describe it.
*/
+ private Map<String, ColumnStatisticsObj> tableColStats() throws
HiveException {
+ if (!StatsUtils.areColumnStatsUptoDateForQueryAnswering(tbl,
tbl.getParameters(), colNames)) {
+ Logger.debug("Stats for table : " + tbl.getTableName() + " columns "
+ colNames
+ " are not up to date.");
Review Comment:
It would be more readable here to use the '{}' syntax
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
