englefly commented on code in PR #69322:
URL: https://github.com/apache/doris/pull/69322#discussion_r4346732502
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/stats/HboUtils.java:
##########
@@ -502,27 +548,77 @@ private static InputTableStatisticsInfo
buildHboInputTableStatisticsInfo(Physica
* @param planToIdMap planToIdMap
* @param scanToFilterMap scanToFilterMap
* @param curPlanNodeRuntimeStats curPlanNodeRuntimeStats
+ * @param nodeFingerprints snapshot of nereids plan node id -> hbo
fingerprint taken during
+ * planning (only populated when the struct-info fingerprint is
enabled)
* @return plan statistics map
*/
public static Map<PlanNodeAndHash, PlanStatisticsWithInputInfo>
genPlanStatisticsMap(
Map<Integer, PhysicalPlan> idToPlanMap, Map<PhysicalPlan, Integer>
planToIdMap,
Map<RelationId, Set<Expression>> scanToFilterMap,
- List<TPlanNodeRuntimeStatsItem> curPlanNodeRuntimeStats) {
+ List<TPlanNodeRuntimeStatsItem> curPlanNodeRuntimeStats,
+ Map<Integer, String> nodeFingerprints) {
Map<PlanNodeAndHash, PlanStatisticsWithInputInfo>
outputPlanStatisticsMap = new HashMap<>();
+ // the fingerprint a plan node was published under, so a second node
with the same fingerprint
+ // can be folded into it (the map above is keyed by the plan node
identity as well)
+ Map<String, PlanNodeAndHash> publishedKeyByHash = new HashMap<>();
+ int missingFingerprints = 0;
+ int skippedWithoutSnapshot = 0;
for (TPlanNodeRuntimeStatsItem nodeStats : curPlanNodeRuntimeStats) {
int nodeId = nodeStats.node_id;
PhysicalPlan planNode = idToPlanMap.get(nodeId);
- if (planNode != null) {
- PlanStatistics curPlanStatistics =
PlanStatistics.buildFromStatsItem(
- nodeStats, planNode, scanToFilterMap);
- InputTableStatisticsInfo inputTableStatisticsInfo =
buildHboInputTableStatisticsInfo(
- planNode, planToIdMap, scanToFilterMap,
curPlanNodeRuntimeStats);
- Optional<String> hash = inputTableStatisticsInfo.getHash();
- PlanNodeAndHash planNodeAndHash = new
PlanNodeAndHash((AbstractPlan) planNode, hash);
- PlanStatisticsWithInputInfo planHashWithInputInfo = new
PlanStatisticsWithInputInfo(
- nodeId, curPlanStatistics, inputTableStatisticsInfo);
- outputPlanStatisticsMap.put(planNodeAndHash,
planHashWithInputInfo);
+ if (planNode == null) {
+ continue;
}
+ PlanStatistics curPlanStatistics =
PlanStatistics.buildFromStatsItem(
+ nodeStats, planNode, scanToFilterMap);
+ InputTableStatisticsInfo inputTableStatisticsInfo =
buildHboInputTableStatisticsInfo(
+ planNode, planToIdMap, scanToFilterMap,
curPlanNodeRuntimeStats);
+ Optional<String> hash;
+ if (nodeFingerprints != null &&
nodeFingerprints.containsKey(nodeId)) {
+ String snapshot = nodeFingerprints.get(nodeId);
+ if (GroupStructInfo.UNSUPPORTED_FINGERPRINT.equals(snapshot)) {
+ // the node is deliberately excluded from the hbo
fingerprint (its group is
+ // unsupported), so its missing fingerprint is not a
diagnostic gap
+ continue;
+ }
+ hash = Optional.ofNullable(snapshot);
+ } else {
+ // no planning-time snapshot for this node: skip it
+ skippedWithoutSnapshot++;
+ if (nodeFingerprints != null && !nodeFingerprints.isEmpty()) {
+ missingFingerprints++;
+ }
+ continue;
+ }
+ if (!hash.isPresent()) {
+ continue;
+ }
+ PlanNodeAndHash planNodeAndHash = new
PlanNodeAndHash((AbstractPlan) planNode, hash);
+ PlanNodeAndHash publishedKey = publishedKeyByHash.get(hash.get());
+ if (publishedKey == null) {
+ publishedKeyByHash.put(hash.get(), planNodeAndHash);
+ outputPlanStatisticsMap.put(planNodeAndHash, new
PlanStatisticsWithInputInfo(
+ nodeId, curPlanStatistics, inputTableStatisticsInfo));
+ continue;
+ }
+ // a second runtime node with the same fingerprint (the two scans
of a self join, for
+ // instance): the provider stores one entry per fingerprint, so
the input statistics of both
+ // nodes are folded here - writing both would let the last write
discard the other node's
+ // rows, and the drift comparison of the published entry expects
the rows of every scan
+ PlanStatisticsWithInputInfo published =
outputPlanStatisticsMap.get(publishedKey);
+ outputPlanStatisticsMap.put(publishedKey, new
PlanStatisticsWithInputInfo(
+ published.getId(), published.getPlanStatistics(),
Review Comment:
learned 条目不在本pr处理. learned hbo 要删掉重新实现
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
