zhuzhurk commented on a change in pull request #18376:
URL: https://github.com/apache/flink/pull/18376#discussion_r791440840
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/scheduler/SsgNetworkMemoryCalculationUtils.java
##########
@@ -148,6 +167,80 @@ private static TaskInputsOutputsDescriptor
buildTaskInputsOutputsDescriptor(
return ret;
}
+ private static Map<IntermediateDataSetID, Integer>
getMaxInputChannelNumsForDynamicGraph(
+ ExecutionJobVertex ejv) {
+
+ Map<IntermediateDataSetID, Integer> ret = new HashMap<>();
+
+ final Map<Tuple2<IntermediateDataSetID, ExecutionVertexID>,
ConsumedPartitionGroup>
+ consumedPartitionGroups = new HashMap<>();
+ for (ExecutionVertex vertex : ejv.getTaskVertices()) {
+ for (ConsumedPartitionGroup partitionGroup :
vertex.getAllConsumedPartitionGroups()) {
+ consumedPartitionGroups.put(
+ new
Tuple2<>(partitionGroup.getIntermediateDataSetID(), vertex.getID()),
+ partitionGroup);
+ }
+ }
+
+ for (IntermediateResult consumedResult : ejv.getInputs()) {
+ ret.put(
+ consumedResult.getId(),
+ getMaxInputChannelNumForResult(
+ ejv,
+ consumedResult.getId(),
+ (resultId, vertexId) ->
+ consumedPartitionGroups.get(new
Tuple2<>(resultId, vertexId))));
+ }
+
+ return ret;
+ }
+
+ private static Map<IntermediateDataSetID, Integer>
getMaxSubpartitionNumsForDynamicGraph(
+ ExecutionJobVertex ejv) {
+
+ Map<IntermediateDataSetID, Integer> ret = new HashMap<>();
+
+ for (IntermediateResult intermediateResult :
ejv.getProducedDataSets()) {
+ final int maxNum =
+ Arrays.stream(intermediateResult.getPartitions())
+
.map(IntermediateResultPartition::getNumberOfSubpartitions)
+ .reduce(0, Integer::max);
+ ret.put(intermediateResult.getId(), maxNum);
+ }
+
+ return ret;
+ }
+
+ @VisibleForTesting
+ static int getMaxInputChannelNumForResult(
Review comment:
You are right.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
