Github user fhueske commented on a diff in the pull request: https://github.com/apache/flink/pull/1553#discussion_r53153223 --- Diff: flink-optimizer/src/main/java/org/apache/flink/optimizer/operators/ReduceProperties.java --- @@ -59,37 +65,69 @@ public DriverStrategy getStrategy() { @Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD || - (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) + (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) { + // adjust a sort (changes grouping, so it must be for this driver to combining sort + if(in.getSource().getOptimizerNode() instanceof PartitionNode) { + Channel toCombiner = new Channel(in.getSource()); + toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); + // create an input node for combine with same parallelism as input node + ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode(); + combinerNode.setParallelism(in.getSource().getParallelism()); + if(toCombiner.getSource().getInputs().iterator().hasNext()) { + Channel source = toCombiner.getSource().getInputs().iterator().next(); + SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, + "Combine ("+node.getOperator().getName()+")", source, + DriverStrategy.SORTED_PARTIAL_REDUCE, this.keyList); + addCombinerProperties(toCombiner, combiner); + Channel combinerChannel = new Channel(combiner); + combinerChannel.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); + // Create the partition single input plan node from the existing partition node + PlanNode partitionplanNode = in.getSource().getPlanNode(); + SingleInputPlanNode partition = new SingleInputPlanNode(in.getSource().getOptimizerNode(), partitionplanNode.getNodeName(), + combinerChannel, partitionplanNode.getDriverStrategy()); + partition.setCosts(partitionplanNode.getNodeCosts()); + partition.initProperties(partitionplanNode.getGlobalProperties(), partitionplanNode.getLocalProperties()); + // Create a reducer such that the input of the reducer is the partition node + Channel toReducer = new Channel(partition); + toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), + in.getShipStrategySortOrder(), in.getDataExchangeMode()); + return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", toReducer, + DriverStrategy.SORTED_REDUCE, this.keyList); + } + } return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in, - DriverStrategy.SORTED_REDUCE, this.keyList); + DriverStrategy.SORTED_REDUCE, this.keyList); } else { // non forward case. all local properties are killed anyways, so we can safely plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); - + // create an input node for combine with same parallelism as input node ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, - "Combine ("+node.getOperator().getName()+")", toCombiner, --- End diff -- No reformatting please.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---