johnyangk commented on a change in pull request #111: [NEMO-139, 6] Logic in the scheduler for appending jobs, Support RDD caching URL: https://github.com/apache/incubator-nemo/pull/111#discussion_r211455846
########## File path: runtime/master/src/main/java/edu/snu/nemo/runtime/master/PlanAppender.java ########## @@ -0,0 +1,128 @@ +/* + * Copyright (C) 2018 Seoul National University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package edu.snu.nemo.runtime.master; + +import edu.snu.nemo.common.Pair; +import edu.snu.nemo.common.dag.DAG; +import edu.snu.nemo.common.dag.DAGBuilder; +import edu.snu.nemo.common.ir.IdManager; +import edu.snu.nemo.common.ir.edge.executionproperty.CacheIDProperty; +import edu.snu.nemo.common.ir.edge.executionproperty.DuplicateEdgeGroupProperty; +import edu.snu.nemo.common.ir.edge.executionproperty.DuplicateEdgeGroupPropertyValue; +import edu.snu.nemo.common.ir.vertex.CachedSourceVertex; +import edu.snu.nemo.common.ir.vertex.IRVertex; +import edu.snu.nemo.common.ir.vertex.executionproperty.MarkerProperty; +import edu.snu.nemo.runtime.common.exception.PlanAppenderException; +import edu.snu.nemo.runtime.common.plan.PhysicalPlan; +import edu.snu.nemo.runtime.common.plan.RuntimeEdge; +import edu.snu.nemo.runtime.common.plan.Stage; +import edu.snu.nemo.runtime.common.plan.StageEdge; + +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +/** + * An implementation of plan appender which append two plans regarding to caching. + */ +public final class PlanAppender { + + /** + * Private constructor. + */ + private PlanAppender() { + // Private constructor. + } + + /** + * Append to plans regarding to caching. + * + * @param originalPlan the original plan. + * @param planToAppend the plan to append. + * @return the appended plan. + */ + public static PhysicalPlan appendPlan(final PhysicalPlan originalPlan, + final PhysicalPlan planToAppend) { + // Add the stage DAG of the original plan to the builder at first. + final DAGBuilder<Stage, StageEdge> physicalDAGBuilder = new DAGBuilder<>(originalPlan.getStageDAG()); + + // Scan cached data in the original plan. + final Map<UUID, StageEdge> cachedEdges = new HashMap<>(); + originalPlan.getStageDAG().getVertices().forEach( + stage -> originalPlan.getStageDAG().getIncomingEdgesOf(stage).stream() + // Cached edge toward a ghost is a representative edge. Review comment: This seems like an important assumption. Can you add a pointer to comments in {@link MarkerProperty}, or a TODO to emphasize/explain this assumption? ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: us...@infra.apache.org With regards, Apache Git Services
