Repository: crunch Updated Branches: refs/heads/master e3ba6fcd0 -> e520d9f6e
CRUNCH-638: Improve dot file generation for better supportability. Contributed by GergŠPásztor. Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/e520d9f6 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/e520d9f6 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/e520d9f6 Branch: refs/heads/master Commit: e520d9f6e55bdcd16e5ad9715f46ddce4ce96f76 Parents: e3ba6fc Author: Tom White <[email protected]> Authored: Tue Mar 7 14:38:52 2017 +0000 Committer: Tom White <[email protected]> Committed: Tue Mar 7 14:38:52 2017 +0000 ---------------------------------------------------------------------- .../apache/crunch/impl/mr/plan/DotfilesIT.java | 3 +- .../apache/crunch/impl/mr/plan/DotfileUtil.java | 30 ++++++++++++++++---- .../apache/crunch/impl/mr/plan/MSCRPlanner.java | 5 ++-- 3 files changed, 29 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java b/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java index c33348a..99e25b7 100644 --- a/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java +++ b/crunch-core/src/it/java/org/apache/crunch/impl/mr/plan/DotfilesIT.java @@ -116,10 +116,11 @@ public class DotfilesIT { String[] dotfileNames = dotfileNames(dotfileDir.getRootFile()); - assertEquals(5, dotfileNames.length); + assertEquals(6, dotfileNames.length); assertTrue(containsFileEndingWith(dotfileNames, "jobplan.dot")); assertTrue(containsFileEndingWith(dotfileNames, "split_graph_plan.dot")); + assertTrue(containsFileEndingWith(dotfileNames, "split_graph_with_components_plan.dot")); assertTrue(containsFileEndingWith(dotfileNames, "rt_plan.dot")); assertTrue(containsFileEndingWith(dotfileNames, "base_graph_plan.dot")); assertTrue(containsFileEndingWith(dotfileNames, "lineage_plan.dot")); http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java index 6475e87..ee854d9 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/DotfileUtil.java @@ -47,6 +47,7 @@ public class DotfileUtil { private String rtNodesDotfile = ""; private String basePlanGraphDotfile = ""; private String splitGraphPlanDotfile = ""; + private String splitGraphWithComponentsPlanDotfile = ""; private String pcollectionLineageDotfile = ""; private String planDotFile = ""; @@ -75,8 +76,22 @@ public class DotfileUtil { void buildBaseGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph) { if (isDebugDotfilesEnabled(conf)) { try { - basePlanGraphDotfile = new DotfileWriterGraph(graph, outputs, null).buildDiagram("Base Graph (" - + jarClass.getSimpleName() + ")"); + basePlanGraphDotfile = new DotfileWriterGraph(graph, outputs, null) + .buildDiagram("Base Graph (" + jarClass.getSimpleName() + ")"); + } catch (Exception ex) { + LOG.error("Problem creating debug dotfile:", ex); + } + } + } + + /** + * Builds the split graph dotfile only if the dotfile-debug mode is enabled. + */ + void buildSplitGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph) { + if (isDebugDotfilesEnabled(conf)) { + try { + splitGraphPlanDotfile = new DotfileWriterGraph(graph, outputs, null) + .buildDiagram("Split Graph (" + jarClass.getSimpleName() + ")"); } catch (Exception ex) { LOG.error("Problem creating debug dotfile:", ex); } @@ -84,13 +99,15 @@ public class DotfileUtil { } /** - * Builds a split graph dotfile only if the dotfile-debug mode is enabled. + * Builds a split graph with components dotfile only if the dotfile-debug mode is enabled. */ - void buildSplitGraphDotfile(Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph, List<List<Vertex>> components) { + void buildSplitGraphWithComponentsDotfile( + Map<PCollectionImpl<?>, Set<Target>> outputs, Graph graph, List<List<Vertex>> components + ) { if (isDebugDotfilesEnabled(conf)) { try { - splitGraphPlanDotfile = new DotfileWriterGraph(graph, outputs, components) - .buildDiagram("Graph With Components (" + jarClass.getSimpleName() + ")"); + splitGraphWithComponentsPlanDotfile = new DotfileWriterGraph(graph, outputs, components) + .buildDiagram("Split Graph With Components (" + jarClass.getSimpleName() + ")"); } catch (Exception ex) { LOG.error("Problem creating debug dotfile:", ex); } @@ -145,6 +162,7 @@ public class DotfileUtil { exec.addNamedDotFile("rt_plan", rtNodesDotfile); exec.addNamedDotFile("base_graph_plan", basePlanGraphDotfile); exec.addNamedDotFile("split_graph_plan", splitGraphPlanDotfile); + exec.addNamedDotFile("split_graph_with_components_plan", splitGraphWithComponentsPlanDotfile); exec.addNamedDotFile("lineage_plan", pcollectionLineageDotfile); } } catch (Exception ex) { http://git-wip-us.apache.org/repos/asf/crunch/blob/e520d9f6/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java ---------------------------------------------------------------------- diff --git a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java index eace851..5113a2b 100644 --- a/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java +++ b/crunch-core/src/main/java/org/apache/crunch/impl/mr/plan/MSCRPlanner.java @@ -137,8 +137,9 @@ public class MSCRPlanner { List<List<Vertex>> components = graph.connectedComponents(); // Generate the debug graph dotfiles (if configuration is enabled) - dotfileUtil.buildBaseGraphDotfile(outputs, graph); - dotfileUtil.buildSplitGraphDotfile(outputs, graph, components); + dotfileUtil.buildBaseGraphDotfile(outputs, baseGraph); + dotfileUtil.buildSplitGraphDotfile(outputs, graph); + dotfileUtil.buildSplitGraphWithComponentsDotfile(outputs, graph, components); // For each component, we will create one or more job prototypes, // depending on its profile.
