Repository: giraph Updated Branches: refs/heads/trunk 6dd7d1425 -> b9d20edec
[GIRAPH-1013] Cleanup use of conf for local testing Summary: Right now we are creating two immutable conf options, and using them both, which is unnecessary and confusing. Change to do it only once, and not need to pass it around (TestGraph has it) Test Plan: mvn clean install Reviewers: laxman.dhulipala, maja.kabiljo, dionysis.logothetis, sergey.edunov Differential Revision: https://reviews.facebook.net/D39987 Project: http://git-wip-us.apache.org/repos/asf/giraph/repo Commit: http://git-wip-us.apache.org/repos/asf/giraph/commit/b9d20ede Tree: http://git-wip-us.apache.org/repos/asf/giraph/tree/b9d20ede Diff: http://git-wip-us.apache.org/repos/asf/giraph/diff/b9d20ede Branch: refs/heads/trunk Commit: b9d20edece212b3898fd61f0e6799c2ebbf99986 Parents: 6dd7d14 Author: Igor Kabiljo <[email protected]> Authored: Thu Jun 11 16:17:53 2015 -0700 Committer: Igor Kabiljo <[email protected]> Committed: Tue Jun 16 11:09:03 2015 -0700 ---------------------------------------------------------------------- .../giraph/block_app/framework/BlockUtils.java | 21 ++++------ .../framework/api/local/LocalBlockRunner.java | 42 +++++++++----------- .../block_app/framework/BlockExecutionTest.java | 4 +- .../java/org/apache/giraph/utils/TestGraph.java | 4 ++ 4 files changed, 33 insertions(+), 38 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java ---------------------------------------------------------------------- diff --git a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java index 3175a55..e00909c 100644 --- a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java +++ b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/BlockUtils.java @@ -138,8 +138,7 @@ public class BlockUtils { // They will not be used here Block executionBlock = blockFactory.createBlock(immConf); checkBlockTypes( - executionBlock, blockFactory.createExecutionStage(immConf), - conf, immConf); + executionBlock, blockFactory.createExecutionStage(immConf), immConf); // check for non 'static final' fields in BlockFactories Class<?> bfClass = blockFactory.getClass(); @@ -163,19 +162,15 @@ public class BlockUtils { public static void checkBlockTypes( Block executionBlock, Object executionStage, - GiraphConfiguration conf, - final ImmutableClassesGiraphConfiguration immConf) { + final ImmutableClassesGiraphConfiguration conf) { LOG.info("Executing application - " + executionBlock); - final Class<?> vertexIdClass = GiraphConstants.VERTEX_ID_CLASS.get(conf); - final Class<?> vertexValueClass = - GiraphConstants.VERTEX_VALUE_CLASS.get(conf); - final Class<?> edgeValueClass = - GiraphConstants.EDGE_VALUE_CLASS.get(conf); + final Class<?> vertexIdClass = conf.getVertexIdClass(); + final Class<?> vertexValueClass = conf.getVertexValueClass(); + final Class<?> edgeValueClass = conf.getEdgeValueClass(); final Class<?> workerContextValueClass = - BlockUtils.BLOCK_WORKER_CONTEXT_VALUE_CLASS.get(conf); - final Class<?> executionStageClass = - executionStage.getClass(); + BLOCK_WORKER_CONTEXT_VALUE_CLASS.get(conf); + final Class<?> executionStageClass = executionStage.getClass(); // Check for type inconsistencies executionBlock.forAllPossiblePieces(new Consumer<AbstractPiece>() { @@ -193,7 +188,7 @@ public class BlockUtils { ReflectionUtils.verifyTypes( edgeValueClass, classList[2], "edgeValue", piece.getClass()); - MessageClasses classes = piece.getMessageClasses(immConf); + MessageClasses classes = piece.getMessageClasses(conf); Class<?> messageType = classes.getMessageClass(); if (messageType == null) { messageType = NoMessage.class; http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java ---------------------------------------------------------------------- diff --git a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java index ea6817f..ad6ca15 100644 --- a/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java +++ b/giraph-block-app/src/main/java/org/apache/giraph/block_app/framework/api/local/LocalBlockRunner.java @@ -37,7 +37,6 @@ import org.apache.giraph.block_app.framework.internal.BlockWorkerLogic; import org.apache.giraph.block_app.framework.internal.BlockWorkerPieces; import org.apache.giraph.block_app.framework.output.BlockOutputHandle; import org.apache.giraph.conf.BooleanConfOption; -import org.apache.giraph.conf.GiraphConfiguration; import org.apache.giraph.conf.ImmutableClassesGiraphConfiguration; import org.apache.giraph.conf.IntConfOption; import org.apache.giraph.graph.Vertex; @@ -86,12 +85,11 @@ public class LocalBlockRunner { public static <I extends WritableComparable, V extends Writable, E extends Writable> TestGraph<I, V, E> runApp( - TestGraph<I, V, E> graph, GiraphConfiguration conf, - boolean useFullDigraphTests) throws Exception { + TestGraph<I, V, E> graph, boolean useFullDigraphTests) throws Exception { if (useFullDigraphTests) { - return InternalVertexRunner.runWithInMemoryOutput(conf, graph); + return InternalVertexRunner.runWithInMemoryOutput(graph.getConf(), graph); } else { - runApp(graph, conf); + runApp(graph); return graph; } } @@ -102,9 +100,9 @@ public class LocalBlockRunner { */ public static <I extends WritableComparable, V extends Writable, E extends Writable> - void runApp(TestGraph<I, V, E> graph, GiraphConfiguration conf) { + void runApp(TestGraph<I, V, E> graph) { VertexSaver<I, V, E> noOpVertexSaver = noOpVertexSaver(); - runAppWithVertexOutput(graph, noOpVertexSaver, conf); + runAppWithVertexOutput(graph, noOpVertexSaver); } /** @@ -114,11 +112,10 @@ public class LocalBlockRunner { public static <I extends WritableComparable, V extends Writable, E extends Writable> void runBlock( - TestGraph<I, V, E> graph, Block block, Object executionStage, - GiraphConfiguration conf) { + TestGraph<I, V, E> graph, Block block, Object executionStage) { VertexSaver<I, V, E> noOpVertexSaver = noOpVertexSaver(); runBlockWithVertexOutput( - block, executionStage, graph, noOpVertexSaver, conf); + block, executionStage, graph, noOpVertexSaver); } @@ -129,12 +126,12 @@ public class LocalBlockRunner { public static <I extends WritableComparable, V extends Writable, E extends Writable> void runAppWithVertexOutput( - TestGraph<I, V, E> graph, final VertexSaver<I, V, E> vertexSaver, - GiraphConfiguration conf) { - BlockFactory<?> factory = BlockUtils.createBlockFactory(conf); + TestGraph<I, V, E> graph, final VertexSaver<I, V, E> vertexSaver) { + BlockFactory<?> factory = BlockUtils.createBlockFactory(graph.getConf()); runBlockWithVertexOutput( - factory.createBlock(conf), factory.createExecutionStage(conf), - graph, vertexSaver, conf); + factory.createBlock(graph.getConf()), + factory.createExecutionStage(graph.getConf()), + graph, vertexSaver); } /** @@ -145,20 +142,19 @@ public class LocalBlockRunner { <I extends WritableComparable, V extends Writable, E extends Writable> void runBlockWithVertexOutput( Block block, Object executionStage, TestGraph<I, V, E> graph, - final VertexSaver<I, V, E> vertexSaver, GiraphConfiguration conf + final VertexSaver<I, V, E> vertexSaver ) { + ImmutableClassesGiraphConfiguration<I, V, E> conf = graph.getConf(); int numWorkers = NUM_THREADS.get(conf); boolean runAllChecks = RUN_ALL_CHECKS.get(conf); boolean serializeMaster = SERIALIZE_MASTER.get(conf); final boolean doOutputDuringComputation = conf.doOutputDuringComputation(); - ImmutableClassesGiraphConfiguration<I, V, E> immConf = - new ImmutableClassesGiraphConfiguration(conf); final InternalApi internalApi = - new InternalApi(graph, immConf, runAllChecks); + new InternalApi(graph, conf, runAllChecks); final InternalWorkerApi internalWorkerApi = internalApi.getWorkerApi(); - BlockUtils.checkBlockTypes(block, executionStage, conf, immConf); + BlockUtils.checkBlockTypes(block, executionStage, conf); BlockMasterLogic<Object> blockMasterLogic = new BlockMasterLogic<>(); blockMasterLogic.initialize(block, executionStage, internalApi); @@ -177,12 +173,12 @@ public class LocalBlockRunner { if (runAllChecks) { for (Vertex<I, V, E> vertex : graph) { - V value = immConf.createVertexValue(); + V value = conf.createVertexValue(); WritableUtils.copyInto(vertex.getValue(), value); vertex.setValue(value); vertex.setEdges((Iterable) WritableUtils.createCopy( - (Writable) vertex.getEdges(), immConf.getOutEdgesClass(), immConf)); + (Writable) vertex.getEdges(), conf.getOutEdgesClass(), conf)); } } @@ -194,7 +190,7 @@ public class LocalBlockRunner { blockMasterLogic = (BlockMasterLogic) WritableUtils.createCopy( new KryoWritableWrapper<>(blockMasterLogic), KryoWritableWrapper.class, - immConf).get(); + conf).get(); blockMasterLogic.initializeAfterRead(internalApi); } http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java ---------------------------------------------------------------------- diff --git a/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java b/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java index b77c797..170f307 100644 --- a/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java +++ b/giraph-block-app/src/test/java/org/apache/giraph/block_app/framework/BlockExecutionTest.java @@ -109,7 +109,7 @@ public class BlockExecutionTest { protected Class<BooleanWritable> getMessageClass() { return BooleanWritable.class; } - }, new Object(), conf); + }, new Object()); Assert.assertEquals(1, graph.getVertex(new LongWritable(1)).getValue().get()); Assert.assertEquals(2, graph.getVertex(new LongWritable(2)).getValue().get()); @@ -149,7 +149,7 @@ public class BlockExecutionTest { public void masterCompute(BlockMasterApi masterApi, Object executionStage) { value.set(numVertices.getReducedValue(masterApi).get()); } - }, new Object(), conf); + }, new Object()); Assert.assertEquals(4, value.get()); } http://git-wip-us.apache.org/repos/asf/giraph/blob/b9d20ede/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java ---------------------------------------------------------------------- diff --git a/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java b/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java index 7a37167..6bdd4b9 100644 --- a/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java +++ b/giraph-core/src/main/java/org/apache/giraph/utils/TestGraph.java @@ -64,6 +64,10 @@ public class TestGraph<I extends WritableComparable, return vertices; } + public ImmutableClassesGiraphConfiguration<I, V, E> getConf() { + return conf; + } + /** * Clear all data *
