TINKERPOP-1996 Pass configurations from with() through to Hadoop This will allow users to override or add to the Hadoop/Spark/OLAP configuration as needed
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/4d979cf8 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/4d979cf8 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/4d979cf8 Branch: refs/heads/TINKERPOP-1774 Commit: 4d979cf8dc4482d574191a76a0c60e281f887563 Parents: 51dc821 Author: Stephen Mallette <sp...@genoprime.com> Authored: Fri Jul 20 07:35:48 2018 -0400 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Fri Jul 20 07:35:48 2018 -0400 ---------------------------------------------------------------------- .../traversal/step/sideEffect/HadoopIoStep.java | 29 ++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/4d979cf8/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java index ca369b6..2e96276 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/sideEffect/HadoopIoStep.java @@ -29,6 +29,7 @@ import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; import org.apache.tinkerpop.gremlin.process.traversal.IO; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; @@ -37,7 +38,9 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory; /** * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the - * {@link CloneVertexProgram} for its implementation. + * {@link CloneVertexProgram} for its implementation. Standard Hadoop OLAP configurations can be passed using the + * {@link GraphTraversal#with(String, Object)} step modulator as all options aside from those in {@link IO} will be + * transferred. * * @author Stephen Mallette (http://stephen.genoprime.com) */ @@ -69,7 +72,6 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting { @Override public void configure(final Object... keyValues) { - // TODO: probably should write to the Configuration selectively - no need for actual Parameters????????? this.parameters.set(null, keyValues); } @@ -121,6 +123,8 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting { graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName); graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file); + + addParametersToConfiguration(graph); } private void configureForWrite(final Graph graph) { @@ -137,6 +141,27 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting { graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName); graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file); + + addParametersToConfiguration(graph); + } + + /** + * Overwrites all configurations from values passed using {@link GraphTraversal#with(String, Object)}. + */ + private void addParametersToConfiguration(final Graph graph) { + parameters.getRaw(IO.writer, IO.writer, IO.registry).entrySet().forEach(kv -> { + if (kv.getValue().size() == 1) + graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(0)); + else { + // reset the default configuration with the first option then add to that for List options + for (int ix = 0; ix < kv.getValue().size(); ix++) { + if (ix == 0) + graph.configuration().setProperty(kv.getKey().toString(), kv.getValue().get(ix)); + else + graph.configuration().addProperty(kv.getKey().toString(), kv.getValue().get(ix)); + } + } + }); } private String detectReader() {