TINKERPOP-1996 Got read/write() tests running for OLAP Introduced new Graph.Features to provider better separation between graph mutation features and graph loading features - they are two different things as demonstrated by io(). Fixed HadoopIoStep/Strategy so that they properly handle the different input/output format types expected.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/f148e933 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/f148e933 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/f148e933 Branch: refs/heads/TINKERPOP-1774 Commit: f148e9331a945e0f4f707ea937b610e5902701c7 Parents: bd275a7 Author: Stephen Mallette <sp...@genoprime.com> Authored: Fri Jul 13 15:17:17 2018 -0400 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Thu Jul 19 13:40:10 2018 -0400 ---------------------------------------------------------------------- .../tinkerpop/gremlin/structure/Graph.java | 27 +++++++++ .../gremlin/AbstractGraphProvider.java | 18 ++++-- .../gremlin/process/ProcessComputerSuite.java | 4 ++ .../process/traversal/step/map/ReadTest.java | 21 ++++--- .../process/traversal/step/map/WriteTest.java | 14 ++++- .../traversal/step/map/HadoopIoStep.java | 60 +++++++++++++++++++- .../traversal/strategy/HadoopIoStrategy.java | 2 +- .../gremlin/hadoop/structure/HadoopGraph.java | 16 ++++++ 8 files changed, 142 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java index f1fc54a..f62b897 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/Graph.java @@ -23,6 +23,8 @@ import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; import org.apache.tinkerpop.gremlin.structure.io.Io; import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; import org.apache.tinkerpop.gremlin.structure.util.FeatureDescriptor; @@ -439,6 +441,8 @@ public interface Graph extends AutoCloseable, Host { public static final String FEATURE_PERSISTENCE = "Persistence"; public static final String FEATURE_THREADED_TRANSACTIONS = "ThreadedTransactions"; public static final String FEATURE_CONCURRENT_ACCESS = "ConcurrentAccess"; + public static final String FEATURE_IO_READ = "IoRead"; + public static final String FEATURE_IO_WRITE = "IoWrite"; /** * Determines if the {@code Graph} implementation supports {@link GraphComputer} based processing. @@ -489,6 +493,29 @@ public interface Graph extends AutoCloseable, Host { } /** + * Determines if the {@code Graph} implementations supports read operations as executed with the + * {@link GraphTraversalSource#io(String)} step. Graph implementations will generally support this by + * default as any graph that can support direct mutation through the Structure API will by default + * accept data from the standard TinkerPop {@link GraphReader} implementations. However, some graphs like + * {@code HadoopGraph} don't accept direct mutations but can still do reads from that {@code io()} step. + */ + @FeatureDescriptor(name = FEATURE_IO_READ) + public default boolean supportsIoRead() { + return true; + } + + /** + * Determines if the {@code Graph} implementations supports write operations as executed with the + * {@link GraphTraversalSource#io(String)} step. Graph implementations will generally support this by + * default given the standard TinkerPop {@link GraphWriter} implementations. However, some graphs like + * {@code HadoopGraph} will use a different approach to handle writes. + */ + @FeatureDescriptor(name = FEATURE_IO_WRITE) + public default boolean supportsIoWrite() { + return true; + } + + /** * Gets the features related to "graph sideEffects" operation. */ public default VariableFeatures variables() { http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java index 95c6b57..75d033b 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractGraphProvider.java @@ -20,6 +20,7 @@ package org.apache.tinkerpop.gremlin; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; import org.apache.commons.configuration.BaseConfiguration; @@ -131,12 +132,19 @@ public abstract class AbstractGraphProvider implements GraphProvider { return methodName.replaceAll("[0-9, -]+$", String.valueOf(random)); } - protected void readIntoGraph(final Graph g, final String path) throws IOException { - final GraphReader reader = GryoReader.build() - .mapper(g.io(GryoIo.build()).mapper().create()) - .create(); + /** + * Used by the default implementation of {@link AbstractGraphProvider#loadGraphData(Graph, LoadGraphWith, Class, String)} + * to read the graph from a Kryo file using the default {@link GryoReader} implementation. If the default + * implementation does not work (perhaps a graph implementation needs to register some special {@link IoRegistry} + * then this method or its caller should be overridden to suit the implementation. + * + * @param graph the graph to load to + * @param path the path to the file to load into the graph + */ + protected void readIntoGraph(final Graph graph, final String path) throws IOException { + final GraphReader reader = GryoReader.build().create(); try (final InputStream stream = AbstractGremlinTest.class.getResourceAsStream(path)) { - reader.readGraph(stream, g); + reader.readGraph(stream, graph); } } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java index b224c8b..eab562d 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java @@ -70,11 +70,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ValueMapTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.VertexTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ExplainTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupCountTest; @@ -163,10 +165,12 @@ public class ProcessComputerSuite extends AbstractGremlinSuite { ProjectTest.Traversals.class, ProgramTest.Traversals.class, PropertiesTest.Traversals.class, + ReadTest.Traversals.class, SelectTest.Traversals.class, UnfoldTest.Traversals.class, ValueMapTest.Traversals.class, VertexTest.Traversals.class, + WriteTest.Traversals.class, // sideEffect AddEdgeTest.Traversals.class, http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java index 9e53169..ab59194 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java @@ -18,12 +18,14 @@ */ package org.apache.tinkerpop.gremlin.process.traversal.step.map; +import org.apache.tinkerpop.gremlin.FeatureRequirement; import org.apache.tinkerpop.gremlin.FeatureRequirementSet; import org.apache.tinkerpop.gremlin.TestHelper; import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest; import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner; import org.apache.tinkerpop.gremlin.process.traversal.IO; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.IoTest; import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLResourceAccess; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess; @@ -33,6 +35,7 @@ import org.junit.runner.RunWith; import java.io.IOException; +import static org.apache.tinkerpop.gremlin.structure.Graph.Features.GraphFeatures.FEATURE_IO_READ; import static org.junit.Assert.assertFalse; /** @@ -54,8 +57,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest { public abstract Traversal<Object,Object> get_g_io_read_withXreader_graphmlX(final String fileToRead) throws IOException; @Test - @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE) - public void g_readXkryoX() throws IOException { + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ) + public void g_io_readXkryoX() throws IOException { final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/'); final Traversal<Object,Object> traversal = get_g_io_readXkryoX(fileToRead); printTraversalForm(traversal); @@ -65,7 +68,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest { } @Test - @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE) + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ) public void g_io_read_withXreader_gryoX() throws IOException { final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/'); final Traversal<Object,Object> traversal = get_g_io_read_withXreader_gryoX(fileToRead); @@ -76,8 +79,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest { } @Test - @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE) - public void g_readXjsonX() throws IOException { + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ) + public void g_io_readXjsonX() throws IOException { final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/'); final Traversal<Object,Object> traversal = get_g_io_readXjsonX(fileToRead); printTraversalForm(traversal); @@ -87,7 +90,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest { } @Test - @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE) + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ) public void g_io_read_withXreader_graphsonX() throws IOException { final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphSONResourceAccess.class, "tinkerpop-modern-v3d0.json", "").getAbsolutePath().replace('\\', '/'); final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphsonX(fileToRead); @@ -98,8 +101,8 @@ public abstract class ReadTest extends AbstractGremlinProcessTest { } @Test - @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE) - public void g_readXxmlX() throws IOException { + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ) + public void g_io_readXxmlX() throws IOException { final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/'); final Traversal<Object,Object> traversal = get_g_io_readXxmlX(fileToRead); printTraversalForm(traversal); @@ -109,7 +112,7 @@ public abstract class ReadTest extends AbstractGremlinProcessTest { } @Test - @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE) + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_READ) public void g_io_read_withXreader_graphmlX() throws IOException { final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GraphMLResourceAccess.class, "tinkerpop-modern.xml", "").getAbsolutePath().replace('\\', '/'); final Traversal<Object,Object> traversal = get_g_io_read_withXreader_graphmlX(fileToRead); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java index e739c0a..55e6f9c 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteTest.java @@ -18,12 +18,14 @@ */ package org.apache.tinkerpop.gremlin.process.traversal.step.map; +import org.apache.tinkerpop.gremlin.FeatureRequirement; import org.apache.tinkerpop.gremlin.LoadGraphWith; import org.apache.tinkerpop.gremlin.TestHelper; import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest; import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner; import org.apache.tinkerpop.gremlin.process.traversal.IO; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.structure.Graph; import org.junit.Test; import org.junit.runner.RunWith; @@ -54,7 +56,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void g_writeXkryoX() throws IOException { + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE) + public void g_io_writeXkryoX() throws IOException { final String fileToWrite = TestHelper.generateTempFile(WriteTest.class, "tinkerpop-modern-v3d0", ".kryo").getAbsolutePath().replace('\\', '/'); final File f = new File(fileToWrite); @@ -69,6 +72,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE) public void g_io_write_withXwrite_gryoX() throws IOException { final String fileToWrite = TestHelper.generateTempFile(WriteTest.class, "tinkerpop-modern-v3d0", ".kryo").getAbsolutePath().replace('\\', '/'); @@ -84,7 +88,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void g_writeXjsonX() throws IOException { + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE) + public void g_io_writeXjsonX() throws IOException { final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern-v3d0", ".json").getAbsolutePath().replace('\\', '/'); final File f = new File(fileToWrite); @@ -99,6 +104,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE) public void g_io_write_withXwriter_graphsonX() throws IOException { final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern-v3d0", ".json").getAbsolutePath().replace('\\', '/'); @@ -114,7 +120,8 @@ public abstract class WriteTest extends AbstractGremlinProcessTest { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void g_writeXxmlX() throws IOException { + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE) + public void g_io_writeXxmlX() throws IOException { final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern", ".xml").getAbsolutePath().replace('\\', '/'); final File f = new File(fileToWrite); @@ -129,6 +136,7 @@ public abstract class WriteTest extends AbstractGremlinProcessTest { @Test @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + @FeatureRequirement(featureClass = Graph.Features.GraphFeatures.class, feature = Graph.Features.GraphFeatures.FEATURE_IO_WRITE) public void g_io_write_withXwriter_graphmlX() throws IOException { final String fileToWrite = TestHelper.generateTempFile(WriteTest.class,"tinkerpop-modern", ".xml").getAbsolutePath().replace('\\', '/'); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java index 62937da..97fdea4 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopIoStep.java @@ -18,18 +18,30 @@ */ package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map; +import org.apache.hadoop.mapred.InputFormat; import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; import org.apache.tinkerpop.gremlin.process.computer.GraphFilter; import org.apache.tinkerpop.gremlin.process.computer.Memory; import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; +import org.apache.tinkerpop.gremlin.process.traversal.IO; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.step.ReadWriting; import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.GraphReader; +import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; +import java.lang.reflect.Method; + /** * An OLAP oriented step for doing IO operations with {@link GraphTraversalSource#io(String)} which uses the * {@link CloneVertexProgram} for its implementation. @@ -103,12 +115,56 @@ public class HadoopIoStep extends VertexProgramStep implements ReadWriting { } private void configureForRead(final Graph graph) { - graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat"); + final String inputFormatClassNameOrKeyword = parameters.get(IO.reader, this::detectReader).get(0); + String inputFormatClassName; + if (inputFormatClassNameOrKeyword.equals(IO.graphson)) + inputFormatClassName = GraphSONInputFormat.class.getName(); + else if (inputFormatClassNameOrKeyword.equals(IO.gryo)) + inputFormatClassName = GryoInputFormat.class.getName(); + else if (inputFormatClassNameOrKeyword.equals(IO.graphml)) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + inputFormatClassName = inputFormatClassNameOrKeyword; + + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, inputFormatClassName); graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, file); } private void configureForWrite(final Graph graph) { - graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat"); + final String outputFormatClassNameOrKeyword = parameters.get(IO.writer, this::detectWriter).get(0); + String outputFormatClassName; + if (outputFormatClassNameOrKeyword.equals(IO.graphson)) + outputFormatClassName = GraphSONOutputFormat.class.getName(); + else if (outputFormatClassNameOrKeyword.equals(IO.gryo)) + outputFormatClassName = GryoOutputFormat.class.getName(); + else if (outputFormatClassNameOrKeyword.equals(IO.graphml)) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + outputFormatClassName = outputFormatClassNameOrKeyword; + + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, outputFormatClassName); graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, file); } + + private String detectReader() { + if (file.endsWith(".kryo")) + return GryoInputFormat.class.getName(); + else if (file.endsWith(".json")) + return GraphSONInputFormat.class.getName(); + else if (file.endsWith(".xml")) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); + } + + private String detectWriter() { + if (file.endsWith(".kryo")) + return GryoOutputFormat.class.getName(); + else if (file.endsWith(".json")) + return GraphSONOutputFormat.class.getName(); + else if (file.endsWith(".xml")) + throw new IllegalStateException("GraphML is not a supported file format for OLAP"); + else + throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java index 6d3899e..cbc9b07 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java @@ -55,7 +55,7 @@ public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalS final HadoopIoStep hadoopIoStep = new HadoopIoStep(traversal, readWriting.getFile()); hadoopIoStep.setMode(readWriting.getMode()); readWriting.getParameters().getRaw().entrySet().forEach(kv -> - hadoopIoStep.configure(kv.getKey(), kv.getValue()) + hadoopIoStep.configure(kv.getKey(), kv.getValue().get(0)) ); TraversalHelper.replaceStep((Step) readWriting, hadoopIoStep, traversal); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/f148e933/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java index 14c5360..9ec0cfd 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java @@ -142,6 +142,22 @@ import java.util.stream.Stream; method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", computers = {"ALL"}) +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals", + method = "g_io_readXxmlX", + reason = "Hadoop-Gremlin does not support reads/writes with GraphML.") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals", + method = "g_io_read_withXreader_graphmlX", + reason = "Hadoop-Gremlin does not support reads/writes with GraphML.") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals", + method = "g_io_writeXxmlX", + reason = "Hadoop-Gremlin does not support reads/writes with GraphML.") +@Graph.OptOut( + test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteTest$Traversals", + method = "g_io_write_withXwriter_graphmlX", + reason = "Hadoop-Gremlin does not support reads/writes with GraphML.") public final class HadoopGraph implements Graph { public static final Logger LOGGER = LoggerFactory.getLogger(HadoopGraph.class);