TINKERPOP-1996 Have the basics of OLAP read()/write() steps working This is still fairly skeletal at this point. Just trying to make sure things work properly before building read()/write() out fully.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/0785090f Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/0785090f Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/0785090f Branch: refs/heads/TINKERPOP-1774 Commit: 0785090f67ff88a834f77c7181a69594049c65ea Parents: ec1d05f Author: Stephen Mallette <sp...@genoprime.com> Authored: Tue Jul 10 12:21:52 2018 -0400 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Thu Jul 19 13:39:29 2018 -0400 ---------------------------------------------------------------------- .../decoration/VertexProgramStrategy.java | 34 ++++---- .../process/traversal/TraversalStrategies.java | 2 + .../gremlin/process/traversal/step/Reading.java | 27 +++++++ .../gremlin/process/traversal/step/Writing.java | 26 +++++++ .../process/traversal/step/map/ReadStep.java | 10 ++- .../process/traversal/step/map/WriteStep.java | 8 +- .../strategy/verification/IoUsageStrategy.java | 8 ++ .../decoration/VertexProgramStrategyTest.java | 3 + .../gremlin/process/ProcessStandardSuite.java | 2 + .../process/traversal/step/map/ReadTest.java | 64 +++++++++++++++ .../traversal/step/map/HadoopReadStep.java | 82 ++++++++++++++++++++ .../traversal/step/map/HadoopWriteStep.java | 82 ++++++++++++++++++++ .../traversal/strategy/HadoopIoStrategy.java | 68 ++++++++++++++++ .../gremlin/hadoop/structure/HadoopGraph.java | 8 ++ 14 files changed, 407 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java index 89e40cb..c83039a 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/strategy/decoration/VertexProgramStrategy.java @@ -34,6 +34,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.Reading; +import org.apache.tinkerpop.gremlin.process.traversal.step.Writing; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; @@ -100,23 +102,26 @@ public final class VertexProgramStrategy extends AbstractTraversalStrategy<Trave // wrap all non-VertexComputing steps into a TraversalVertexProgramStep currentStep = traversal.getStartStep(); - while (!(currentStep instanceof EmptyStep)) { - Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>(); - Step<?, ?> firstLegalOLAPStep = getFirstLegalOLAPStep(currentStep); - Step<?, ?> lastLegalOLAPStep = getLastLegalOLAPStep(currentStep); - if (!(firstLegalOLAPStep instanceof EmptyStep)) { - int index = TraversalHelper.stepIndex(firstLegalOLAPStep, traversal); - TraversalHelper.removeToTraversal(firstLegalOLAPStep, lastLegalOLAPStep.getNextStep(), (Traversal.Admin) computerTraversal); - final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, computerTraversal); - traversal.addStep(index, traversalVertexProgramStep); - } - currentStep = traversal.getStartStep(); + if (!(currentStep instanceof Reading) && !(currentStep instanceof Writing)) { while (!(currentStep instanceof EmptyStep)) { - if (!(currentStep instanceof VertexComputing)) - break; - currentStep = currentStep.getNextStep(); + final Traversal.Admin<?, ?> computerTraversal = new DefaultTraversal<>(); + final Step<?, ?> firstLegalOLAPStep = getFirstLegalOLAPStep(currentStep); + final Step<?, ?> lastLegalOLAPStep = getLastLegalOLAPStep(currentStep); + if (!(firstLegalOLAPStep instanceof EmptyStep)) { + final int index = TraversalHelper.stepIndex(firstLegalOLAPStep, traversal); + TraversalHelper.removeToTraversal(firstLegalOLAPStep, lastLegalOLAPStep.getNextStep(), (Traversal.Admin) computerTraversal); + final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, computerTraversal); + traversal.addStep(index, traversalVertexProgramStep); + } + currentStep = traversal.getStartStep(); + while (!(currentStep instanceof EmptyStep)) { + if (!(currentStep instanceof VertexComputing)) + break; + currentStep = currentStep.getNextStep(); + } } } + // if the last vertex computing step is a TraversalVertexProgramStep convert to OLTP with ComputerResultStep TraversalHelper.getLastStepOfAssignableClass(VertexComputing.class, traversal).ifPresent(step -> { if (step instanceof TraversalVertexProgramStep) { @@ -126,6 +131,7 @@ public final class VertexProgramStrategy extends AbstractTraversalStrategy<Trave TraversalHelper.insertAfterStep(computerResultStep, (Step) step, traversal); } }); + // if there is a dangling vertex computing step, add an identity traversal (solve this in the future with a specialized MapReduce) if (traversal.getEndStep() instanceof VertexComputing && !(traversal.getEndStep() instanceof TraversalVertexProgramStep)) { final TraversalVertexProgramStep traversalVertexProgramStep = new TraversalVertexProgramStep(traversal, __.identity().asAdmin()); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java index ef3e841..66b0236 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/TraversalStrategies.java @@ -36,6 +36,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.Path import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.CountStrategy; import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.RepeatUnrollStrategy; import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.IoUsageStrategy; import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.StandardVerificationStrategy; import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies; import org.apache.tinkerpop.gremlin.structure.Graph; @@ -222,6 +223,7 @@ public interface TraversalStrategies extends Serializable, Cloneable { PathRetractionStrategy.instance(), LazyBarrierStrategy.instance(), ProfileStrategy.instance(), + IoUsageStrategy.instance(), StandardVerificationStrategy.instance()); GRAPH_CACHE.put(Graph.class, graphStrategies); GRAPH_CACHE.put(EmptyGraph.class, new DefaultTraversalStrategies()); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java new file mode 100644 index 0000000..b79378b --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Reading.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step; + +/** + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public interface Reading extends Configuring { + + public String getLocalFile(); +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java new file mode 100644 index 0000000..7b455f2 --- /dev/null +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Writing.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step; + +/** + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public interface Writing extends Configuring { + public String getLocalFile(); +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java index afc3e53..2612d18 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadStep.java @@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.Traverser; import org.apache.tinkerpop.gremlin.process.traversal.TraverserGenerator; import org.apache.tinkerpop.gremlin.process.traversal.step.Configuring; +import org.apache.tinkerpop.gremlin.process.traversal.step.Reading; import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; @@ -43,7 +44,7 @@ import java.util.Map; * * @author Stephen Mallette (http://stephen.genoprime.com) */ -public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Configuring { +public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Reading { private Parameters parameters = new Parameters(); private boolean first = true; @@ -54,12 +55,16 @@ public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object if (null == localFile || localFile.isEmpty()) throw new IllegalArgumentException("localFile cannot be null or empty"); - if (!new File(localFile).exists()) throw new IllegalStateException(localFile + " does not exist"); this.localFile = localFile; } @Override + public String getLocalFile() { + return localFile; + } + + @Override public Parameters getParameters() { return this.parameters; } @@ -76,6 +81,7 @@ public class ReadStep extends AbstractStep<Map<String,Object>, Map<String,Object this.first = false; final TraverserGenerator generator = this.getTraversal().getTraverserGenerator(); final File file = new File(localFile); + if (!file.exists()) throw new IllegalStateException(localFile + " does not exist"); try (final InputStream stream = new FileInputStream(file)) { final Graph graph = (Graph) this.traversal.getGraph().get(); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java index e9346cf..3a035e7 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/WriteStep.java @@ -22,6 +22,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.Traverser; import org.apache.tinkerpop.gremlin.process.traversal.TraverserGenerator; import org.apache.tinkerpop.gremlin.process.traversal.step.Configuring; +import org.apache.tinkerpop.gremlin.process.traversal.step.Writing; import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementException; @@ -43,7 +44,7 @@ import java.util.Map; * * @author Stephen Mallette (http://stephen.genoprime.com) */ -public class WriteStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Configuring { +public class WriteStep extends AbstractStep<Map<String,Object>, Map<String,Object>> implements Writing { private Parameters parameters = new Parameters(); private boolean first = true; @@ -59,6 +60,11 @@ public class WriteStep extends AbstractStep<Map<String,Object>, Map<String,Objec } @Override + public String getLocalFile() { + return localFile; + } + + @Override public Parameters getParameters() { return this.parameters; } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java index 95761ff..e9c7ec8 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/IoUsageStrategy.java @@ -26,6 +26,9 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.WriteStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; +import java.util.Collections; +import java.util.Set; + /** * {@code IoUsageStrategy} prevents the {@link GraphTraversalSource#read(String)} and * {@link GraphTraversalSource#write(String)} steps from being used outside of their intended scope, which is as the @@ -57,4 +60,9 @@ public final class IoUsageStrategy extends AbstractTraversalStrategy<TraversalSt public static IoUsageStrategy instance() { return INSTANCE; } + + @Override + public Set<Class<? extends VerificationStrategy>> applyPrior() { + return Collections.singleton(ComputerVerificationStrategy.class); + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java index c091bb3..71cb22d 100644 --- a/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java +++ b/gremlin-core/src/test/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/decoration/VertexProgramStrategyTest.java @@ -32,6 +32,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy; import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversalStrategies; import org.apache.tinkerpop.gremlin.process.traversal.util.EmptyTraversal; +import org.apache.tinkerpop.gremlin.structure.util.empty.EmptyGraph; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -68,6 +69,8 @@ public class VertexProgramStrategyTest { final ComputerResultStep computerResultStep = new ComputerResultStep(EmptyTraversal.instance()); return Arrays.asList(new Traversal[][]{ + { EmptyGraph.instance().traversal().read("blah.json"), EmptyGraph.instance().traversal().read("blah.json")}, + { EmptyGraph.instance().traversal().write("blah.json"), EmptyGraph.instance().traversal().write("blah.json")}, {__.V().out().count(), start().addStep(traversal(__.V().out().count())).addStep(computerResultStep)}, {__.V().pageRank().out().count(), start().pageRank().asAdmin().addStep(traversal(__.V().out().count())).addStep(computerResultStep)}, {__.V().out().pageRank(), start().addStep(traversal(__.V().out())).pageRank().asAdmin().addStep(traversal(__.identity())).addStep(computerResultStep)}, http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java index f7c19ac..43da8b7 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessStandardSuite.java @@ -63,6 +63,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProjectTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.SelectTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.SumTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.UnfoldTest; @@ -153,6 +154,7 @@ public class ProcessStandardSuite extends AbstractGremlinSuite { ProfileTest.Traversals.class, ProjectTest.Traversals.class, PropertiesTest.Traversals.class, + ReadTest.Traversals.class, SelectTest.Traversals.class, VertexTest.Traversals.class, UnfoldTest.Traversals.class, http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java new file mode 100644 index 0000000..2cfc3f7 --- /dev/null +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/ReadTest.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.process.traversal.step.map; + +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest; +import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner; +import org.apache.tinkerpop.gremlin.process.IgnoreEngine; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.io.IoTest; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.assertTrue; + +/** + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +@RunWith(GremlinProcessRunner.class) +public abstract class ReadTest extends AbstractGremlinProcessTest { + + public abstract Traversal<Map<String,Object>, Map<String,Object>> get_g_read() throws IOException; + + @Test + public void g_read() throws IOException { + final Traversal<Map<String,Object>, Map<String,Object>> traversal = get_g_read(); + printTraversalForm(traversal); + assertTrue(traversal.hasNext()); + + IoTest.assertModernGraph(graph, false, true); + } + + public static class Traversals extends ReadTest { + @Override + public Traversal<Map<String,Object>, Map<String,Object>> get_g_read() throws IOException { + final String fileToRead = TestHelper.generateTempFileFromResource(ReadTest.class, GryoResourceAccess.class, "tinkerpop-modern-v3d0.kryo", "").getAbsolutePath().replace('\\', '/'); + return g.read(fileToRead); + } + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java new file mode 100644 index 0000000..c896d76 --- /dev/null +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopReadStep.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map; + +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.process.computer.GraphFilter; +import org.apache.tinkerpop.gremlin.process.computer.Memory; +import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; +import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.Reading; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +/** + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class HadoopReadStep extends VertexProgramStep implements Reading { + + private Parameters parameters = new Parameters(); + + public HadoopReadStep(final Traversal.Admin traversal, final String localFile) { + super(traversal); + + final Graph graph = (Graph) traversal.getGraph().get(); + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat"); + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, localFile); + } + + @Override + public String getLocalFile() { + return (String) ((Graph) traversal.getGraph().get()).configuration().getProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION); + } + + @Override + public void configure(final Object... keyValues) { + // TODO: probably should write to the Configuration selectively - no need for actual Parameters????????? + this.parameters.set(null, keyValues); + } + + @Override + public Parameters getParameters() { + return parameters; + } + + @Override + public String toString() { + return StringFactory.stepString(this, new GraphFilter(this.computer)); + } + + @Override + public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) { + return CloneVertexProgram.build().create(graph); + } + + @Override + public HadoopReadStep clone() { + return (HadoopReadStep) super.clone(); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java new file mode 100644 index 0000000..0711a2e --- /dev/null +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/step/map/HadoopWriteStep.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map; + +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.process.computer.GraphFilter; +import org.apache.tinkerpop.gremlin.process.computer.Memory; +import org.apache.tinkerpop.gremlin.process.computer.clone.CloneVertexProgram; +import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.VertexProgramStep; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.Reading; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Parameters; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.util.StringFactory; + +/** + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class HadoopWriteStep extends VertexProgramStep implements Reading { + + private Parameters parameters = new Parameters(); + + public HadoopWriteStep(final Traversal.Admin traversal, final String localFile) { + super(traversal); + + final Graph graph = (Graph) traversal.getGraph().get(); + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, "org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat"); + graph.configuration().setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, localFile); + } + + @Override + public String getLocalFile() { + return (String) ((Graph) traversal.getGraph().get()).configuration().getProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION); + } + + @Override + public void configure(final Object... keyValues) { + // TODO: probably should write to the Configuration selectively - no need for actual Parameters????????? + this.parameters.set(null, keyValues); + } + + @Override + public Parameters getParameters() { + return parameters; + } + + @Override + public String toString() { + return StringFactory.stepString(this, new GraphFilter(this.computer)); + } + + @Override + public CloneVertexProgram generateProgram(final Graph graph, final Memory memory) { + return CloneVertexProgram.build().create(graph); + } + + @Override + public HadoopWriteStep clone() { + return (HadoopWriteStep) super.clone(); + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java new file mode 100644 index 0000000..47986f9 --- /dev/null +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/traversal/strategy/HadoopIoStrategy.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy; + +import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map.HadoopReadStep; +import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.step.map.HadoopWriteStep; +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.step.Reading; +import org.apache.tinkerpop.gremlin.process.traversal.step.Writing; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; + +/** + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public final class HadoopIoStrategy extends AbstractTraversalStrategy<TraversalStrategy.ProviderOptimizationStrategy> + implements TraversalStrategy.ProviderOptimizationStrategy { + + private static final HadoopIoStrategy INSTANCE = new HadoopIoStrategy(); + + private HadoopIoStrategy() { + } + + @Override + public void apply(final Traversal.Admin<?, ?> traversal) { + // replace Reading and Writing steps with hadoop specific ones + if (traversal.getStartStep() instanceof Reading) { + final Reading reading = (Reading) traversal.getStartStep(); + final HadoopReadStep hadoopReadStep = new HadoopReadStep(traversal, reading.getLocalFile()); + reading.getParameters().getRaw().entrySet().forEach(kv -> + hadoopReadStep.configure(null, kv.getKey(), kv.getValue()) + ); + + TraversalHelper.replaceStep((Step) reading, hadoopReadStep, traversal); + } else if (traversal.getStartStep() instanceof Writing) { + final Writing writing = (Writing) traversal.getStartStep(); + final HadoopWriteStep hadoopWriteStep = new HadoopWriteStep(traversal, writing.getLocalFile()); + writing.getParameters().getRaw().entrySet().forEach(kv -> + hadoopWriteStep.configure(null, kv.getKey(), kv.getValue()) + ); + + TraversalHelper.replaceStep((Step) writing, hadoopWriteStep, traversal); + } + } + + public static HadoopIoStrategy instance() { + return INSTANCE; + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0785090f/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java index ea8334b..14c5360 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java @@ -24,9 +24,11 @@ import org.apache.commons.configuration.ConfigurationException; import org.apache.commons.configuration.PropertiesConfiguration; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.process.computer.AbstractHadoopGraphComputer; +import org.apache.tinkerpop.gremlin.hadoop.process.computer.traversal.strategy.HadoopIoStrategy; import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopEdgeIterator; import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopVertexIterator; import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategies; import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Transaction; @@ -148,6 +150,12 @@ public final class HadoopGraph implements Graph { this.setProperty(Graph.GRAPH, HadoopGraph.class.getName()); }}; + static { + TraversalStrategies.GlobalCache.registerStrategies(HadoopGraph.class, + TraversalStrategies.GlobalCache.getStrategies(Graph.class).clone().addStrategies( + HadoopIoStrategy.instance())); + } + protected final HadoopConfiguration configuration; private HadoopGraph(final Configuration configuration) {