Created a new gremlin-tools/ packaged called gremlin-tinkergraph-test/ which includes an AbstractTinkerGraphProvider to allow providers to more easily leverage TinkerGraph in their testing. Both spark-gremlin/ and akka-gremlin/ use it to test SparkGraphComputer and AkkaGraphActors against TinkerGraph.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/cb9cc637 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/cb9cc637 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/cb9cc637 Branch: refs/heads/TINKERPOP-1564 Commit: cb9cc6377cc4d16265d5bf99e584bf25e71f6852 Parents: 1bd6e09 Author: Marko A. Rodriguez <okramma...@gmail.com> Authored: Fri Jan 20 08:11:47 2017 -0700 Committer: Marko A. Rodriguez <okramma...@gmail.com> Committed: Mon Jan 23 14:22:53 2017 -0700 ---------------------------------------------------------------------- akka-gremlin/pom.xml | 8 +- .../akka/process/actors/WorkerActor.java | 5 - .../process/actors/AkkaGroovyPlayTest.groovy | 49 ---- .../akka/process/actors/AkkaActorsProvider.java | 138 ++-------- .../tinkerpop/gremlin/process/actors/Actor.java | 20 +- .../traversal/TraversalWorkerProgram.java | 6 +- gremlin-tools/gremlin-tinkergraph-test/pom.xml | 59 +++++ .../AbstractTinkerGraphProvider.java | 257 +++++++++++++++++++ gremlin-tools/pom.xml | 1 + spark-gremlin/pom.xml | 2 +- ...PartitionerComputerProcessIntegrateTest.java | 33 +++ .../computer/SparkHadoopGraphProvider.java | 4 - .../computer/SparkTinkerGraphProvider.java | 117 +++++++++ ...PartitionerComputerProcessIntegrateTest.java | 34 --- .../TinkerGraphPartitionerProvider.java | 224 ---------------- 15 files changed, 502 insertions(+), 455 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/akka-gremlin/pom.xml ---------------------------------------------------------------------- diff --git a/akka-gremlin/pom.xml b/akka-gremlin/pom.xml index e3b12ef..a756f7e 100644 --- a/akka-gremlin/pom.xml +++ b/akka-gremlin/pom.xml @@ -69,19 +69,13 @@ <!-- TEST --> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>gremlin-groovy</artifactId> - <version>3.3.0-SNAPSHOT</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.tinkerpop</groupId> <artifactId>gremlin-test</artifactId> <version>3.3.0-SNAPSHOT</version> <scope>test</scope> </dependency> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>tinkergraph-gremlin</artifactId> + <artifactId>gremlin-tinkergraph-test</artifactId> <version>3.3.0-SNAPSHOT</version> <scope>test</scope> </dependency> http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/akka-gremlin/src/main/java/org/apache/tinkerpop/gremlin/akka/process/actors/WorkerActor.java ---------------------------------------------------------------------- diff --git a/akka-gremlin/src/main/java/org/apache/tinkerpop/gremlin/akka/process/actors/WorkerActor.java b/akka-gremlin/src/main/java/org/apache/tinkerpop/gremlin/akka/process/actors/WorkerActor.java index 02c4e4c..9ce3704 100644 --- a/akka-gremlin/src/main/java/org/apache/tinkerpop/gremlin/akka/process/actors/WorkerActor.java +++ b/akka-gremlin/src/main/java/org/apache/tinkerpop/gremlin/akka/process/actors/WorkerActor.java @@ -99,11 +99,6 @@ public final class WorkerActor extends AbstractActor implements RequiresMessageQ } @Override - public Partitioner partitioner() { - return this.partitioner; - } - - @Override public Address.Worker address() { return this.self; } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/akka-gremlin/src/test/groovy/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaGroovyPlayTest.groovy ---------------------------------------------------------------------- diff --git a/akka-gremlin/src/test/groovy/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaGroovyPlayTest.groovy b/akka-gremlin/src/test/groovy/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaGroovyPlayTest.groovy deleted file mode 100644 index b4730c4..0000000 --- a/akka-gremlin/src/test/groovy/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaGroovyPlayTest.groovy +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tinkerpop.gremlin.akka.process.actors - -import org.apache.tinkerpop.gremlin.process.computer.Computer -import org.apache.tinkerpop.gremlin.structure.T -import org.apache.tinkerpop.gremlin.structure.io.IoCore -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph -import org.apache.tinkerpop.gremlin.util.TimeUtil -import org.junit.Ignore -import org.junit.Test - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -class AkkaGroovyPlayTest { - - @Test - @Ignore - public void testStuff() { - - def graph = TinkerGraph.open() - graph.io(IoCore.gryo()).readGraph("/Users/marko/Desktop/test.kryo") - def g = graph.traversal() - def a = graph.traversal().withProcessor(AkkaGraphActors.open().workers(8)); - - println TimeUtil.clockWithResult(1) { g.V().id().select("c").count().next() } - println TimeUtil.clockWithResult(1) { g.V().id().select("c").dedup().count().next() } - println TimeUtil.clockWithResult(1) { a.V().id().select("c").count().next() } - println TimeUtil.clockWithResult(1) { a.V().id().select("c").dedup().count().next() } - } -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/akka-gremlin/src/test/java/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaActorsProvider.java ---------------------------------------------------------------------- diff --git a/akka-gremlin/src/test/java/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaActorsProvider.java b/akka-gremlin/src/test/java/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaActorsProvider.java index 1309491..7a27994 100644 --- a/akka-gremlin/src/test/java/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaActorsProvider.java +++ b/akka-gremlin/src/test/java/org/apache/tinkerpop/gremlin/akka/process/actors/AkkaActorsProvider.java @@ -20,9 +20,7 @@ package org.apache.tinkerpop.gremlin.akka.process.actors; import org.apache.commons.configuration.Configuration; -import org.apache.tinkerpop.gremlin.AbstractGraphProvider; import org.apache.tinkerpop.gremlin.LoadGraphWith; -import org.apache.tinkerpop.gremlin.TestHelper; import org.apache.tinkerpop.gremlin.akka.process.actors.io.gryo.GryoSerializer; import org.apache.tinkerpop.gremlin.process.actors.GraphActors; import org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionTest; @@ -35,22 +33,11 @@ import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.Elemen import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategyProcessTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategyProcessTest; import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess; -import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerElement; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraphVariables; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerProperty; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertexProperty; +import org.apache.tinkerpop.gremlin.tinkergraph.AbstractTinkerGraphProvider; import java.util.Arrays; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Map; import java.util.Random; import java.util.Set; @@ -58,7 +45,7 @@ import java.util.Set; /** * @author Marko A. Rodriguez (http://markorodriguez.com) */ -public class AkkaActorsProvider extends AbstractGraphProvider { +public class AkkaActorsProvider extends AbstractTinkerGraphProvider { private static final Random RANDOM = new Random(); @@ -81,128 +68,37 @@ public class AkkaActorsProvider extends AbstractGraphProvider { ElementIdStrategyProcessTest.class.getCanonicalName(), TraversalInterruptionTest.class.getCanonicalName())); - private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{ - add(TinkerEdge.class); - add(TinkerElement.class); - add(TinkerGraph.class); - add(TinkerGraphVariables.class); - add(TinkerProperty.class); - add(TinkerVertex.class); - add(TinkerVertexProperty.class); - }}; - - private static Map<String, String> PATHS = new HashMap<>(); - - static { - try { - final List<String> kryoResources = Arrays.asList( - "tinkerpop-modern.kryo", - "grateful-dead.kryo", - "tinkerpop-classic.kryo", - "tinkerpop-crew.kryo"); - for (final String fileName : kryoResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); - } - - final List<String> graphsonResources = Arrays.asList( - "tinkerpop-modern-v2d0-typed.json", - "grateful-dead-v2d0-typed.json", - "tinkerpop-classic-v2d0-typed.json", - "tinkerpop-crew-v2d0-typed.json"); - for (final String fileName : graphsonResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); - } - } catch (Exception e) { - e.printStackTrace(); - } + @Override + public Set<String> getSkipTests() { + return SKIP_TESTS; } @Override public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, final LoadGraphWith.GraphData loadGraphWith) { - final TinkerGraph.DefaultIdManager idManager = selectIdMakerFromGraphData(loadGraphWith); - final String idMaker = (idManager.equals(TinkerGraph.DefaultIdManager.ANY) ? selectIdMakerFromGraphData(loadGraphWith) : idManager).name(); - return new HashMap<String, Object>() {{ - put(Graph.GRAPH, TinkerGraph.class.getName()); - put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_ID_MANAGER, idMaker); - put(TinkerGraph.GREMLIN_TINKERGRAPH_EDGE_ID_MANAGER, idMaker); - put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_PROPERTY_ID_MANAGER, idMaker); - put("skipTest", SKIP_TESTS.contains(testMethodName) || SKIP_TESTS.contains(test.getCanonicalName())); - if (null != loadGraphWith) { - put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, loadGraphDataViaHadoopConfig(loadGraphWith)); - put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo"); - if (loadGraphWith == LoadGraphWith.GraphData.CREW) - put(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name()); - } - // Akka specific configuration - put(Constants.AKKA_LOG_DEAD_LETTERS_DURING_SHUTDOWN, false); - put(Constants.AKKA_ACTOR_PROVIDER, "remote"); - put(Constants.AKKA_ACTOR_SERIALIZE_MESSAGES, "on"); - put(Constants.AKKA_ACTOR_SERIALIZERS_GRYO, GryoSerializer.class.getCanonicalName()); - put(Constants.AKKA_REMOTE_ENABLED_TRANSPORTS, Collections.singletonList("akka.remote.netty.tcp")); - put(Constants.AKKA_REMOTE_NETTY_TCP_HOSTNAME, "127.0.0.1"); - put(Constants.AKKA_REMOTE_NETTY_TCP_PORT, 2552); - }}; - } - - public String loadGraphDataViaHadoopConfig(final LoadGraphWith.GraphData graphData) { - final String type = ".kryo"; - if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) { - return PATHS.get("grateful-dead" + type); - } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) { - return PATHS.get("tinkerpop-modern" + type); - } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) { - return PATHS.get("tinkerpop-classic" + type); - } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) { - return PATHS.get("tinkerpop-crew" + type); - } else { - throw new RuntimeException("Could not load graph with " + graphData); - } - } - - @Override - public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) { - + final Map<String, Object> configuration = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); + // Akka specific configuration + configuration.put(Constants.AKKA_LOG_DEAD_LETTERS_DURING_SHUTDOWN, false); + configuration.put(Constants.AKKA_ACTOR_PROVIDER, "remote"); + configuration.put(Constants.AKKA_ACTOR_SERIALIZE_MESSAGES, "on"); + configuration.put(Constants.AKKA_ACTOR_SERIALIZERS_GRYO, GryoSerializer.class.getCanonicalName()); + configuration.put(Constants.AKKA_REMOTE_ENABLED_TRANSPORTS, Collections.singletonList("akka.remote.netty.tcp")); + configuration.put(Constants.AKKA_REMOTE_NETTY_TCP_HOSTNAME, "127.0.0.1"); + configuration.put(Constants.AKKA_REMOTE_NETTY_TCP_PORT, 2552); + return configuration; } @Override public void clear(final Graph graph, final Configuration configuration) throws Exception { - // if (graph != null) graph.close(); - } - - @Override - public Set<Class> getImplementations() { - return IMPLEMENTATION; - } - - /** - * Test that load with specific graph data can be configured with a specific id manager as the data type to - * be used in the test for that graph is known. - */ - protected TinkerGraph.DefaultIdManager selectIdMakerFromGraphData(final LoadGraphWith.GraphData loadGraphWith) { - if (null == loadGraphWith) return TinkerGraph.DefaultIdManager.ANY; - if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) - return TinkerGraph.DefaultIdManager.INTEGER; - else if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) - return TinkerGraph.DefaultIdManager.INTEGER; - else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) - return TinkerGraph.DefaultIdManager.INTEGER; - else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) - return TinkerGraph.DefaultIdManager.INTEGER; - else - throw new IllegalStateException(String.format("Need to define a new %s for %s", TinkerGraph.IdManager.class.getName(), loadGraphWith.name())); + // don't delete the loaded data } -///////////////////////////// -///////////////////////////// -///////////////////////////// @Override public GraphTraversalSource traversal(final Graph graph) { - if ((Boolean) graph.configuration().getProperty("skipTest")) + if ((Boolean) graph.configuration().getProperty(GREMLIN_TINKERGRAPH_SKIP_TEST)) return graph.traversal(); - //throw new VerificationException("This test current does not work with Gremlin-Python", EmptyTraversal.instance()); else { final GraphTraversalSource g = graph.traversal(); return RANDOM.nextBoolean() ? http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/Actor.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/Actor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/Actor.java index 0445968..5d06168 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/Actor.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/Actor.java @@ -36,13 +36,6 @@ import java.util.List; public interface Actor { /** - * Get the {@link Partitioner} associated with the {@link GraphActors} system. - * - * @return the partitioner used to partition (logically and/or physically) the {@link org.apache.tinkerpop.gremlin.structure.Graph} - */ - public Partitioner partitioner(); - - /** * Get the {@link Address} of the actors. * * @return the actors's address @@ -67,8 +60,21 @@ public interface Actor { public interface Master extends Actor { + /** + * Get the master actors address. + * + * @return the master actor's address + */ + @Override public Address.Master address(); + /** + * Get the {@link Partitioner} associated with the {@link GraphActors} system. + * + * @return the partitioner used to partition (logically and/or physically) the {@link org.apache.tinkerpop.gremlin.structure.Graph} + */ + public Partitioner partitioner(); + public void close(); public <R> ActorsResult<R> result(); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/traversal/TraversalWorkerProgram.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/traversal/TraversalWorkerProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/traversal/TraversalWorkerProgram.java index b3b73d5..4e49f03 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/traversal/TraversalWorkerProgram.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/actors/traversal/TraversalWorkerProgram.java @@ -90,8 +90,8 @@ final class TraversalWorkerProgram implements ActorProgram.Worker<Object> { final int i = this.self.workers().indexOf(this.self.address()); this.neighborWorker = this.self.workers().get(i == this.self.workers().size() - 1 ? 0 : i + 1); this.isLeader = i == 0; - for (int j = 0; j < this.self.partitioner().getPartitions().size(); j++) { - this.partitionToWorkerMap.put(this.self.partitioner().getPartitions().get(j), this.self.workers().get(j)); + for (int j = 0; j < this.self.partition().partitioner().getPartitions().size(); j++) { + this.partitionToWorkerMap.put(this.self.partition().partitioner().getPartitions().get(j), this.self.workers().get(j)); } } @@ -177,7 +177,7 @@ final class TraversalWorkerProgram implements ActorProgram.Worker<Object> { if (traverser.isHalted()) this.self.send(this.self.master(), traverser); else if (traverser.get() instanceof Element && !this.self.partition().contains((Element) traverser.get())) - this.self.send(this.partitionToWorkerMap.get(this.self.partitioner().find((Element) traverser.get())), traverser); + this.self.send(this.partitionToWorkerMap.get(this.self.partition().partitioner().find((Element) traverser.get())), traverser); else this.self.send(this.self.address(), traverser); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/gremlin-tools/gremlin-tinkergraph-test/pom.xml ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-tinkergraph-test/pom.xml b/gremlin-tools/gremlin-tinkergraph-test/pom.xml new file mode 100644 index 0000000..cead50a --- /dev/null +++ b/gremlin-tools/gremlin-tinkergraph-test/pom.xml @@ -0,0 +1,59 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <artifactId>gremlin-tools</artifactId> + <groupId>org.apache.tinkerpop</groupId> + <version>3.3.0-SNAPSHOT</version> + </parent> + <artifactId>gremlin-tinkergraph-test</artifactId> + <name>Apache TinkerPop :: Gremlin TinkerGraph Test</name> + <dependencies> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>tinkergraph-gremlin</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tinkerpop</groupId> + <artifactId>gremlin-test</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + <optional>true</optional> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/gremlin-tools/gremlin-tinkergraph-test/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/AbstractTinkerGraphProvider.java ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-tinkergraph-test/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/AbstractTinkerGraphProvider.java b/gremlin-tools/gremlin-tinkergraph-test/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/AbstractTinkerGraphProvider.java new file mode 100644 index 0000000..a25ac66 --- /dev/null +++ b/gremlin-tools/gremlin-tinkergraph-test/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/AbstractTinkerGraphProvider.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.tinkergraph; + +import org.apache.commons.configuration.Configuration; +import org.apache.tinkerpop.gremlin.AbstractGraphProvider; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.GraphTest; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.apache.tinkerpop.gremlin.structure.io.IoEdgeTest; +import org.apache.tinkerpop.gremlin.structure.io.IoVertexTest; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedGraphTest; +import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphTest; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerElement; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraphVariables; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerProperty; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertexProperty; + +import java.io.File; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +public abstract class AbstractTinkerGraphProvider extends AbstractGraphProvider { + + protected static final String GREMLIN_TINKERGRAPH_SKIP_TEST = "gremlin.tinkergraph.skipTest"; + + private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{ + add(TinkerEdge.class); + add(TinkerElement.class); + add(TinkerGraph.class); + add(TinkerGraphVariables.class); + add(TinkerProperty.class); + add(TinkerVertex.class); + add(TinkerVertexProperty.class); + }}; + + private static Map<String, String> PATHS = new HashMap<>(); + + static { + try { + final List<String> kryoResources = Arrays.asList( + "tinkerpop-modern.kryo", + "grateful-dead.kryo", + "tinkerpop-classic.kryo", + "tinkerpop-crew.kryo"); + for (final String fileName : kryoResources) { + PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + public abstract Set<String> getSkipTests(); + + @Override + public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, + final LoadGraphWith.GraphData loadGraphWith) { + final TinkerGraph.DefaultIdManager idManager = selectIdMakerFromGraphData(loadGraphWith); + final String idMaker = (idManager.equals(TinkerGraph.DefaultIdManager.ANY) ? selectIdMakerFromTest(test, testMethodName) : idManager).name(); + return new HashMap<String, Object>() {{ + put(Graph.GRAPH, TinkerGraph.class.getName()); + put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_ID_MANAGER, idMaker); + put(TinkerGraph.GREMLIN_TINKERGRAPH_EDGE_ID_MANAGER, idMaker); + put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_PROPERTY_ID_MANAGER, idMaker); + put(GREMLIN_TINKERGRAPH_SKIP_TEST, getSkipTests().contains(testMethodName) || getSkipTests().contains(test.getCanonicalName())); + if (requiresListCardinalityAsDefault(loadGraphWith, test, testMethodName)) + put(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name()); + if (requiresPersistence(test, testMethodName)) { + put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo"); + final File tempDir = TestHelper.makeTestDataPath(test, "temp"); + put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, + tempDir.getAbsolutePath() + File.separator + testMethodName + ".kryo"); + } + if (null != loadGraphWith) { + put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, getGraphLocation(loadGraphWith)); + put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo"); + } + }}; + } + + + private final String getGraphLocation(final LoadGraphWith.GraphData graphData) { + final String type = ".kryo"; + if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) { + return PATHS.get("grateful-dead" + type); + } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) { + return PATHS.get("tinkerpop-modern" + type); + } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) { + return PATHS.get("tinkerpop-classic" + type); + } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) { + return PATHS.get("tinkerpop-crew" + type); + } else { + throw new RuntimeException("Could not load graph with " + graphData); + } + } + + @Override + public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) { + // all data is loaded via file + } + + @Override + public void clear(final Graph graph, final Configuration configuration) throws Exception { + if (graph != null) + graph.close(); + + // in the even the graph is persisted we need to clean up + final String graphLocation = configuration.getString(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, null); + if (graphLocation != null) { + final File f = new File(graphLocation); + f.delete(); + } + } + + @Override + public Set<Class> getImplementations() { + return IMPLEMENTATION; + } + + /** + * Determines if a test requires TinkerGraph persistence to be configured with graph location and format. + */ + protected static boolean requiresPersistence(final Class<?> test, final String testMethodName) { + return test == GraphTest.class && testMethodName.equals("shouldPersistDataOnClose"); + } + + /** + * Determines if a test requires a different cardinality as the default or not. + */ + protected static boolean requiresListCardinalityAsDefault(final LoadGraphWith.GraphData loadGraphWith, + final Class<?> test, final String testMethodName) { + return loadGraphWith == LoadGraphWith.GraphData.CREW + || (test == StarGraphTest.class && testMethodName.equals("shouldAttachWithCreateMethod")) + || (test == DetachedGraphTest.class && testMethodName.equals("testAttachableCreateMethod")); + } + + /** + * Some tests require special configuration for TinkerGraph to properly configure the id manager. + */ + protected TinkerGraph.DefaultIdManager selectIdMakerFromTest(final Class<?> test, final String testMethodName) { + if (test.equals(GraphTest.class)) { + final Set<String> testsThatNeedLongIdManager = new HashSet<String>() {{ + add("shouldIterateVerticesWithNumericIdSupportUsingDoubleRepresentation"); + add("shouldIterateVerticesWithNumericIdSupportUsingDoubleRepresentations"); + add("shouldIterateVerticesWithNumericIdSupportUsingIntegerRepresentation"); + add("shouldIterateVerticesWithNumericIdSupportUsingIntegerRepresentations"); + add("shouldIterateVerticesWithNumericIdSupportUsingFloatRepresentation"); + add("shouldIterateVerticesWithNumericIdSupportUsingFloatRepresentations"); + add("shouldIterateVerticesWithNumericIdSupportUsingStringRepresentation"); + add("shouldIterateVerticesWithNumericIdSupportUsingStringRepresentations"); + add("shouldIterateEdgesWithNumericIdSupportUsingDoubleRepresentation"); + add("shouldIterateEdgesWithNumericIdSupportUsingDoubleRepresentations"); + add("shouldIterateEdgesWithNumericIdSupportUsingIntegerRepresentation"); + add("shouldIterateEdgesWithNumericIdSupportUsingIntegerRepresentations"); + add("shouldIterateEdgesWithNumericIdSupportUsingFloatRepresentation"); + add("shouldIterateEdgesWithNumericIdSupportUsingFloatRepresentations"); + add("shouldIterateEdgesWithNumericIdSupportUsingStringRepresentation"); + add("shouldIterateEdgesWithNumericIdSupportUsingStringRepresentations"); + }}; + + final Set<String> testsThatNeedUuidIdManager = new HashSet<String>() {{ + add("shouldIterateVerticesWithUuidIdSupportUsingStringRepresentation"); + add("shouldIterateVerticesWithUuidIdSupportUsingStringRepresentations"); + add("shouldIterateEdgesWithUuidIdSupportUsingStringRepresentation"); + add("shouldIterateEdgesWithUuidIdSupportUsingStringRepresentations"); + }}; + + if (testsThatNeedLongIdManager.contains(testMethodName)) + return TinkerGraph.DefaultIdManager.LONG; + else if (testsThatNeedUuidIdManager.contains(testMethodName)) + return TinkerGraph.DefaultIdManager.UUID; + } else if (test.equals(IoEdgeTest.class)) { + final Set<String> testsThatNeedLongIdManager = new HashSet<String>() {{ + add("shouldReadWriteEdge[graphson-v1]"); + add("shouldReadWriteDetachedEdgeAsReference[graphson-v1]"); + add("shouldReadWriteDetachedEdge[graphson-v1]"); + add("shouldReadWriteEdge[graphson-v2]"); + add("shouldReadWriteDetachedEdgeAsReference[graphson-v2]"); + add("shouldReadWriteDetachedEdge[graphson-v2]"); + }}; + + if (testsThatNeedLongIdManager.contains(testMethodName)) + return TinkerGraph.DefaultIdManager.LONG; + } else if (test.equals(IoVertexTest.class)) { + final Set<String> testsThatNeedLongIdManager = new HashSet<String>() {{ + add("shouldReadWriteVertexWithBOTHEdges[graphson-v1]"); + add("shouldReadWriteVertexWithINEdges[graphson-v1]"); + add("shouldReadWriteVertexWithOUTEdges[graphson-v1]"); + add("shouldReadWriteVertexNoEdges[graphson-v1]"); + add("shouldReadWriteDetachedVertexNoEdges[graphson-v1]"); + add("shouldReadWriteDetachedVertexAsReferenceNoEdges[graphson-v1]"); + add("shouldReadWriteVertexMultiPropsNoEdges[graphson-v1]"); + add("shouldReadWriteVertexWithBOTHEdges[graphson-v2]"); + add("shouldReadWriteVertexWithINEdges[graphson-v2]"); + add("shouldReadWriteVertexWithOUTEdges[graphson-v2]"); + add("shouldReadWriteVertexNoEdges[graphson-v2]"); + add("shouldReadWriteDetachedVertexNoEdges[graphson-v2]"); + add("shouldReadWriteDetachedVertexAsReferenceNoEdges[graphson-v2]"); + add("shouldReadWriteVertexMultiPropsNoEdges[graphson-v2]"); + }}; + + if (testsThatNeedLongIdManager.contains(testMethodName)) + return TinkerGraph.DefaultIdManager.LONG; + } + + return TinkerGraph.DefaultIdManager.ANY; + } + + /** + * Test that load with specific graph data can be configured with a specific id manager as the data type to + * be used in the test for that graph is known. + */ + protected TinkerGraph.DefaultIdManager selectIdMakerFromGraphData(final LoadGraphWith.GraphData loadGraphWith) { + if (null == loadGraphWith) return TinkerGraph.DefaultIdManager.ANY; + if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) + return TinkerGraph.DefaultIdManager.INTEGER; + else if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) + return TinkerGraph.DefaultIdManager.INTEGER; + else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) + return TinkerGraph.DefaultIdManager.INTEGER; + else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) + return TinkerGraph.DefaultIdManager.INTEGER; + else + throw new IllegalStateException(String.format("Need to define a new %s for %s", TinkerGraph.IdManager.class.getName(), loadGraphWith.name())); + } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/gremlin-tools/pom.xml ---------------------------------------------------------------------- diff --git a/gremlin-tools/pom.xml b/gremlin-tools/pom.xml index 864bb95..eb0fbea 100644 --- a/gremlin-tools/pom.xml +++ b/gremlin-tools/pom.xml @@ -16,6 +16,7 @@ <module>gremlin-benchmark</module> <module>gremlin-coverage</module> <module>gremlin-io-test</module> + <module>gremlin-tinkergraph-test</module> </modules> <build> http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/spark-gremlin/pom.xml ---------------------------------------------------------------------- diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml index a08fdfd..9e8dd0e 100644 --- a/spark-gremlin/pom.xml +++ b/spark-gremlin/pom.xml @@ -351,7 +351,7 @@ </dependency> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>tinkergraph-gremlin</artifactId> + <artifactId>gremlin-tinkergraph-test</artifactId> <version>${project.version}</version> <scope>test</scope> </dependency> http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphPartitionerComputerProcessIntegrateTest.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphPartitionerComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphPartitionerComputerProcessIntegrateTest.java new file mode 100644 index 0000000..3b5b8a4 --- /dev/null +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphPartitionerComputerProcessIntegrateTest.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.spark.process.computer; + +import org.apache.tinkerpop.gremlin.GraphProviderClass; +import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; +import org.junit.runner.RunWith; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +@RunWith(ProcessComputerSuite.class) +@GraphProviderClass(provider = SparkTinkerGraphProvider.class, graph = TinkerGraph.class) +public class SparkGraphPartitionerComputerProcessIntegrateTest { +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java index 760bba6..556d9d5 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkHadoopGraphProvider.java @@ -28,7 +28,6 @@ import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider; import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPluginCheck; import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorageCheck; import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.ToyIoRegistry; import org.apache.tinkerpop.gremlin.process.computer.Computer; import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; @@ -42,12 +41,9 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest; import org.apache.tinkerpop.gremlin.spark.structure.Spark; import org.apache.tinkerpop.gremlin.spark.structure.io.PersistedOutputRDD; import org.apache.tinkerpop.gremlin.spark.structure.io.SparkContextStorageCheck; -import org.apache.tinkerpop.gremlin.spark.structure.io.SparkIoRegistryCheck; import org.apache.tinkerpop.gremlin.spark.structure.io.ToyGraphInputRDD; import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoRegistrator; import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; -import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimService; import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShimServiceLoader; import java.util.Map; http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkTinkerGraphProvider.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkTinkerGraphProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkTinkerGraphProvider.java new file mode 100644 index 0000000..19a7f8c --- /dev/null +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkTinkerGraphProvider.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.spark.process.computer; + +import org.apache.commons.configuration.Configuration; +import org.apache.tinkerpop.gremlin.GraphProvider; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.partitioner.PartitionerInputFormat; +import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; +import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest; +import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgramTest; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionComputerTest; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.PeerPressureTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategyProcessTest; +import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.tinkergraph.AbstractTinkerGraphProvider; +import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComputer; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static org.apache.tinkerpop.gremlin.structure.io.gryo.GryoPool.CONFIG_IO_GRYO_POOL_SIZE; + +/** + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +@GraphProvider.Descriptor(computer = SparkGraphComputer.class) +public class SparkTinkerGraphProvider extends AbstractTinkerGraphProvider { + + private static Set<String> SKIP_TESTS = new HashSet<>(Arrays.asList( + TraversalInterruptionComputerTest.class.getCanonicalName(), + PageRankTest.Traversals.class.getCanonicalName(), + ProgramTest.Traversals.class.getCanonicalName(), + PeerPressureTest.Traversals.class.getCanonicalName(), + BulkLoaderVertexProgramTest.class.getCanonicalName(), + PageRankVertexProgramTest.class.getCanonicalName(), + ReadOnlyStrategyProcessTest.class.getCanonicalName(), + "testProfileStrategyCallback", + "testProfileStrategyCallbackSideEffect", + "shouldSucceedWithProperTraverserRequirements", + "shouldStartAndEndWorkersForVertexProgramAndMapReduce", + "shouldFailWithImproperTraverserRequirements")); + + @Override + public Set<String> getSkipTests() { + return SKIP_TESTS; + } + + @Override + public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, + final LoadGraphWith.GraphData loadGraphWith) { + + final Map<String, Object> configuration = super.getBaseConfiguration(graphName, test, testMethodName, loadGraphWith); + + // Spark specific configuration + configuration.put("spark.master", "local[4]"); + configuration.put("spark.serializer", GryoSerializer.class.getCanonicalName()); + configuration.put("spark.kryo.registrationRequired", true); + configuration.put(Constants.GREMLIN_HADOOP_GRAPH_READER, PartitionerInputFormat.class.getCanonicalName()); + configuration.put(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName()); + configuration.put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, getWorkingDirectory()); + configuration.put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); + configuration.put(CONFIG_IO_GRYO_POOL_SIZE, 16); + if (!(Boolean) configuration.getOrDefault(GREMLIN_TINKERGRAPH_SKIP_TEST, false)) + configuration.put(GraphComputer.GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName()); + configuration.put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); + return configuration; + } + + @Override + public void clear(final Graph graph, final Configuration configuration) throws Exception { + // don't clear loaded data files + } + + @Override + public GraphTraversalSource traversal(final Graph graph) { + if (graph.configuration().getBoolean(GREMLIN_TINKERGRAPH_SKIP_TEST, false)) + return graph.traversal().withStrategies(ReadOnlyStrategy.instance()).withComputer(); + else + return graph.traversal().withStrategies(ReadOnlyStrategy.instance()).withProcessor(SparkGraphComputer.open(graph.configuration())); + } + + @Override + public GraphComputer getGraphComputer(final Graph graph) { + if (graph.configuration().getBoolean(GREMLIN_TINKERGRAPH_SKIP_TEST, false)) + return new TinkerGraphComputer((TinkerGraph) graph); + else + return SparkGraphComputer.open(graph.configuration()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/SparkGraphPartitionerComputerProcessIntegrateTest.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/SparkGraphPartitionerComputerProcessIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/SparkGraphPartitionerComputerProcessIntegrateTest.java deleted file mode 100644 index 0cefaa5..0000000 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/SparkGraphPartitionerComputerProcessIntegrateTest.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tinkerpop.gremlin.spark.structure.io.partitioner; - -import org.apache.tinkerpop.gremlin.GraphProviderClass; -import org.apache.tinkerpop.gremlin.process.ProcessComputerSuite; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; -import org.junit.Ignore; -import org.junit.runner.RunWith; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -@RunWith(ProcessComputerSuite.class) -@GraphProviderClass(provider = TinkerGraphPartitionerProvider.class, graph = TinkerGraph.class) -public class SparkGraphPartitionerComputerProcessIntegrateTest { -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/cb9cc637/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/TinkerGraphPartitionerProvider.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/TinkerGraphPartitionerProvider.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/TinkerGraphPartitionerProvider.java deleted file mode 100644 index 15fa3ac..0000000 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/partitioner/TinkerGraphPartitionerProvider.java +++ /dev/null @@ -1,224 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tinkerpop.gremlin.spark.structure.io.partitioner; - -import org.apache.commons.configuration.Configuration; -import org.apache.tinkerpop.gremlin.AbstractGraphProvider; -import org.apache.tinkerpop.gremlin.AbstractGremlinTest; -import org.apache.tinkerpop.gremlin.GraphProvider; -import org.apache.tinkerpop.gremlin.LoadGraphWith; -import org.apache.tinkerpop.gremlin.TestHelper; -import org.apache.tinkerpop.gremlin.hadoop.Constants; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.partitioner.PartitionerInputFormat; -import org.apache.tinkerpop.gremlin.process.computer.GraphComputer; -import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest; -import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgramTest; -import org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionComputerTest; -import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; -import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DropTest; -import org.apache.tinkerpop.gremlin.process.traversal.step.map.PageRankTest; -import org.apache.tinkerpop.gremlin.process.traversal.step.map.PeerPressureTest; -import org.apache.tinkerpop.gremlin.process.traversal.step.map.ProgramTest; -import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategy; -import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ReadOnlyStrategyProcessTest; -import org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer; -import org.apache.tinkerpop.gremlin.spark.structure.io.gryo.GryoSerializer; -import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess; -import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; -import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComputer; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerEdge; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerElement; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraphVariables; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerProperty; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertex; -import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertexProperty; - -import java.io.IOException; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -@GraphProvider.Descriptor(computer = SparkGraphComputer.class) -public class TinkerGraphPartitionerProvider extends AbstractGraphProvider { - - private static Set<String> SKIP_TESTS = new HashSet<>(Arrays.asList( - TraversalInterruptionComputerTest.class.getCanonicalName(), - PageRankTest.Traversals.class.getCanonicalName(), - ProgramTest.Traversals.class.getCanonicalName(), - PeerPressureTest.Traversals.class.getCanonicalName(), - BulkLoaderVertexProgramTest.class.getCanonicalName(), - PageRankVertexProgramTest.class.getCanonicalName(), - ReadOnlyStrategyProcessTest.class.getCanonicalName(), - "testProfileStrategyCallback", - "testProfileStrategyCallbackSideEffect", - "shouldSucceedWithProperTraverserRequirements", - "shouldStartAndEndWorkersForVertexProgramAndMapReduce", - "shouldFailWithImproperTraverserRequirements")); - - private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{ - add(TinkerEdge.class); - add(TinkerElement.class); - add(TinkerGraph.class); - add(TinkerGraphVariables.class); - add(TinkerProperty.class); - add(TinkerVertex.class); - add(TinkerVertexProperty.class); - }}; - - private static Map<String, String> PATHS = new HashMap<>(); - - static { - try { - final List<String> kryoResources = Arrays.asList( - "tinkerpop-modern.kryo", - "grateful-dead.kryo", - "tinkerpop-classic.kryo", - "tinkerpop-crew.kryo"); - for (final String fileName : kryoResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); - } - - final List<String> graphsonResources = Arrays.asList( - "tinkerpop-modern-v2d0-typed.json", - "grateful-dead-v2d0-typed.json", - "tinkerpop-classic-v2d0-typed.json", - "tinkerpop-crew-v2d0-typed.json"); - for (final String fileName : graphsonResources) { - PATHS.put(fileName, TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, "").getAbsolutePath().replace('\\', '/')); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - - @Override - public Map<String, Object> getBaseConfiguration(final String graphName, final Class<?> test, final String testMethodName, - final LoadGraphWith.GraphData loadGraphWith) { - - final TinkerGraph.DefaultIdManager idManager = selectIdMakerFromGraphData(loadGraphWith); - final String idMaker = (idManager.equals(TinkerGraph.DefaultIdManager.ANY) ? selectIdMakerFromGraphData(loadGraphWith) : idManager).name(); - final boolean skipTest = SKIP_TESTS.contains(testMethodName) || SKIP_TESTS.contains(test.getCanonicalName()); - return new HashMap<String, Object>() {{ - put(Graph.GRAPH, TinkerGraph.class.getName()); - put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_ID_MANAGER, idMaker); - put(TinkerGraph.GREMLIN_TINKERGRAPH_EDGE_ID_MANAGER, idMaker); - put(TinkerGraph.GREMLIN_TINKERGRAPH_VERTEX_PROPERTY_ID_MANAGER, idMaker); - put("skipTest", SKIP_TESTS.contains(testMethodName) || SKIP_TESTS.contains(test.getCanonicalName())); - if (null != loadGraphWith) { - put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_LOCATION, loadGraphDataViaHadoopConfig(loadGraphWith)); - put(TinkerGraph.GREMLIN_TINKERGRAPH_GRAPH_FORMAT, "gryo"); - if (loadGraphWith == LoadGraphWith.GraphData.CREW) - put(TinkerGraph.GREMLIN_TINKERGRAPH_DEFAULT_VERTEX_PROPERTY_CARDINALITY, VertexProperty.Cardinality.list.name()); - } - - // Spark specific configuration - put("spark.master", "local[4]"); - put("spark.serializer", GryoSerializer.class.getCanonicalName()); - put("spark.kryo.registrationRequired", true); - put(Constants.GREMLIN_HADOOP_GRAPH_READER, PartitionerInputFormat.class.getCanonicalName()); - put(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName()); - put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, getWorkingDirectory()); - put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); - if (!skipTest) - put(GraphComputer.GRAPH_COMPUTER, SparkGraphComputer.class.getCanonicalName()); - put(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true); - }}; - - - } - - public String loadGraphDataViaHadoopConfig(final LoadGraphWith.GraphData graphData) { - final String type = ".kryo"; - if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) { - return PATHS.get("grateful-dead" + type); - } else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) { - return PATHS.get("tinkerpop-modern" + type); - } else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) { - return PATHS.get("tinkerpop-classic" + type); - } else if (graphData.equals(LoadGraphWith.GraphData.CREW)) { - return PATHS.get("tinkerpop-crew" + type); - } else { - throw new RuntimeException("Could not load graph with " + graphData); - } - } - - @Override - public void loadGraphData(final Graph graph, final LoadGraphWith loadGraphWith, final Class testClass, final String testName) { - - } - - @Override - public void clear(final Graph graph, final Configuration configuration) throws Exception { - // if (graph != null) graph.close(); - } - - @Override - public Set<Class> getImplementations() { - return IMPLEMENTATION; - } - - /** - * Test that load with specific graph data can be configured with a specific id manager as the data type to - * be used in the test for that graph is known. - */ - protected TinkerGraph.DefaultIdManager selectIdMakerFromGraphData(final LoadGraphWith.GraphData loadGraphWith) { - if (null == loadGraphWith) return TinkerGraph.DefaultIdManager.ANY; - if (loadGraphWith.equals(LoadGraphWith.GraphData.CLASSIC)) - return TinkerGraph.DefaultIdManager.INTEGER; - else if (loadGraphWith.equals(LoadGraphWith.GraphData.MODERN)) - return TinkerGraph.DefaultIdManager.INTEGER; - else if (loadGraphWith.equals(LoadGraphWith.GraphData.CREW)) - return TinkerGraph.DefaultIdManager.INTEGER; - else if (loadGraphWith.equals(LoadGraphWith.GraphData.GRATEFUL)) - return TinkerGraph.DefaultIdManager.INTEGER; - else - throw new IllegalStateException(String.format("Need to define a new %s for %s", TinkerGraph.IdManager.class.getName(), loadGraphWith.name())); - } - - ///////////////////////////// - ///////////////////////////// - ///////////////////////////// - - @Override - public GraphTraversalSource traversal(final Graph graph) { - if (graph.configuration().getBoolean("skipTest", false)) - return graph.traversal().withStrategies(ReadOnlyStrategy.instance()).withComputer(); - else - return graph.traversal().withStrategies(ReadOnlyStrategy.instance()).withProcessor(SparkGraphComputer.open(graph.configuration())); - } - - @Override - public GraphComputer getGraphComputer(final Graph graph) { - if (graph.configuration().getBoolean("skipTest", false)) - return new TinkerGraphComputer((TinkerGraph) graph); - else - return SparkGraphComputer.open(graph.configuration()); - } -} \ No newline at end of file
