This is an automated email from the ASF dual-hosted git repository. spmallette pushed a commit to branch TINKERPOP-2601 in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit e75f383131869925650fc6e27ebf21ac06a264cf Author: Stephen Mallette <[email protected]> AuthorDate: Wed Aug 18 10:50:39 2021 -0400 TINKERPOP-2601 Set up gherkin for HadoopGraph --- .../gremlin/AbstractFileGraphProvider.java | 70 +-------- .../TestFiles.java} | 29 +--- hadoop-gremlin/pom.xml | 12 ++ .../gremlin/hadoop/structure/HadoopGraph.java | 5 - .../gremlin/hadoop/HadoopGraphFeatureTest.java | 168 +++++++++++++++++++++ .../structure/io/RecordReaderWriterTest.java | 3 +- .../io/script/ScriptRecordReaderWriterTest.java | 5 +- .../src/test/resources/cucumber.properties | 1 + .../spark/process/computer/LocalPropertyTest.java | 3 +- .../optimization/SparkInterceptorStrategyTest.java | 5 +- .../SparkSingleIterationStrategyTest.java | 3 +- .../gremlin/spark/structure/SparkTest.java | 3 +- .../gremlin/spark/structure/io/OutputRDDTest.java | 3 +- .../io/PersistedInputOutputRDDIntegrateTest.java | 11 +- 14 files changed, 212 insertions(+), 109 deletions(-) diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java index cedbf38..499822f 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java @@ -19,16 +19,9 @@ package org.apache.tinkerpop.gremlin; import org.apache.commons.configuration2.Configuration; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.structure.Graph; -import org.apache.tinkerpop.gremlin.structure.io.Storage; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess; -import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; -import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; import java.util.Random; /** @@ -43,51 +36,6 @@ public abstract class AbstractFileGraphProvider extends AbstractGraphProvider { protected boolean graphSONInput = false; - public static Map<String, String> PATHS = new HashMap<>(); - static { - try { - final List<String> kryoResources = Arrays.asList( - "tinkerpop-modern-v3d0.kryo", - "grateful-dead-v3d0.kryo", - "tinkerpop-classic-v3d0.kryo", - "tinkerpop-crew-v3d0.kryo", - "tinkerpop-sink-v3d0.kryo"); - for (final String fileName : kryoResources) { - PATHS.put(fileName, - Storage.toPath(TestHelper.generateTempFileFromResource(GryoResourceAccess.class, fileName, ""))); - } - - final List<String> graphsonResources = Arrays.asList( - "tinkerpop-modern-typed-v2d0.json", - "tinkerpop-modern-v3d0.json", - "grateful-dead-typed-v2d0.json", - "grateful-dead-v3d0.json", - "tinkerpop-classic-typed-v2d0.json", - "tinkerpop-classic-v3d0.json", - "tinkerpop-crew-typed-v2d0.json", - "tinkerpop-crew-v3d0.json", - "tinkerpop-sink-v3d0.json"); - for (final String fileName : graphsonResources) { - PATHS.put(fileName, - Storage.toPath(TestHelper.generateTempFileFromResource(GraphSONResourceAccess.class, fileName, ""))); - } - - final List<String> scriptResources = Arrays.asList( - "tinkerpop-classic.txt", - "script-input.groovy", - "script-output.groovy", - "grateful-dead.txt", - "script-input-grateful-dead.groovy", - "script-output-grateful-dead.groovy"); - for (final String fileName : scriptResources) { - PATHS.put(fileName, - Storage.toPath(TestHelper.generateTempFileFromResource(ScriptResourceAccess.class, fileName, ""))); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - @Override public void clear(final Graph graph, final Configuration configuration) throws Exception { if (graph != null) @@ -95,20 +43,6 @@ public abstract class AbstractFileGraphProvider extends AbstractGraphProvider { } protected String getInputLocation(final Graph g, final LoadGraphWith.GraphData graphData) { - final String type = this.graphSONInput ? "-v3d0.json" : "-v3d0.kryo"; - - if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) - return PATHS.get("grateful-dead" + type); - else if (graphData.equals(LoadGraphWith.GraphData.MODERN)) - return PATHS.get("tinkerpop-modern" + type); - else if (graphData.equals(LoadGraphWith.GraphData.CLASSIC)) - return PATHS.get("tinkerpop-classic" + type); - else if (graphData.equals(LoadGraphWith.GraphData.CREW)) - return PATHS.get("tinkerpop-crew" + type); - else if (graphData.equals(LoadGraphWith.GraphData.SINK)) - return PATHS.get("tinkerpop-sink" + type); - else - throw new RuntimeException("Could not load graph with " + graphData); + return TestFiles.getInputLocation(graphData, graphSONInput); } - } diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/features/TestFiles.java similarity index 81% copy from gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java copy to gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/features/TestFiles.java index cedbf38..c923452 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/AbstractFileGraphProvider.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/features/TestFiles.java @@ -16,10 +16,10 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.tinkerpop.gremlin; +package org.apache.tinkerpop.gremlin.features; -import org.apache.commons.configuration2.Configuration; -import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.TestHelper; import org.apache.tinkerpop.gremlin.structure.io.Storage; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONResourceAccess; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; @@ -31,19 +31,10 @@ import java.util.List; import java.util.Map; import java.util.Random; -/** - * A base {@link GraphProvider} that is typically for use with Hadoop-based graphs as it enables access to the various - * resource data files that are used in the tests. - * - * @author Stephen Mallette (http://stephen.genoprime.com) - */ -public abstract class AbstractFileGraphProvider extends AbstractGraphProvider { - +public class TestFiles { protected static final Random RANDOM = TestHelper.RANDOM; - protected boolean graphSONInput = false; - - public static Map<String, String> PATHS = new HashMap<>(); + public static final Map<String, String> PATHS = new HashMap<>(); static { try { final List<String> kryoResources = Arrays.asList( @@ -88,14 +79,8 @@ public abstract class AbstractFileGraphProvider extends AbstractGraphProvider { } } - @Override - public void clear(final Graph graph, final Configuration configuration) throws Exception { - if (graph != null) - graph.close(); - } - - protected String getInputLocation(final Graph g, final LoadGraphWith.GraphData graphData) { - final String type = this.graphSONInput ? "-v3d0.json" : "-v3d0.kryo"; + public static String getInputLocation(final LoadGraphWith.GraphData graphData, final boolean useGraphSON) { + final String type = useGraphSON ? "-v3d0.json" : "-v3d0.kryo"; if (graphData.equals(LoadGraphWith.GraphData.GRATEFUL)) return PATHS.get("grateful-dead" + type); diff --git a/hadoop-gremlin/pom.xml b/hadoop-gremlin/pom.xml index 19fa70e..64bb79e 100644 --- a/hadoop-gremlin/pom.xml +++ b/hadoop-gremlin/pom.xml @@ -148,6 +148,18 @@ limitations under the License. <scope>test</scope> </dependency> <dependency> + <groupId>com.google.inject</groupId> + <artifactId>guice</artifactId> + <version>4.2.3</version> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>com.google.guava</groupId> + <artifactId>guava</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> <groupId>ch.qos.logback</groupId> <artifactId>logback-classic</artifactId> <scope>test</scope> diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java index 5a2578b..a36722a 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java @@ -144,11 +144,6 @@ import java.util.stream.StreamSupport; reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", computers = {"ALL"}) @Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.MatchTest$GreedyMatchTraversals", - method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) [email protected]( test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals", method = "g_io_readXxmlX", reason = "Hadoop-Gremlin does not support reads/writes with GraphML.") diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphFeatureTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphFeatureTest.java new file mode 100644 index 0000000..e43b00b --- /dev/null +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphFeatureTest.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.hadoop; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Stage; +import io.cucumber.guice.CucumberModules; +import io.cucumber.guice.GuiceFactory; +import io.cucumber.guice.InjectorSource; +import io.cucumber.java.Scenario; +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; +import org.apache.tinkerpop.gremlin.features.World; +import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; +import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.javatuples.Pair; +import org.junit.AssumptionViolatedException; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData; + +@RunWith(Cucumber.class) +@CucumberOptions( + tags = "not @RemoteOnly and not @MultiMetaProperties", + glue = { "org.apache.tinkerpop.gremlin.features" }, + objectFactory = GuiceFactory.class, + features = { "../gremlin-test/features" }, + plugin = {"pretty", "junit:target/cucumber.xml"}) +public class HadoopGraphFeatureTest { + private static final Logger logger = LoggerFactory.getLogger(HadoopGraphFeatureTest.class); + + private static final String skipReasonLength = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute."; + + private static final List<Pair<String, String>> skip = new ArrayList<Pair<String,String>>() {{ + add(Pair.with("g_V_both_both_count", skipReasonLength)); + add(Pair.with("g_V_repeatXoutX_timesX3X_count", skipReasonLength)); + add(Pair.with("g_V_repeatXoutX_timesX8X_count", skipReasonLength)); + add(Pair.with("g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count", skipReasonLength)); + add(Pair.with("g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX", skipReasonLength)); + add(Pair.with("g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX", skipReasonLength)); + add(Pair.with("g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX", skipReasonLength)); + add(Pair.with("g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX", skipReasonLength)); + add(Pair.with("g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX", skipReasonLength)); + add(Pair.with("g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX", skipReasonLength)); + add(Pair.with("g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX", skipReasonLength)); + add(Pair.with("g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX", skipReasonLength)); + add(Pair.with("g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", skipReasonLength)); + }}; + + public static final class ServiceModule extends AbstractModule { + @Override + protected void configure() { + bind(World.class).to(HadoopGraphWorld.class); + } + } + + public static class HadoopGraphWorld implements World { + + private static final HadoopGraph modern = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.MODERN))); + private static final HadoopGraph classic = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.CLASSIC))); + private static final HadoopGraph crew = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.CREW))); + private static final HadoopGraph sink = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.SINK))); + private static final HadoopGraph grateful = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(GraphData.GRATEFUL))); + private static final HadoopGraph empty = HadoopGraph.open(new MapConfiguration(getBaseConfiguration(null))); + + static { + readIntoGraph(modern, GraphData.MODERN); + readIntoGraph(classic, GraphData.CLASSIC); + readIntoGraph(crew, GraphData.CREW); + readIntoGraph(sink, GraphData.SINK); + readIntoGraph(grateful, GraphData.GRATEFUL); + } + + @Override + public GraphTraversalSource getGraphTraversalSource(final GraphData graphData) { + if (null == graphData) + throw new AssumptionViolatedException("HadoopGraph does not support graph mutations"); + else if (graphData == GraphData.CLASSIC) + return classic.traversal(); + else if (graphData == GraphData.CREW) + return crew.traversal(); + else if (graphData == GraphData.MODERN) + return modern.traversal(); + else if (graphData == GraphData.SINK) + return sink.traversal(); + else if (graphData == GraphData.GRATEFUL) + return grateful.traversal(); + else + throw new UnsupportedOperationException("GraphData not supported: " + graphData.name()); + } + + @Override + public void beforeEachScenario(final Scenario scenario) { + final Optional<Pair<String,String>> skipped = skip.stream(). + filter(s -> s.getValue0().equals(scenario.getName())).findFirst(); + if (skipped.isPresent()) + throw new AssumptionViolatedException(skipped.get().getValue1()); + + cleanEmpty(); + } + + private void cleanEmpty() { + final GraphTraversalSource g = empty.traversal(); + g.V().drop().iterate(); + } + + private static void readIntoGraph(final Graph graph, final GraphData graphData) { + ((HadoopGraph) graph).configuration().setInputLocation(TestFiles.getInputLocation(graphData, false)); + } + + private static String getWorkingDirectory() { + return TestHelper.makeTestDataDirectory(HadoopGraphFeatureTest.class, "graph-provider-data"); + } + + private static Map<String, Object> getBaseConfiguration(final GraphData graphData) { + return new HashMap<String, Object>() {{ + put(Graph.GRAPH, HadoopGraph.class.getName()); + put(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); + put(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName()); + put(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, getWorkingDirectory()); + put(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); + }}; + } + } + + public static final class WorldInjectorSource implements InjectorSource { + @Override + public Injector getInjector() { + return Guice.createInjector(Stage.PRODUCTION, CucumberModules.createScenarioModule(), new ServiceModule()); + } + } + +} diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java index ae93e37..48a6755 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/RecordReaderWriterTest.java @@ -33,6 +33,7 @@ import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; import org.apache.hadoop.util.ReflectionUtils; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider; import org.apache.tinkerpop.gremlin.structure.Direction; import org.apache.tinkerpop.gremlin.structure.Vertex; @@ -70,7 +71,7 @@ public abstract class RecordReaderWriterTest { @Test public void shouldSplitFileAndWriteProperSplits() throws Exception { for (int numberOfSplits = 1; numberOfSplits < 10; numberOfSplits++) { - final File testFile = new File(HadoopGraphProvider.PATHS.get(getInputFilename())); + final File testFile = new File(TestFiles.PATHS.get(getInputFilename())); logger.info("Testing: {}", testFile + " (splits {}", numberOfSplits + ")"); final List<FileSplit> splits = generateFileSplits(testFile, numberOfSplits); final Class<? extends InputFormat<NullWritable, VertexWritable>> inputFormatClass = getInputFormat(); diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java index bb0d783..554b20e 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordReaderWriterTest.java @@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.InputFormat; import org.apache.hadoop.mapreduce.OutputFormat; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider; import org.apache.tinkerpop.gremlin.hadoop.structure.io.RecordReaderWriterTest; import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; @@ -36,8 +37,8 @@ public class ScriptRecordReaderWriterTest extends RecordReaderWriterTest { @Override protected Configuration configure(final File outputDirectory) { final Configuration configuration = super.configure(outputDirectory); - configuration.set(ScriptRecordReader.SCRIPT_FILE, HadoopGraphProvider.PATHS.get("script-input-grateful-dead.groovy")); - configuration.set(ScriptRecordWriter.SCRIPT_FILE, HadoopGraphProvider.PATHS.get("script-output-grateful-dead.groovy")); + configuration.set(ScriptRecordReader.SCRIPT_FILE, TestFiles.PATHS.get("script-input-grateful-dead.groovy")); + configuration.set(ScriptRecordWriter.SCRIPT_FILE, TestFiles.PATHS.get("script-output-grateful-dead.groovy")); return configuration; } diff --git a/hadoop-gremlin/src/test/resources/cucumber.properties b/hadoop-gremlin/src/test/resources/cucumber.properties new file mode 100644 index 0000000..66a963b --- /dev/null +++ b/hadoop-gremlin/src/test/resources/cucumber.properties @@ -0,0 +1 @@ +guice.injector-source=org.apache.tinkerpop.gremlin.hadoop.HadoopGraphFeatureTest$WorldInjectorSource \ No newline at end of file diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java index 829c405..f26d6b0 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/LocalPropertyTest.java @@ -26,6 +26,7 @@ import org.apache.spark.SparkContext; import org.apache.spark.api.java.JavaSparkContext; import org.apache.spark.api.java.JavaSparkStatusTracker; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; @@ -56,7 +57,7 @@ public class LocalPropertyTest extends AbstractSparkTest { configuration.setProperty("spark.master", "local[4]"); configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName()); configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName()); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java index b1d58a6..869963c 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkInterceptorStrategyTest.java @@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o import org.apache.commons.configuration2.Configuration; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep; @@ -60,7 +61,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest { @Test public void shouldHandleSideEffectsCorrectly() throws Exception { final Configuration configuration = getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString())); @@ -88,7 +89,7 @@ public class SparkInterceptorStrategyTest extends AbstractSparkTest { @Test public void shouldSuccessfullyEvaluateInterceptedTraversals() throws Exception { final Configuration configuration = getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString())); diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java index 6202f33..cbd8cd4 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/process/computer/traversal/strategy/optimization/SparkSingleIterationStrategyTest.java @@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.process.computer.traversal.strategy.o import org.apache.commons.configuration2.Configuration; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep; @@ -76,7 +77,7 @@ public class SparkSingleIterationStrategyTest extends AbstractSparkTest { public void shouldSuccessfullyEvaluateSingleIterationTraversals() throws Exception { final String outputLocation = TestHelper.makeTestDataDirectory(SparkSingleIterationStrategyTest.class, UUID.randomUUID().toString()); Configuration configuration = getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, outputLocation); diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java index 5aa78a4..17ef288 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/SparkTest.java @@ -23,6 +23,7 @@ import org.apache.commons.configuration2.BaseConfiguration; import org.apache.commons.configuration2.Configuration; import org.apache.spark.rdd.RDD; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; @@ -59,7 +60,7 @@ public class SparkTest extends AbstractSparkTest { configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName()); configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName()); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, false); diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java index 7042a6c..b675615 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/OutputRDDTest.java @@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.spark.structure.io; import org.apache.commons.configuration2.BaseConfiguration; import org.apache.commons.configuration2.Configuration; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; @@ -46,7 +47,7 @@ public class OutputRDDTest extends AbstractSparkTest { configuration.setProperty("spark.master", "local[4]"); configuration.setProperty("spark.serializer", GryoSerializer.class.getCanonicalName()); configuration.setProperty(Graph.GRAPH, HadoopGraph.class.getName()); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, ExampleOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, TestHelper.makeTestDataDirectory(this.getClass(), "shouldWriteToArbitraryRDD")); diff --git a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java index f26efbc..ded8698 100644 --- a/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java +++ b/spark-gremlin/src/test/java/org/apache/tinkerpop/gremlin/spark/structure/io/PersistedInputOutputRDDIntegrateTest.java @@ -22,6 +22,7 @@ package org.apache.tinkerpop.gremlin.spark.structure.io; import org.apache.commons.configuration2.Configuration; import org.apache.spark.storage.StorageLevel; import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.features.TestFiles; import org.apache.tinkerpop.gremlin.hadoop.Constants; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoOutputFormat; @@ -56,7 +57,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest { Spark.create("local[4]"); final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, GryoOutputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); @@ -84,7 +85,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest { counter++; final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_STORAGE_LEVEL, storageLevel); @@ -112,7 +113,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest { Spark.create("local[4]"); final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); @@ -138,7 +139,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest { final String rddName = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final String rddName2 = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, UUID.randomUUID().toString()); final Configuration configuration = super.getBaseConfiguration(); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); @@ -211,7 +212,7 @@ public class PersistedInputOutputRDDIntegrateTest extends AbstractSparkTest { final String rddName2 = TestHelper.makeTestDataDirectory(PersistedInputOutputRDDIntegrateTest.class, "testComplexChain", "graphRDD2"); final Configuration configuration = super.getBaseConfiguration(); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_READER, GryoInputFormat.class.getCanonicalName()); - configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, SparkHadoopGraphProvider.PATHS.get("tinkerpop-modern-v3d0.kryo")); + configuration.setProperty(Constants.GREMLIN_HADOOP_INPUT_LOCATION, TestFiles.PATHS.get("tinkerpop-modern-v3d0.kryo")); configuration.setProperty(Constants.GREMLIN_HADOOP_GRAPH_WRITER, PersistedOutputRDD.class.getCanonicalName()); configuration.setProperty(Constants.GREMLIN_HADOOP_OUTPUT_LOCATION, rddName); configuration.setProperty(Constants.GREMLIN_SPARK_PERSIST_CONTEXT, true);
