Tweaks to SparkGraphComputer configuration settings. More GraphComputerTest things. Before the break, I want to have Configuration all stubbed out for GraphComputer and GraphActors and then this branch is simply a function of testing GraphActors at scale when we return.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/2f1d4583 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/2f1d4583 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/2f1d4583 Branch: refs/heads/TINKERPOP-1564 Commit: 2f1d4583246c51ee8023b18b451f5baa8a4c159b Parents: 517ad3e Author: Marko A. Rodriguez <okramma...@gmail.com> Authored: Wed Dec 21 11:57:48 2016 -0700 Committer: Marko A. Rodriguez <okramma...@gmail.com> Committed: Mon Jan 23 14:22:53 2017 -0700 ---------------------------------------------------------------------- .../gremlin/process/computer/GraphComputer.java | 4 ++++ .../process/computer/GraphComputerTest.java | 24 ++++++++------------ .../computer/AbstractHadoopGraphComputer.java | 6 ++--- .../process/computer/SparkGraphComputer.java | 2 ++ .../process/computer/TinkerGraphComputer.java | 4 +++- 5 files changed, 22 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2f1d4583/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java index 25074fd..14ab0ec 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputer.java @@ -275,6 +275,10 @@ public interface GraphComputer extends Processor { private Exceptions() { } + public static UnsupportedOperationException graphNotSupported(final Graph graph) { + return new UnsupportedOperationException("The provided graph is not supported: " + graph.getClass().getCanonicalName()); + } + public static UnsupportedOperationException adjacentVertexLabelsCanNotBeRead() { return new UnsupportedOperationException("The label of an adjacent vertex can not be read"); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2f1d4583/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java index a166423..9a99d0a 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java @@ -138,30 +138,26 @@ public class GraphComputerTest extends AbstractGremlinProcessTest { final Configuration graphConfiguration = new BaseConfiguration(); ConfigurationUtils.copy(graph.configuration(), graphConfiguration); final GraphComputer graphComputer = graphProvider.getGraphComputer(graph); - final Configuration graphComputerConfiguration = graphComputer.configuration(); - final Configuration graphComputerConfigurationClone = new BaseConfiguration(); - ConfigurationUtils.copy(graphComputerConfiguration, graphComputerConfigurationClone); - assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputerConfigurationClone)); // creating a graph computer shouldn't alter the graph configuration assertEquals(ConfigurationConverter.getMap(graphConfiguration), ConfigurationConverter.getMap(graph.configuration())); // creating a graph computer should add the graph computer's class name - assertTrue(graphComputerConfiguration.containsKey(GraphComputer.GRAPH_COMPUTER)); - assertEquals(graphComputer.getClass().getCanonicalName(), graphComputerConfiguration.getString(GraphComputer.GRAPH_COMPUTER)); + assertTrue(graphComputer.configuration().containsKey(GraphComputer.GRAPH_COMPUTER)); + assertEquals(graphComputer.getClass().getCanonicalName(), graphComputer.configuration().getString(GraphComputer.GRAPH_COMPUTER)); // specifying worker count should alter the graph computer configuration - int workers = graphComputerConfiguration.containsKey(GraphComputer.WORKERS) ? graphComputerConfiguration.getInt(GraphComputer.WORKERS) : 1; + int workers = graphComputer.configuration().getInt(GraphComputer.WORKERS,1); graphComputer.workers(workers + 1); - assertTrue(graphComputerConfiguration.containsKey(GraphComputer.WORKERS)); - assertEquals(graphComputerConfiguration.getInt(GraphComputer.WORKERS), workers + 1); - assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputer.configuration())); - graphComputerConfigurationClone.clear(); - ConfigurationUtils.copy(graphComputer.configuration(), graphComputerConfigurationClone); - assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputerConfigurationClone)); + assertTrue(graphComputer.configuration().containsKey(GraphComputer.WORKERS)); + assertEquals(graphComputer.configuration().getInt(GraphComputer.WORKERS), workers + 1); + // make a copy of configuration before executing graph computer + final Configuration tempConfiguration = new BaseConfiguration(); + ConfigurationUtils.copy(graphComputer.configuration(),tempConfiguration); // execute graph computer graphComputer.program(new VertexProgramG()).submit(graph); // executing a graph computer should not alter the graph configuration assertEquals(ConfigurationConverter.getMap(graphConfiguration), ConfigurationConverter.getMap(graph.configuration())); // executing a graph computer should not alter the graph computer configuration - // TODO: assertEquals(ConfigurationConverter.getMap(graphComputerConfiguration), ConfigurationConverter.getMap(graphComputerConfigurationClone)); + assertEquals(ConfigurationConverter.getMap(tempConfiguration), ConfigurationConverter.getMap(graphComputer.configuration())); + assertEquals(ConfigurationConverter.getMap(graphConfiguration), ConfigurationConverter.getMap(graph.configuration())); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2f1d4583/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java index a81efb6..b607446 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/AbstractHadoopGraphComputer.java @@ -80,10 +80,10 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer { } protected AbstractHadoopGraphComputer(final org.apache.commons.configuration.Configuration configuration) { - this.configuration = new HadoopConfiguration(configuration); + this.configuration = new HadoopConfiguration(); this.configuration.setProperty(GRAPH_COMPUTER, this.getClass().getCanonicalName()); + GraphComputerHelper.configure(this, configuration); this.logger = LoggerFactory.getLogger(this.getClass()); - GraphComputerHelper.configure(this, this.configuration); } @Override @@ -135,7 +135,7 @@ public abstract class AbstractHadoopGraphComputer implements GraphComputer { @Override public Future<ComputerResult> submit(final Graph graph) { - ConfigurationUtils.copy(graph.configuration(), this.configuration); + GraphComputerHelper.configure(this,graph.configuration()); return this.submit(); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2f1d4583/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java index 8e62c62..ea77a8d 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/process/computer/SparkGraphComputer.java @@ -131,6 +131,8 @@ public final class SparkGraphComputer extends AbstractHadoopGraphComputer { @Override public GraphComputer configure(final String key, final Object value) { + if (key.equals(SparkLauncher.SPARK_MASTER) && value.toString().startsWith("local")) + this.workers(Integer.valueOf(value.toString().substring(6).replace("]", ""))); this.configuration.setProperty(key, value); return this; } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/2f1d4583/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java index 3b91598..2bfd59b 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java @@ -96,7 +96,7 @@ public final class TinkerGraphComputer implements GraphComputer { private TinkerGraphComputer(final Configuration configuration) { this.graph = null; this.configuration = new BaseConfiguration(); - ConfigurationUtils.copy(configuration,this.configuration); + ConfigurationUtils.copy(configuration, this.configuration); this.configuration.setProperty(GRAPH_COMPUTER, TinkerGraphComputer.class.getCanonicalName()); GraphComputerHelper.configure(this, configuration); } @@ -164,6 +164,8 @@ public final class TinkerGraphComputer implements GraphComputer { @Override public Future<ComputerResult> submit(final Graph graph) { + if (!(graph instanceof TinkerGraph)) + throw GraphComputer.Exceptions.graphNotSupported(graph); this.graph = (TinkerGraph) graph; return this.submit(); }