TINKERPOP-1612 Remove references to gremlin-groovy-test in hadoop stuff
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/41bdcd76 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/41bdcd76 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/41bdcd76 Branch: refs/heads/TINKERPOP-1612-wip Commit: 41bdcd76bc3b4cfe700aa3bdcb5f0b08264ee936 Parents: 5e90ed3 Author: Stephen Mallette <sp...@genoprime.com> Authored: Thu Jan 26 10:21:10 2017 -0500 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Thu Jan 26 10:21:10 2017 -0500 ---------------------------------------------------------------------- giraph-gremlin/pom.xml | 11 - .../groovy/plugin/GiraphGremlinPlugin.java | 65 --- ...inkerpop.gremlin.groovy.plugin.GremlinPlugin | 1 - ...pache.tinkerpop.gremlin.jsr223.GremlinPlugin | 1 + ...GraphComputerGroovyProcessIntegrateTest.java | 33 -- .../src/test/resources/gremlin-server.yaml | 8 +- .../gremlin/groovy/util/SugarTestHelper.groovy | 1 - .../groovy/plugin/HadoopGremlinPlugin.java | 104 ----- .../groovy/plugin/HadoopRemoteAcceptor.java | 122 ------ .../gremlin/hadoop/structure/HadoopGraph.java | 103 ----- .../structure/io/script/ScriptRecordWriter.java | 11 +- ...inkerpop.gremlin.groovy.plugin.GremlinPlugin | 1 - .../gremlin/hadoop/HadoopGraphProvider.java | 5 +- .../gremlin/hadoop/HadoopGremlinSuite.java | 2 +- .../groovy/plugin/HadoopGremlinPluginCheck.java | 395 ------------------ .../hadoop/jsr223/HadoopGremlinPluginCheck.java | 397 +++++++++++++++++++ .../HadoopGraphGroovyProcessStandardTest.java | 33 -- pom.xml | 2 +- spark-gremlin/pom.xml | 11 - .../spark/groovy/plugin/SparkGremlinPlugin.java | 74 ---- ...inkerpop.gremlin.groovy.plugin.GremlinPlugin | 1 - .../computer/SparkHadoopGraphProvider.java | 16 +- ...GraphComputerGroovyProcessIntegrateTest.java | 33 -- .../gremlin/spark/util/SugarTestHelper.java | 56 +++ 24 files changed, 470 insertions(+), 1016 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/giraph-gremlin/pom.xml ---------------------------------------------------------------------- diff --git a/giraph-gremlin/pom.xml b/giraph-gremlin/pom.xml index 9a21f36..8f2ee83 100644 --- a/giraph-gremlin/pom.xml +++ b/giraph-gremlin/pom.xml @@ -43,11 +43,6 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>gremlin-groovy</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tinkerpop</groupId> <artifactId>hadoop-gremlin</artifactId> <version>${project.version}</version> <exclusions> @@ -165,12 +160,6 @@ limitations under the License. </dependency> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>gremlin-groovy-test</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.tinkerpop</groupId> <artifactId>tinkergraph-gremlin</artifactId> <version>${project.version}</version> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java ---------------------------------------------------------------------- diff --git a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java b/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java deleted file mode 100644 index 80d98f9..0000000 --- a/giraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/giraph/groovy/plugin/GiraphGremlinPlugin.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tinkerpop.gremlin.giraph.groovy.plugin; - -import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer; -import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin; -import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException; -import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; -import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException; -import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor; - -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - * @deprecated As of release 3.2.4, replaced by {@link org.apache.tinkerpop.gremlin.giraph.jsr223.GiraphGremlinPlugin}. - */ -@Deprecated -public final class GiraphGremlinPlugin extends AbstractGremlinPlugin { - - protected static String NAME = "tinkerpop.giraph"; - - protected static final Set<String> IMPORTS = new HashSet<String>() {{ - add(IMPORT_SPACE + GiraphGraphComputer.class.getPackage().getName() + DOT_STAR); - }}; - - @Override - public String getName() { - return NAME; - } - - @Override - public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException { - pluginAcceptor.addImports(IMPORTS); - } - - @Override - public boolean requireRestart() { - return true; - } - - @Override - public Optional<RemoteAcceptor> remoteAcceptor() { - return Optional.empty(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin ---------------------------------------------------------------------- diff --git a/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin b/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin deleted file mode 100644 index 9c8f3d0..0000000 --- a/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin +++ /dev/null @@ -1 +0,0 @@ -org.apache.tinkerpop.gremlin.giraph.groovy.plugin.GiraphGremlinPlugin \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin ---------------------------------------------------------------------- diff --git a/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin b/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin new file mode 100644 index 0000000..9a92905 --- /dev/null +++ b/giraph-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinPlugin @@ -0,0 +1 @@ +org.apache.tinkerpop.gremlin.giraph.jsr223.GiraphGremlinPlugin \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java ---------------------------------------------------------------------- diff --git a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java b/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java deleted file mode 100644 index bb1da85..0000000 --- a/giraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/giraph/process/computer/groovy/GiraphGraphComputerGroovyProcessIntegrateTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tinkerpop.gremlin.giraph.process.computer.groovy; - -import org.apache.tinkerpop.gremlin.GraphProviderClass; -import org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphHadoopGraphProvider; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; -import org.apache.tinkerpop.gremlin.process.GroovyProcessComputerSuite; -import org.junit.runner.RunWith; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -@RunWith(GroovyProcessComputerSuite.class) -@GraphProviderClass(provider = GiraphHadoopGraphProvider.class, graph = HadoopGraph.class) -public class GiraphGraphComputerGroovyProcessIntegrateTest { -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml ---------------------------------------------------------------------- diff --git a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml index d1f91ef..5da82d7 100644 --- a/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml +++ b/gremlin-archetype/gremlin-archetype-server/src/main/resources/archetype-resources/src/test/resources/gremlin-server.yaml @@ -23,13 +23,11 @@ scriptEvaluationTimeout: 30000 serializedResponseTimeout: 30000 graphs: { graph: conf/tinkergraph-empty.properties} -plugins: - - tinkerpop.tinkergraph scriptEngines: { gremlin-groovy: { - imports: [java.lang.Math], - staticImports: [java.lang.Math.PI], - scripts: [scripts/generate-modern.groovy]}} + plugins: { org.apache.tinkerpop.gremlin.server.jsr223.GremlinServerGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.tinkergraph.jsr223.TinkerGraphGremlinPlugin: {}, + org.apache.tinkerpop.gremlin.jsr223.ScriptFileGremlinPlugin: {files: [scripts/generate-modern.groovy]}}}} serializers: - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { useMapperFromGraph: graph }} # application/vnd.gremlin-v1.0+gryo - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoMessageSerializerV1d0, config: { serializeResultToString: true }} # application/vnd.gremlin-v1.0+gryo-stringd http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/gremlin-groovy/src/test/groovy/org/apache/tinkerpop/gremlin/groovy/util/SugarTestHelper.groovy ---------------------------------------------------------------------- diff --git a/gremlin-groovy/src/test/groovy/org/apache/tinkerpop/gremlin/groovy/util/SugarTestHelper.groovy b/gremlin-groovy/src/test/groovy/org/apache/tinkerpop/gremlin/groovy/util/SugarTestHelper.groovy index cf2db11..3f4df07 100644 --- a/gremlin-groovy/src/test/groovy/org/apache/tinkerpop/gremlin/groovy/util/SugarTestHelper.groovy +++ b/gremlin-groovy/src/test/groovy/org/apache/tinkerpop/gremlin/groovy/util/SugarTestHelper.groovy @@ -18,7 +18,6 @@ */ package org.apache.tinkerpop.gremlin.groovy.util -import org.apache.tinkerpop.gremlin.GraphProvider import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.DefaultGraphTraversal import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java deleted file mode 100644 index ca446ef..0000000 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPlugin.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tinkerpop.gremlin.hadoop.groovy.plugin; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; -import org.apache.tinkerpop.gremlin.groovy.plugin.AbstractGremlinPlugin; -import org.apache.tinkerpop.gremlin.groovy.plugin.IllegalEnvironmentException; -import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; -import org.apache.tinkerpop.gremlin.groovy.plugin.PluginInitializationException; -import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor; -import org.apache.tinkerpop.gremlin.hadoop.Constants; -import org.apache.tinkerpop.gremlin.hadoop.process.computer.mapreduce.MapReduceGraphComputer; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopConfiguration; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorage; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONInputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.gryo.GryoInputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.io.script.ScriptInputFormat; -import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil; - -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - * @deprecated As of release 3.2.4, replaced by {@link org.apache.tinkerpop.gremlin.hadoop.jsr223.HadoopGremlinPlugin}. - */ -@Deprecated -public final class HadoopGremlinPlugin extends AbstractGremlinPlugin { - - protected static String NAME = "tinkerpop.hadoop"; - - protected static final Set<String> IMPORTS = new HashSet<String>() {{ - add("import org.apache.hadoop.hdfs.*"); - add("import org.apache.hadoop.conf.*"); - add("import org.apache.hadoop.fs.*"); - add("import org.apache.hadoop.util.*"); - add("import org.apache.hadoop.io.*"); - add("import org.apache.hadoop.io.compress.*"); - add("import org.apache.hadoop.mapreduce.lib.input.*"); - add("import org.apache.hadoop.mapreduce.lib.output.*"); - add("import org.apache.log4j.*"); - add(IMPORT_SPACE + Constants.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + HadoopConfiguration.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + ConfUtil.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + VertexWritable.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + GryoInputFormat.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + GraphSONInputFormat.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + ScriptInputFormat.class.getPackage().getName() + DOT_STAR); - add(IMPORT_SPACE + SequenceFileInputFormat.class.getCanonicalName()); - //// - add(IMPORT_SPACE + MapReduceGraphComputer.class.getPackage().getName() + DOT_STAR); - }}; - - @Override - public String getName() { - return NAME; - } - - @Override - public void afterPluginTo(final PluginAcceptor pluginAcceptor) throws PluginInitializationException, IllegalEnvironmentException { - pluginAcceptor.addImports(IMPORTS); - try { - pluginAcceptor.addBinding("hdfs", FileSystemStorage.open(FileSystem.get(new Configuration()))); - pluginAcceptor.addBinding("fs", FileSystemStorage.open(FileSystem.getLocal(new Configuration()))); - if (null == System.getenv(Constants.HADOOP_GREMLIN_LIBS)) - HadoopGraph.LOGGER.warn("Be sure to set the environmental variable: " + Constants.HADOOP_GREMLIN_LIBS); - else - HadoopGraph.LOGGER.info(Constants.HADOOP_GREMLIN_LIBS + " is set to: " + System.getenv(Constants.HADOOP_GREMLIN_LIBS)); - } catch (final Exception e) { - throw new PluginInitializationException(e.getMessage(), e); - } - } - - @Override - public boolean requireRestart() { - return true; - } - - @Override - public Optional<RemoteAcceptor> remoteAcceptor() { - return Optional.of(new HadoopRemoteAcceptor(this.shell)); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopRemoteAcceptor.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopRemoteAcceptor.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopRemoteAcceptor.java deleted file mode 100644 index acae442..0000000 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopRemoteAcceptor.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tinkerpop.gremlin.hadoop.groovy.plugin; - -import org.apache.tinkerpop.gremlin.groovy.loaders.SugarLoader; -import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor; -import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteException; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; -import org.apache.tinkerpop.gremlin.process.computer.ComputerResult; -import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram; -import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ComputerResultStep; -import org.apache.tinkerpop.gremlin.process.traversal.Traversal; -import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource; -import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep; -import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy; -import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal; -import org.codehaus.groovy.tools.shell.Groovysh; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - * @deprecated As of release 3.2.4, replaced by {@link org.apache.tinkerpop.gremlin.hadoop.jsr223.HadoopRemoteAcceptor}. - */ -@Deprecated -public final class HadoopRemoteAcceptor implements RemoteAcceptor { - - private static final String USE_SUGAR = "useSugar"; - private static final String USE_TRAVERSAL_SOURCE = "useTraversalSource"; - private static final String SPACE = " "; - - private HadoopGraph hadoopGraph; - private Groovysh shell; - private boolean useSugar = false; - private TraversalSource traversalSource; - - public HadoopRemoteAcceptor(final Groovysh shell) { - this.shell = shell; - } - - @Override - public Object connect(final List<String> args) throws RemoteException { - if (args.size() != 1 && args.size() != 2) { - throw new IllegalArgumentException("Usage: :remote connect " + HadoopGremlinPlugin.NAME + " <variable name of graph> <optional variable name of traversal source>"); - } - this.hadoopGraph = (HadoopGraph) this.shell.getInterp().getContext().getVariable(args.get(0)); - if (args.size() == 2) - this.traversalSource = ((TraversalSource) this.shell.getInterp().getContext().getVariable(args.get(1))); - else - this.traversalSource = this.hadoopGraph.traversal(); - /// - final HashMap<String, Object> configuration = new HashMap<>(); - configuration.put(USE_SUGAR, this.useSugar); - configuration.put(USE_TRAVERSAL_SOURCE, this.traversalSource); - return Collections.unmodifiableMap(configuration); - } - - @Override - public Object configure(final List<String> args) throws RemoteException { - for (int i = 0; i < args.size(); i = i + 2) { - if (args.get(i).equals(USE_SUGAR)) - this.useSugar = Boolean.valueOf(args.get(i + 1)); - else if (args.get(i).equals(USE_TRAVERSAL_SOURCE)) { - this.traversalSource = ((TraversalSource) this.shell.getInterp().getContext().getVariable(args.get(i + 1))); - } else - throw new IllegalArgumentException("The provided configuration is unknown: " + args.get(i) + ":" + args.get(i + 1)); - } - /// - final HashMap<String, Object> configuration = new HashMap<>(); - configuration.put(USE_SUGAR, this.useSugar); - configuration.put(USE_TRAVERSAL_SOURCE, this.traversalSource); - return Collections.unmodifiableMap(configuration); - } - - @Override - public Object submit(final List<String> args) throws RemoteException { - try { - String script = RemoteAcceptor.getScript(String.join(SPACE, args), this.shell); - if (this.useSugar) - script = SugarLoader.class.getCanonicalName() + ".load()\n" + script; - final TraversalVertexProgram program = TraversalVertexProgram.build().traversal(this.traversalSource, "gremlin-groovy", script).create(this.hadoopGraph); - final ComputerResult computerResult = VertexProgramStrategy.getComputer(this.traversalSource.getStrategies()).get().apply(this.hadoopGraph).program(program).submit().get(); - this.shell.getInterp().getContext().setVariable(RESULT, computerResult); - /// - final Traversal.Admin<ComputerResult, ?> traversal = new DefaultTraversal<>(computerResult.graph()); - traversal.addStep(new ComputerResultStep<>(traversal)); - traversal.addStart(traversal.getTraverserGenerator().generate(computerResult, EmptyStep.instance(), 1l)); - return traversal; - } catch (final Exception e) { - throw new RemoteException(e); - } - } - - @Override - public boolean allowRemoteConsole() { - return true; - } - - @Override - public void close() throws IOException { - this.hadoopGraph.close(); - } -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java index 344fc9d..fdcfef2 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java @@ -84,41 +84,6 @@ import java.util.stream.Stream; reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.", computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"}) @Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_matchXa_hasXname_GarciaX__a_0writtenBy_b__a_0sungBy_bX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_matchXa_knows_b__c_knows_bX", - reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.", - computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_matchXa_created_b__c_created_bX_selectXa_b_cX_byXnameX", - reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.", - computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_out_asXcX_matchXb_knows_a__c_created_eX_selectXcX", - reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.", - computers = {"org.apache.tinkerpop.gremlin.giraph.process.computer.GiraphGraphComputer"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_matchXa_0sungBy_b__a_0sungBy_c__b_writtenBy_d__c_writtenBy_e__d_hasXname_George_HarisonX__e_hasXname_Bob_MarleyXX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"org.apache.tinkerpop.gremlin.spark.process.computer.SparkGraphComputer"}) // this is a nasty long test, just do it once in Java MatchTest -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_d__c_sungBy_d__d_hasXname_GarciaXX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$Traversals", - method = "g_V_matchXa_0sungBy_b__a_0writtenBy_c__b_writtenBy_dX_whereXc_sungBy_dX_whereXd_hasXname_GarciaXX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest$Traversals", method = "g_V_both_both_count", reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", @@ -139,26 +104,6 @@ import java.util.stream.Stream; reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", computers = {"ALL"}) @Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals", - method = "g_V_both_both_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals", - method = "g_V_repeatXoutX_timesX3X_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals", - method = "g_V_repeatXoutX_timesX8X_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyCountTest$Traversals", - method = "g_V_repeatXoutX_timesX5X_asXaX_outXwrittenByX_asXbX_selectXa_bX_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.ProfileTest$Traversals", method = "grateful_V_out_out_profile", reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.") @@ -167,14 +112,6 @@ import java.util.stream.Stream; method = "grateful_V_out_out_profileXmetricsX", reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.") @Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyProfileTest$Traversals", - method = "grateful_V_out_out_profile", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.") -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyProfileTest$Traversals", - method = "grateful_V_out_out_profileXmetricsX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.") -@Graph.OptOut( test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroupTest", method = "g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX", reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", @@ -205,36 +142,6 @@ import java.util.stream.Stream; reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", computers = {"ALL"}) @Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals", - method = "g_V_hasLabelXsongX_groupXaX_byXnameX_byXproperties_groupCount_byXlabelXX_out_capXaX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals", - method = "g_V_outXfollowedByX_group_byXsongTypeX_byXbothE_group_byXlabelX_byXweight_sumXX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals", - method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTest$Traversals", - method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTestV3d0$Traversals", - method = "g_V_repeatXbothXfollowedByXX_timesX2X_group_byXsongTypeX_byXcountX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyGroupTestV3d0$Traversals", - method = "g_V_repeatXbothXfollowedByXX_timesX2X_groupXaX_byXsongTypeX_byXcountX_capXaX", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( test = "org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest", method = "shouldStartAndEndWorkersForVertexProgramAndMapReduce", reason = "Spark executes map and combine in a lazy fashion and thus, fails the blocking aspect of this test", @@ -258,16 +165,6 @@ import java.util.stream.Stream; method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$CountMatchTraversals", - method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) -@Graph.OptOut( - test = "org.apache.tinkerpop.gremlin.process.traversal.step.map.GroovyMatchTest$GreedyMatchTraversals", - method = "g_V_matchXa_followedBy_count_isXgtX10XX_b__a_0followedBy_count_isXgtX10XX_bX_count", - reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.", - computers = {"ALL"}) public final class HadoopGraph implements Graph { public static final Logger LOGGER = LoggerFactory.getLogger(HadoopGraph.class); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java index d445fcd..29272e4 100644 --- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java +++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/script/ScriptRecordWriter.java @@ -24,10 +24,9 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.RecordWriter; import org.apache.hadoop.mapreduce.TaskAttemptContext; -import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider; -import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider; -import org.apache.tinkerpop.gremlin.groovy.jsr223.GremlinGroovyScriptEngine; import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable; +import org.apache.tinkerpop.gremlin.jsr223.CachedGremlinScriptEngineManager; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineManager; import javax.script.Bindings; import javax.script.ScriptEngine; @@ -49,6 +48,7 @@ public final class ScriptRecordWriter extends RecordWriter<NullWritable, VertexW private final static String UTF8 = "UTF-8"; private final static byte[] NEWLINE; private final DataOutputStream out; + private final static GremlinScriptEngineManager manager = new CachedGremlinScriptEngineManager(); private final ScriptEngine engine; static { @@ -62,8 +62,7 @@ public final class ScriptRecordWriter extends RecordWriter<NullWritable, VertexW public ScriptRecordWriter(final DataOutputStream out, final TaskAttemptContext context) throws IOException { this.out = out; final Configuration configuration = context.getConfiguration(); - this.engine = new GremlinGroovyScriptEngine((CompilerCustomizerProvider) new DefaultImportCustomizerProvider()); - //this.engine = ScriptEngineCache.get(configuration.get(SCRIPT_ENGINE, ScriptEngineCache.DEFAULT_SCRIPT_ENGINE)); + this.engine = manager.getEngineByName(configuration.get(SCRIPT_ENGINE, "gremlin-groovy")); final FileSystem fs = FileSystem.get(configuration); try { this.engine.eval(new InputStreamReader(fs.open(new Path(configuration.get(SCRIPT_FILE))))); @@ -90,7 +89,7 @@ public final class ScriptRecordWriter extends RecordWriter<NullWritable, VertexW } @Override - public synchronized void close(TaskAttemptContext context) throws IOException { + public synchronized void close(final TaskAttemptContext context) throws IOException { this.out.close(); } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin b/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin deleted file mode 100644 index 841eeb6..0000000 --- a/hadoop-gremlin/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin +++ /dev/null @@ -1 +0,0 @@ -org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPlugin \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java index 0834bb5..2c51524 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGraphProvider.java @@ -38,6 +38,7 @@ import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoResourceAccess; import org.apache.tinkerpop.gremlin.structure.io.script.ScriptResourceAccess; import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -56,7 +57,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider { private boolean graphSONInput = false; public static Map<String, String> PATHS = new HashMap<>(); - private static final Set<Class> IMPLEMENTATION = new HashSet<Class>() {{ + public static final Set<Class> IMPLEMENTATION = Collections.unmodifiableSet(new HashSet<Class>() {{ add(HadoopEdge.class); add(HadoopElement.class); add(HadoopGraph.class); @@ -70,7 +71,7 @@ public class HadoopGraphProvider extends AbstractGraphProvider { add(ComputerGraph.ComputerVertexProperty.class); add(ComputerGraph.ComputerAdjacentVertex.class); add(ComputerGraph.ComputerProperty.class); - }}; + }}); static { try { http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java index 89aef80..3f7009f 100644 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/HadoopGremlinSuite.java @@ -20,7 +20,7 @@ package org.apache.tinkerpop.gremlin.hadoop; import org.apache.tinkerpop.gremlin.AbstractGremlinSuite; -import org.apache.tinkerpop.gremlin.hadoop.groovy.plugin.HadoopGremlinPluginCheck; +import org.apache.tinkerpop.gremlin.hadoop.jsr223.HadoopGremlinPluginCheck; import org.apache.tinkerpop.gremlin.hadoop.structure.io.FileSystemStorageCheck; import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; import org.junit.runners.model.InitializationError; http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java deleted file mode 100644 index 5cdad56..0000000 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/groovy/plugin/HadoopGremlinPluginCheck.java +++ /dev/null @@ -1,395 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.tinkerpop.gremlin.hadoop.groovy.plugin; - -import org.apache.tinkerpop.gremlin.AbstractGremlinTest; -import org.apache.tinkerpop.gremlin.LoadGraphWith; -import org.apache.tinkerpop.gremlin.TestHelper; -import org.apache.tinkerpop.gremlin.groovy.loaders.GremlinLoader; -import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin; -import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; -import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor; -import org.apache.tinkerpop.gremlin.groovy.util.TestableConsolePluginAcceptor; -import org.apache.tinkerpop.gremlin.hadoop.Constants; -import org.apache.tinkerpop.gremlin.hadoop.HadoopGremlinSuite; -import org.apache.tinkerpop.gremlin.process.traversal.Traversal; -import org.apache.tinkerpop.gremlin.util.Gremlin; -import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; -import org.codehaus.groovy.tools.shell.Groovysh; -import org.codehaus.groovy.tools.shell.IO; -import org.junit.Before; -import org.junit.Test; - -import javax.script.ScriptException; -import javax.tools.JavaCompiler; -import javax.tools.SimpleJavaFileObject; -import javax.tools.ToolProvider; -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.net.URI; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.jar.Attributes; -import java.util.jar.JarEntry; -import java.util.jar.JarOutputStream; -import java.util.jar.Manifest; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * This is an test that is mean to be used in the context of the {@link HadoopGremlinSuite} and shouldn't be - * executed on its own. - * - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -public class HadoopGremlinPluginCheck extends AbstractGremlinTest { - // *********************** - // This test will be removed as the old "plugin" infrastructure was deprecated in 3.2.4. need to rework the - // tests a bi for this to see what the model is for validating the jsr223 plugins. note that the code from - // gremlin-groovy-test, specifically TestableConsolePluginAcceptor, has been copied to the bottom of this - // file for reference - // *********************** - - - /** - @Before - public void setupTest() { - try { - this.console = new TestableConsolePluginAcceptor(); - final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin(); - plugin.pluginTo(this.console); - this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get(); - } catch (final Exception e) { - throw new IllegalStateException(e.getMessage(), e); - } - } - - /////////////////// - - private HadoopRemoteAcceptor remote; - private TestableConsolePluginAcceptor console; - - @Test - @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void shouldSupportRemoteTraversal() throws Exception { - this.console.addBinding("graph", this.graph); - this.console.addBinding("g", this.g); - this.remote.connect(Arrays.asList("graph", "g")); - // - Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()")); - assertEquals(6L, traversal.next()); - assertFalse(traversal.hasNext()); - assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); - } - - @Test - @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void shouldSupportRemoteSugarTraversal() throws Exception { - SugarTestHelper.clearRegistry(this.graphProvider); - this.console.addBinding("graph", this.graph); - this.console.addBinding("g", this.g); - // - this.remote.connect(Arrays.asList("graph", "g")); - try { - this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum")); - fail("Should not allow sugar usage"); - } catch (final Exception e) { - // this is good - } - // - this.remote.configure(Arrays.asList("useSugar", "true")); - this.remote.connect(Arrays.asList("graph", "g")); - Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum")); - assertEquals(28l, traversal.next()); - assertFalse(traversal.hasNext()); - assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); - } - - @Test - @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void shouldSupportRemoteGroupTraversal() throws Exception { - SugarTestHelper.clearRegistry(this.graphProvider); - GremlinLoader.load(); - this.console.addBinding("graph", this.graph); - this.console.addBinding("g", this.g); - this.remote.connect(Arrays.asList("graph")); - // - this.remote.connect(Arrays.asList("graph", "g")); - Traversal<?, Map<String, List<String>>> traversal = (Traversal<?, Map<String, List<String>>>) this.remote.submit(Arrays.asList("g.V().out().group().by{it.value('name')[1]}.by('name')")); - Map<String, List<String>> map = traversal.next(); - assertEquals(3, map.size()); - assertEquals(1, map.get("a").size()); - assertEquals("vadas", map.get("a").get(0)); - assertEquals(1, map.get("i").size()); - assertEquals("ripple", map.get("i").get(0)); - assertEquals(4, map.get("o").size()); - assertTrue(map.get("o").contains("josh")); - assertTrue(map.get("o").contains("lop")); - assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); - // - traversal = (Traversal<?, Map<String, List<String>>>) this.remote.submit(Arrays.asList("g.V().out().group().by(label).by{it.value('name')[1]}")); - map = traversal.next(); - assertEquals(2, map.size()); - assertEquals(4, map.get("software").size()); - assertTrue(map.get("software").contains("o")); - assertTrue(map.get("software").contains("i")); - assertEquals(2, map.get("person").size()); - assertTrue(map.get("person").contains("o")); - assertTrue(map.get("person").contains("a")); - assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); - } - - - @Test - @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void shouldSupportHDFSMethods() throws Exception { - List<String> ls = (List<String>) this.console.eval("hdfs.ls()"); - for (final String line : ls) { - assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x")); - assertEquals(" ", line.substring(9, 10)); - } - ls = (List<String>) this.console.eval("fs.ls()"); - for (final String line : ls) { - assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x")); - assertEquals(" ", line.substring(9, 10)); - } - } - - @Test - @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void shouldGracefullyHandleBadGremlinHadoopLibs() throws Exception { - System.setProperty(Constants.HADOOP_GREMLIN_LIBS, TestHelper.makeTestDataDirectory(HadoopGremlinPluginCheck.class, "shouldGracefullyHandleBadGremlinHadoopLibs")); - this.graph.configuration().setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true); - this.console.addBinding("graph", this.graph); - this.console.addBinding("g", this.g); - this.remote.connect(Arrays.asList("graph", "g")); - Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V()")); - assertEquals(6, IteratorUtils.count(traversal)); - assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); - } - - @Test - @LoadGraphWith(LoadGraphWith.GraphData.MODERN) - public void shouldSupportVariousFileSystemsInGremlinHadoopLibs() throws Exception { - - // The whole point of this test is to verify that HADOOP_GREMLIN_LIBS may contain paths with or without - // a file system scheme prefix and that either path is properly handled. If all jar files, that were specified - // in HADOOP_GREMLIN_LIBS, are found in the GraphComputers temporary directory after using the GraphComputer - // (by submitting a traversal), the test is considered to be successful. - // - // The traversal will likely never fail, since both - Spark and Giraph - run in the same JVM during tests. This - // is unfortunate as it doesn't allow us to verify that GraphComputers load jars properly in a distributed - // environment. The test would fail in a distributed environment, IF loading the jars specified in - // HADOOP_GREMLIN_LIBS wouldn't work. That is because we generate new jar files on the fly that are definitely - // not part of any existing directory or the current classpath. - - final String testDataDirectory = TestHelper.makeTestDataDirectory(HadoopGremlinPluginCheck.class, "shouldHandleLocalGremlinHadoopLibs"); - final File jarFile1 = createJarFile(testDataDirectory + File.separator + "1", "Greeter1"); - final File jarFile2 = createJarFile(testDataDirectory + File.separator + "2", "Greeter2"); - final String graphComputerJarTargetBasePath = System.getProperty("java.io.tmpdir") + File.separator + - "hadoop-gremlin-" + Gremlin.version() + "-libs" + File.separator; - final File graphComputerJarTargetPath1 = new File(graphComputerJarTargetBasePath + "1" + File.separator + "Greeter1.jar"); - final File graphComputerJarTargetPath2 = new File(graphComputerJarTargetBasePath + "2" + File.separator + "Greeter2.jar"); - - for (final boolean withScheme : Arrays.asList(false, true)) { - - Stream<String> hadoopGremlinLibs = Arrays.asList(jarFile1, jarFile2).stream().map(f -> f.getParentFile().getAbsolutePath()); - if (withScheme) { - hadoopGremlinLibs = hadoopGremlinLibs.map(path -> "file://" + path); - } - System.setProperty(Constants.HADOOP_GREMLIN_LIBS, String.join(File.pathSeparator, hadoopGremlinLibs.collect(Collectors.toList()))); - - this.graph.configuration().setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true); - this.console.addBinding("graph", this.graph); - this.console.addBinding("g", this.g); - this.remote.connect(Arrays.asList("graph", "g")); - - Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList( - "ClassLoader.getSystemClassLoader().addURL('" + jarFile1.toURI().toURL() + "'.toURL());", - "ClassLoader.getSystemClassLoader().addURL('" + jarFile2.toURI().toURL() + "'.toURL());", - "g.V().choose(hasLabel('person'), " + - "values('name').map {Class.forName('Greeter1').hello(it.get())}, " + - "values('name').map {Class.forName('Greeter2').hello(it.get())})")); - - final List<String> expectedMessages = Arrays.asList("marko", "josh", "peter", "vadas").stream(). - map(name -> "Greeter1 says: Hello " + name + "!").collect(Collectors.toList()); - - expectedMessages.addAll(Arrays.asList("lop", "ripple").stream(). - map(name -> "Greeter2 says: Hello " + name + "!").collect(Collectors.toList())); - - while (traversal.hasNext()) { - final String message = (String) traversal.next(); - assertTrue(expectedMessages.remove(message)); - } - - assertEquals(0, expectedMessages.size()); - } - - assertTrue(graphComputerJarTargetPath1.exists()); - assertTrue(graphComputerJarTargetPath2.exists()); - - assert graphComputerJarTargetPath1.delete(); - assert graphComputerJarTargetPath2.delete(); - } - - private File createJarFile(final String directory, final String className) throws IOException { - - new File(directory).mkdirs(); - - final File classFile = new File(directory + File.separator + className + ".class"); - final File jarFile = new File(directory + File.separator + className + ".jar"); - - jarFile.deleteOnExit(); - - final JavaStringObject source = new JavaStringObject(className, - "public class " + className + " {\n" + - " public static String hello(final String name) {\n" + - " return \"" + className + " says: Hello \" + name + \"!\";\n" + - " }\n" + - "}"); - - final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - final List<String> options = Arrays.asList( - "-d", classFile.getParentFile().getAbsolutePath() - ); - assert compiler.getTask(null, null, null, options, null, Collections.singletonList(source)).call(); - - final Manifest manifest = new Manifest(); - manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); - - try (final JarOutputStream target = new JarOutputStream(new FileOutputStream(jarFile), manifest)) { - final JarEntry entry = new JarEntry(classFile.getName()); - entry.setTime(classFile.lastModified()); - target.putNextEntry(entry); - try (final FileInputStream fis = new FileInputStream(classFile); - final BufferedInputStream in = new BufferedInputStream(fis)) { - final byte buffer[] = new byte[1024]; - while (true) { - final int count = in.read(buffer); - if (count < 0) break; - target.write(buffer, 0, count); - } - } - target.closeEntry(); - } - - assert classFile.delete(); - - return jarFile; - } - - private static class JavaStringObject extends SimpleJavaFileObject { - - private final String code; - - JavaStringObject(final String className, final String code) { - super(URI.create("string:///" + className.replace(".", "/") + Kind.SOURCE.extension), Kind.SOURCE); - this.code = code; - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { - return code; - } - } - **/ - - - ///////////////////////////////////////////////////////////////////////// - ///////////////////////////////////////////////////////////////////////// - ///////////////////////////////////////////////////////////////////////// - - -//package org.apache.tinkerpop.gremlin.groovy.util; -// -//import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin; -//import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; -//import org.codehaus.groovy.tools.shell.Groovysh; -//import org.codehaus.groovy.tools.shell.IO; -// -//import javax.script.ScriptException; -//import java.io.IOException; -//import java.io.OutputStream; -//import java.util.Collections; -//import java.util.HashMap; -//import java.util.Map; -//import java.util.Set; -// -// /** -// * @author Marko A. Rodriguez (http://markorodriguez.com) -// */ -// public class TestableConsolePluginAcceptor implements PluginAcceptor { -// -// public static final String ENVIRONMENT_NAME = "console"; -// public static final String ENVIRONMENT_SHELL = "ConsolePluginAcceptor.shell"; -// public static final String ENVIRONMENT_IO = "ConsolePluginAcceptor.io"; -// -// private Groovysh shell = new Groovysh(new IO(System.in, new OutputStream() { -// @Override -// public void write(int b) throws IOException { -// -// } -// }, System.err)); -// -// @Override -// public void addImports(final Set<String> importStatements) { -// importStatements.forEach(this.shell::execute); -// } -// -// @Override -// public void addBinding(final String key, final Object val) { -// this.shell.getInterp().getContext().setVariable(key, val); -// } -// -// @Override -// public Map<String, Object> getBindings() { -// return Collections.unmodifiableMap(this.shell.getInterp().getContext().getVariables()); -// } -// -// @Override -// public Object eval(final String script) throws ScriptException { -// return this.shell.execute(script); -// } -// -// @Override -// public Map<String, Object> environment() { -// final Map<String, Object> env = new HashMap<>(); -// env.put(GremlinPlugin.ENVIRONMENT, ENVIRONMENT_NAME); -// env.put(ENVIRONMENT_IO, this.shell.getIo()); -// env.put(ENVIRONMENT_SHELL, this.shell); -// return env; -// } -// -// } -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/jsr223/HadoopGremlinPluginCheck.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/jsr223/HadoopGremlinPluginCheck.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/jsr223/HadoopGremlinPluginCheck.java new file mode 100644 index 0000000..baa96da --- /dev/null +++ b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/jsr223/HadoopGremlinPluginCheck.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.tinkerpop.gremlin.hadoop.jsr223; + +import org.apache.tinkerpop.gremlin.AbstractGremlinTest; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.groovy.loaders.GremlinLoader; +import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin; +import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; +import org.apache.tinkerpop.gremlin.groovy.plugin.RemoteAcceptor; +import org.apache.tinkerpop.gremlin.hadoop.Constants; +import org.apache.tinkerpop.gremlin.hadoop.HadoopGremlinSuite; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.util.Gremlin; +import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; +import org.codehaus.groovy.tools.shell.Groovysh; +import org.codehaus.groovy.tools.shell.IO; +import org.junit.Before; +import org.junit.Test; + +import javax.script.ScriptException; +import javax.tools.JavaCompiler; +import javax.tools.SimpleJavaFileObject; +import javax.tools.ToolProvider; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.net.URI; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * This is an test that is mean to be used in the context of the {@link HadoopGremlinSuite} and shouldn't be + * executed on its own. + * + * @author Marko A. Rodriguez (http://markorodriguez.com) + */ +public class HadoopGremlinPluginCheck extends AbstractGremlinTest { + // *********************** + // This test will be removed as the old "plugin" infrastructure was deprecated in 3.2.4. need to rework the + // tests a bi for this to see what the model is for validating the jsr223 plugins. note that the code from + // gremlin-groovy-test, specifically TestableConsolePluginAcceptor, has been copied to the bottom of this + // file for reference + // *********************** + @Test + public void nothing() { + // this is just to stop junit failures until this test gets figured out + } + + /** + @Before + public void setupTest() { + try { + this.console = new TestableConsolePluginAcceptor(); + final HadoopGremlinPlugin plugin = new HadoopGremlinPlugin(); + plugin.pluginTo(this.console); + this.remote = (HadoopRemoteAcceptor) plugin.remoteAcceptor().get(); + } catch (final Exception e) { + throw new IllegalStateException(e.getMessage(), e); + } + } + + /////////////////// + + private HadoopRemoteAcceptor remote; + private TestableConsolePluginAcceptor console; + + @Test + @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + public void shouldSupportRemoteTraversal() throws Exception { + this.console.addBinding("graph", this.graph); + this.console.addBinding("g", this.g); + this.remote.connect(Arrays.asList("graph", "g")); + // + Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V().count()")); + assertEquals(6L, traversal.next()); + assertFalse(traversal.hasNext()); + assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); + } + + @Test + @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + public void shouldSupportRemoteSugarTraversal() throws Exception { + SugarTestHelper.clearRegistry(this.graphProvider); + this.console.addBinding("graph", this.graph); + this.console.addBinding("g", this.g); + // + this.remote.connect(Arrays.asList("graph", "g")); + try { + this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum")); + fail("Should not allow sugar usage"); + } catch (final Exception e) { + // this is good + } + // + this.remote.configure(Arrays.asList("useSugar", "true")); + this.remote.connect(Arrays.asList("graph", "g")); + Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V.name.map{it.length()}.sum")); + assertEquals(28l, traversal.next()); + assertFalse(traversal.hasNext()); + assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); + } + + @Test + @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + public void shouldSupportRemoteGroupTraversal() throws Exception { + SugarTestHelper.clearRegistry(this.graphProvider); + GremlinLoader.load(); + this.console.addBinding("graph", this.graph); + this.console.addBinding("g", this.g); + this.remote.connect(Arrays.asList("graph")); + // + this.remote.connect(Arrays.asList("graph", "g")); + Traversal<?, Map<String, List<String>>> traversal = (Traversal<?, Map<String, List<String>>>) this.remote.submit(Arrays.asList("g.V().out().group().by{it.value('name')[1]}.by('name')")); + Map<String, List<String>> map = traversal.next(); + assertEquals(3, map.size()); + assertEquals(1, map.get("a").size()); + assertEquals("vadas", map.get("a").get(0)); + assertEquals(1, map.get("i").size()); + assertEquals("ripple", map.get("i").get(0)); + assertEquals(4, map.get("o").size()); + assertTrue(map.get("o").contains("josh")); + assertTrue(map.get("o").contains("lop")); + assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); + // + traversal = (Traversal<?, Map<String, List<String>>>) this.remote.submit(Arrays.asList("g.V().out().group().by(label).by{it.value('name')[1]}")); + map = traversal.next(); + assertEquals(2, map.size()); + assertEquals(4, map.get("software").size()); + assertTrue(map.get("software").contains("o")); + assertTrue(map.get("software").contains("i")); + assertEquals(2, map.get("person").size()); + assertTrue(map.get("person").contains("o")); + assertTrue(map.get("person").contains("a")); + assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); + } + + + @Test + @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + public void shouldSupportHDFSMethods() throws Exception { + List<String> ls = (List<String>) this.console.eval("hdfs.ls()"); + for (final String line : ls) { + assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x")); + assertEquals(" ", line.substring(9, 10)); + } + ls = (List<String>) this.console.eval("fs.ls()"); + for (final String line : ls) { + assertTrue(line.startsWith("-") || line.startsWith("r") || line.startsWith("w") || line.startsWith("x")); + assertEquals(" ", line.substring(9, 10)); + } + } + + @Test + @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + public void shouldGracefullyHandleBadGremlinHadoopLibs() throws Exception { + System.setProperty(Constants.HADOOP_GREMLIN_LIBS, TestHelper.makeTestDataDirectory(HadoopGremlinPluginCheck.class, "shouldGracefullyHandleBadGremlinHadoopLibs")); + this.graph.configuration().setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true); + this.console.addBinding("graph", this.graph); + this.console.addBinding("g", this.g); + this.remote.connect(Arrays.asList("graph", "g")); + Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList("g.V()")); + assertEquals(6, IteratorUtils.count(traversal)); + assertNotNull(this.console.getBindings().get(RemoteAcceptor.RESULT)); + } + + @Test + @LoadGraphWith(LoadGraphWith.GraphData.MODERN) + public void shouldSupportVariousFileSystemsInGremlinHadoopLibs() throws Exception { + + // The whole point of this test is to verify that HADOOP_GREMLIN_LIBS may contain paths with or without + // a file system scheme prefix and that either path is properly handled. If all jar files, that were specified + // in HADOOP_GREMLIN_LIBS, are found in the GraphComputers temporary directory after using the GraphComputer + // (by submitting a traversal), the test is considered to be successful. + // + // The traversal will likely never fail, since both - Spark and Giraph - run in the same JVM during tests. This + // is unfortunate as it doesn't allow us to verify that GraphComputers load jars properly in a distributed + // environment. The test would fail in a distributed environment, IF loading the jars specified in + // HADOOP_GREMLIN_LIBS wouldn't work. That is because we generate new jar files on the fly that are definitely + // not part of any existing directory or the current classpath. + + final String testDataDirectory = TestHelper.makeTestDataDirectory(HadoopGremlinPluginCheck.class, "shouldHandleLocalGremlinHadoopLibs"); + final File jarFile1 = createJarFile(testDataDirectory + File.separator + "1", "Greeter1"); + final File jarFile2 = createJarFile(testDataDirectory + File.separator + "2", "Greeter2"); + final String graphComputerJarTargetBasePath = System.getProperty("java.io.tmpdir") + File.separator + + "hadoop-gremlin-" + Gremlin.version() + "-libs" + File.separator; + final File graphComputerJarTargetPath1 = new File(graphComputerJarTargetBasePath + "1" + File.separator + "Greeter1.jar"); + final File graphComputerJarTargetPath2 = new File(graphComputerJarTargetBasePath + "2" + File.separator + "Greeter2.jar"); + + for (final boolean withScheme : Arrays.asList(false, true)) { + + Stream<String> hadoopGremlinLibs = Arrays.asList(jarFile1, jarFile2).stream().map(f -> f.getParentFile().getAbsolutePath()); + if (withScheme) { + hadoopGremlinLibs = hadoopGremlinLibs.map(path -> "file://" + path); + } + System.setProperty(Constants.HADOOP_GREMLIN_LIBS, String.join(File.pathSeparator, hadoopGremlinLibs.collect(Collectors.toList()))); + + this.graph.configuration().setProperty(Constants.GREMLIN_HADOOP_JARS_IN_DISTRIBUTED_CACHE, true); + this.console.addBinding("graph", this.graph); + this.console.addBinding("g", this.g); + this.remote.connect(Arrays.asList("graph", "g")); + + Traversal<?, ?> traversal = (Traversal<?, ?>) this.remote.submit(Arrays.asList( + "ClassLoader.getSystemClassLoader().addURL('" + jarFile1.toURI().toURL() + "'.toURL());", + "ClassLoader.getSystemClassLoader().addURL('" + jarFile2.toURI().toURL() + "'.toURL());", + "g.V().choose(hasLabel('person'), " + + "values('name').map {Class.forName('Greeter1').hello(it.get())}, " + + "values('name').map {Class.forName('Greeter2').hello(it.get())})")); + + final List<String> expectedMessages = Arrays.asList("marko", "josh", "peter", "vadas").stream(). + map(name -> "Greeter1 says: Hello " + name + "!").collect(Collectors.toList()); + + expectedMessages.addAll(Arrays.asList("lop", "ripple").stream(). + map(name -> "Greeter2 says: Hello " + name + "!").collect(Collectors.toList())); + + while (traversal.hasNext()) { + final String message = (String) traversal.next(); + assertTrue(expectedMessages.remove(message)); + } + + assertEquals(0, expectedMessages.size()); + } + + assertTrue(graphComputerJarTargetPath1.exists()); + assertTrue(graphComputerJarTargetPath2.exists()); + + assert graphComputerJarTargetPath1.delete(); + assert graphComputerJarTargetPath2.delete(); + } + + private File createJarFile(final String directory, final String className) throws IOException { + + new File(directory).mkdirs(); + + final File classFile = new File(directory + File.separator + className + ".class"); + final File jarFile = new File(directory + File.separator + className + ".jar"); + + jarFile.deleteOnExit(); + + final JavaStringObject source = new JavaStringObject(className, + "public class " + className + " {\n" + + " public static String hello(final String name) {\n" + + " return \"" + className + " says: Hello \" + name + \"!\";\n" + + " }\n" + + "}"); + + final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); + final List<String> options = Arrays.asList( + "-d", classFile.getParentFile().getAbsolutePath() + ); + assert compiler.getTask(null, null, null, options, null, Collections.singletonList(source)).call(); + + final Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + + try (final JarOutputStream target = new JarOutputStream(new FileOutputStream(jarFile), manifest)) { + final JarEntry entry = new JarEntry(classFile.getName()); + entry.setTime(classFile.lastModified()); + target.putNextEntry(entry); + try (final FileInputStream fis = new FileInputStream(classFile); + final BufferedInputStream in = new BufferedInputStream(fis)) { + final byte buffer[] = new byte[1024]; + while (true) { + final int count = in.read(buffer); + if (count < 0) break; + target.write(buffer, 0, count); + } + } + target.closeEntry(); + } + + assert classFile.delete(); + + return jarFile; + } + + private static class JavaStringObject extends SimpleJavaFileObject { + + private final String code; + + JavaStringObject(final String className, final String code) { + super(URI.create("string:///" + className.replace(".", "/") + Kind.SOURCE.extension), Kind.SOURCE); + this.code = code; + } + + @Override + public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOException { + return code; + } + } + **/ + + + ///////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////// + ///////////////////////////////////////////////////////////////////////// + + +//package org.apache.tinkerpop.gremlin.groovy.util; +// +//import org.apache.tinkerpop.gremlin.groovy.plugin.GremlinPlugin; +//import org.apache.tinkerpop.gremlin.groovy.plugin.PluginAcceptor; +//import org.codehaus.groovy.tools.shell.Groovysh; +//import org.codehaus.groovy.tools.shell.IO; +// +//import javax.script.ScriptException; +//import java.io.IOException; +//import java.io.OutputStream; +//import java.util.Collections; +//import java.util.HashMap; +//import java.util.Map; +//import java.util.Set; +// +// /** +// * @author Marko A. Rodriguez (http://markorodriguez.com) +// */ +// public class TestableConsolePluginAcceptor implements PluginAcceptor { +// +// public static final String ENVIRONMENT_NAME = "console"; +// public static final String ENVIRONMENT_SHELL = "ConsolePluginAcceptor.shell"; +// public static final String ENVIRONMENT_IO = "ConsolePluginAcceptor.io"; +// +// private Groovysh shell = new Groovysh(new IO(System.in, new OutputStream() { +// @Override +// public void write(int b) throws IOException { +// +// } +// }, System.err)); +// +// @Override +// public void addImports(final Set<String> importStatements) { +// importStatements.forEach(this.shell::execute); +// } +// +// @Override +// public void addBinding(final String key, final Object val) { +// this.shell.getInterp().getContext().setVariable(key, val); +// } +// +// @Override +// public Map<String, Object> getBindings() { +// return Collections.unmodifiableMap(this.shell.getInterp().getContext().getVariables()); +// } +// +// @Override +// public Object eval(final String script) throws ScriptException { +// return this.shell.execute(script); +// } +// +// @Override +// public Map<String, Object> environment() { +// final Map<String, Object> env = new HashMap<>(); +// env.put(GremlinPlugin.ENVIRONMENT, ENVIRONMENT_NAME); +// env.put(ENVIRONMENT_IO, this.shell.getIo()); +// env.put(ENVIRONMENT_SHELL, this.shell); +// return env; +// } +// +// } +} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/groovy/HadoopGraphGroovyProcessStandardTest.java ---------------------------------------------------------------------- diff --git a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/groovy/HadoopGraphGroovyProcessStandardTest.java b/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/groovy/HadoopGraphGroovyProcessStandardTest.java deleted file mode 100644 index 10399be..0000000 --- a/hadoop-gremlin/src/test/java/org/apache/tinkerpop/gremlin/hadoop/process/groovy/HadoopGraphGroovyProcessStandardTest.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.tinkerpop.gremlin.hadoop.process.groovy; - -import org.apache.tinkerpop.gremlin.GraphProviderClass; -import org.apache.tinkerpop.gremlin.hadoop.HadoopGraphProvider; -import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph; -import org.apache.tinkerpop.gremlin.process.GroovyProcessStandardSuite; -import org.junit.runner.RunWith; - -/** - * @author Marko A. Rodriguez (http://markorodriguez.com) - */ -@RunWith(GroovyProcessStandardSuite.class) -@GraphProviderClass(provider = HadoopGraphProvider.class, graph = HadoopGraph.class) -public class HadoopGraphGroovyProcessStandardTest { -} http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 29d9091..6d004db 100644 --- a/pom.xml +++ b/pom.xml @@ -995,7 +995,7 @@ limitations under the License. <overview>${basedir}/docs/javadoc/overview.html</overview> <quiet>true</quiet> <sourcepath> - giraph-gremlin/src/main/java:gremlin-core/src/main/java:gremlin-driver/src/main/java:gremlin-groovy/src/main/java:gremlin-groovy-test/src/main/java:gremlin-python/src/main/java:gremlin-server/src/main/java:gremlin-test/src/main/java:hadoop-gremlin/src/main/java:neo4j-gremlin/src/main/java:spark-gremlin/src/main/java:tinkergraph-gremlin/src/main/java + giraph-gremlin/src/main/java:gremlin-core/src/main/java:gremlin-driver/src/main/java:gremlin-groovy/src/main/java:gremlin-python/src/main/java:gremlin-server/src/main/java:gremlin-test/src/main/java:hadoop-gremlin/src/main/java:neo4j-gremlin/src/main/java:spark-gremlin/src/main/java:tinkergraph-gremlin/src/main/java </sourcepath> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/41bdcd76/spark-gremlin/pom.xml ---------------------------------------------------------------------- diff --git a/spark-gremlin/pom.xml b/spark-gremlin/pom.xml index a08fdfd..3ff90a4 100644 --- a/spark-gremlin/pom.xml +++ b/spark-gremlin/pom.xml @@ -47,11 +47,6 @@ </dependency> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>gremlin-groovy</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.tinkerpop</groupId> <artifactId>hadoop-gremlin</artifactId> <version>${project.version}</version> <exclusions> @@ -345,12 +340,6 @@ </dependency> <dependency> <groupId>org.apache.tinkerpop</groupId> - <artifactId>gremlin-groovy-test</artifactId> - <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.tinkerpop</groupId> <artifactId>tinkergraph-gremlin</artifactId> <version>${project.version}</version> <scope>test</scope>
