TINKERPOP-1996 Added support for setting IoRegistries using with() IORegistry instances are important because they feed serializer information to the Reader/Writer instances. Of all the configuration options that one seemed like the most important to make possible using with().
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/ff71c6ab Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/ff71c6ab Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/ff71c6ab Branch: refs/heads/TINKERPOP-1990 Commit: ff71c6abee0b39d7ee95128c3d64906daad96a76 Parents: ae3b149 Author: Stephen Mallette <sp...@genoprime.com> Authored: Thu Jul 19 16:13:57 2018 -0400 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Thu Jul 19 16:13:57 2018 -0400 ---------------------------------------------------------------------- .../tinkerpop/gremlin/process/traversal/IO.java | 7 ++ .../traversal/step/sideEffect/IoStep.java | 75 ++++++++++++-------- .../process/traversal/step/util/Parameters.java | 4 +- .../Process/Traversal/GraphTraversal.cs | 21 ++---- .../src/Gremlin.Net/Process/Traversal/IO.cs | 2 + .../gremlin-javascript/lib/process/traversal.js | 4 ++ .../jython/gremlin_python/process/traversal.py | 2 + .../gremlin/structure/io/util/CustomId.java | 39 ++++++++++ .../step/sideEffect/TinkerGraphIoStepTest.java | 75 ++++++++++++++++++++ 9 files changed, 181 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java index 6668cf1..67b4670 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/IO.java @@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSo import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.GraphReader; import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; @@ -73,4 +74,10 @@ public class IO { * the file extension provided to it. */ public static final String writer = Graph.Hidden.hide("tinkerpop.io.writer"); + + /** + * A key that identifies the fully qualified class names of {@link IoRegistry} instances to use. May be specified + * multiple times (i.e. once for each registry) using the {@link GraphTraversal#with(String, Object)} modulator. + */ + public static final String registry = Graph.Hidden.hide("tinkerpop.io.registry"); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java index 9804333..1d4f40b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/IoStep.java @@ -31,10 +31,13 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.FastNoSuchElementExce import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.io.GraphReader; import org.apache.tinkerpop.gremlin.structure.io.GraphWriter; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLReader; import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLWriter; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONMapper; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONReader; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONWriter; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoWriter; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; @@ -46,6 +49,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.lang.reflect.Method; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; /** * Handles read and write operations into the {@link Graph}. @@ -138,15 +144,19 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting { * extension or simply uses configurations provided by the user on the parameters given to the step. */ private GraphReader constructReader() { - final Object objectOrClass = parameters.get(IO.reader, this::detectReader).get(0); + final Object objectOrClass = parameters.get(IO.reader, this::detectFileType).get(0); if (objectOrClass instanceof GraphReader) return (GraphReader) objectOrClass; else if (objectOrClass instanceof String) { - if (objectOrClass.equals(IO.graphson)) - return GraphSONReader.build().create(); - else if (objectOrClass.equals(IO.gryo)) - return GryoReader.build().create(); - else if (objectOrClass.equals(IO.graphml)) + if (objectOrClass.equals(IO.graphson)) { + final GraphSONMapper.Builder builder = GraphSONMapper.build(); + detectRegistries().forEach(builder::addRegistry); + return GraphSONReader.build().mapper(builder.create()).create(); + } else if (objectOrClass.equals(IO.gryo)){ + final GryoMapper.Builder builder = GryoMapper.build(); + detectRegistries().forEach(builder::addRegistry); + return GryoReader.build().mapper(builder.create()).create(); + } else if (objectOrClass.equals(IO.graphml)) return GraphMLReader.build().create(); else { try { @@ -163,31 +173,24 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting { } } - private GraphReader detectReader() { - if (file.endsWith(".kryo")) - return GryoReader.build().create(); - else if (file.endsWith(".json")) - return GraphSONReader.build().create(); - else if (file.endsWith(".xml")) - return GraphMLReader.build().create(); - else - throw new IllegalStateException("Could not detect the file format - specify the reader explicitly or rename file with a standard extension"); - } - /** * Builds a {@link GraphWriter} instance to use. Attempts to detect the file format to be write using the file * extension or simply uses configurations provided by the user on the parameters given to the step. */ private GraphWriter constructWriter() { - final Object objectOrClass = parameters.get(IO.writer, this::detectWriter).get(0); + final Object objectOrClass = parameters.get(IO.writer, this::detectFileType).get(0); if (objectOrClass instanceof GraphWriter) return (GraphWriter) objectOrClass; else if (objectOrClass instanceof String) { - if (objectOrClass.equals(IO.graphson)) - return GraphSONWriter.build().create(); - else if (objectOrClass.equals(IO.gryo)) - return GryoWriter.build().create(); - else if (objectOrClass.equals(IO.graphml)) + if (objectOrClass.equals(IO.graphson)) { + final GraphSONMapper.Builder builder = GraphSONMapper.build(); + detectRegistries().forEach(builder::addRegistry); + return GraphSONWriter.build().mapper(builder.create()).create(); + } else if (objectOrClass.equals(IO.gryo)){ + final GryoMapper.Builder builder = GryoMapper.build(); + detectRegistries().forEach(builder::addRegistry); + return GryoWriter.build().mapper(builder.create()).create(); + }else if (objectOrClass.equals(IO.graphml)) return GraphMLWriter.build().create(); else { try { @@ -204,21 +207,31 @@ public class IoStep<S> extends AbstractStep<S,S> implements ReadWriting { } } - private GraphWriter detectWriter() { + private String detectFileType() { if (file.endsWith(".kryo")) - return GryoWriter.build().create(); + return IO.gryo; else if (file.endsWith(".json")) - return GraphSONWriter.build().create(); + return IO.graphson; else if (file.endsWith(".xml")) - return GraphMLWriter.build().create(); + return IO.graphml; else throw new IllegalStateException("Could not detect the file format - specify the writer explicitly or rename file with a standard extension"); } - private Configuration getConfFromParameters() { - final Configuration conf = new BaseConfiguration(); - parameters.getRaw().forEach((key, value) -> conf.setProperty(key.toString(), value.get(0))); - return conf; + private List<IoRegistry> detectRegistries() { + final List<Object> k = parameters.get(IO.registry, Collections::emptyList); + return parameters.get(IO.registry, null).stream().map(cn -> { + try { + if (cn instanceof IoRegistry) + return (IoRegistry) cn; + else { + final Class<?> clazz = Class.forName(cn.toString()); + return (IoRegistry) clazz.getMethod("instance").invoke(null); + } + } catch (Exception ex) { + throw new IllegalStateException(ex); + } + }).collect(Collectors.toList()); } @Override http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java index 40d9330..eb57f4b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/util/Parameters.java @@ -100,11 +100,11 @@ public final class Parameters implements Cloneable, Serializable { * Gets the value of a key and if that key isn't present returns the default value from the {@link Supplier}. * * @param key the key to retrieve - * @param defaultValue the default value generator + * @param defaultValue the default value generator which if null will return an empty list */ public <E> List<E> get(final Object key, final Supplier<E> defaultValue) { final List<E> list = (List<E>) this.parameters.get(key); - return (null == list) ? Collections.singletonList(defaultValue.get()) : list; + return (null == list) ? (null == defaultValue ? Collections.emptyList() : Collections.singletonList(defaultValue.get())) : list; } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs ---------------------------------------------------------------------- diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs index 82d72c0..361b246 100644 --- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs +++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/GraphTraversal.cs @@ -1256,20 +1256,20 @@ namespace Gremlin.Net.Process.Traversal } /// <summary> - /// Adds the repeat step to this <see cref="GraphTraversal{SType, EType}" />. + /// Adds the read step to this <see cref="GraphTraversal{SType, EType}" />. /// </summary> - public GraphTraversal<S, E> Repeat (string loopName, ITraversal repeatTraversal) + public GraphTraversal<S, E> Read () { - Bytecode.AddStep("repeat", loopName, repeatTraversal); + Bytecode.AddStep("read"); return Wrap<S, E>(this); } /// <summary> - /// Adds the read step to this <see cref="GraphTraversal{SType, EType}" />. + /// Adds the repeat step to this <see cref="GraphTraversal{SType, EType}" />. /// </summary> - public GraphTraversal<S, E> Read () + public GraphTraversal<S, E> Repeat (string loopName, ITraversal repeatTraversal) { - Bytecode.AddStep("read"); + Bytecode.AddStep("repeat", loopName, repeatTraversal); return Wrap<S, E>(this); } @@ -1713,15 +1713,6 @@ namespace Gremlin.Net.Process.Traversal } /// <summary> - /// Adds the with step to this <see cref="GraphTraversal{SType, EType}" />. - /// </summary> - public GraphTraversal<S, E> With (string key, object value) - { - Bytecode.AddStep("with", key, value); - return Wrap<S, E>(this); - } - - /// <summary> /// Adds the write step to this <see cref="GraphTraversal{SType, EType}" />. /// </summary> public GraphTraversal<S, E> Write () http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs ---------------------------------------------------------------------- diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs index 288f7e3..861b431 100644 --- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs +++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/IO.cs @@ -46,6 +46,8 @@ namespace Gremlin.Net.Process.Traversal public const String reader = "~tinkerpop.io.reader"; + public const String registry = "~tinkerpop.io.registry"; + public const String writer = "~tinkerpop.io.writer"; } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js ---------------------------------------------------------------------- diff --git a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js index 3f69fb1..09aec91 100644 --- a/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js +++ b/gremlin-javascript/src/main/javascript/gremlin-javascript/lib/process/traversal.js @@ -129,6 +129,10 @@ class IO { return "~tinkerpop.io.reader" } + static get registry() { + return "~tinkerpop.io.registry" + } + static get writer() { return "~tinkerpop.io.writer" } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-python/src/main/jython/gremlin_python/process/traversal.py ---------------------------------------------------------------------- diff --git a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py index d9fb4d9..49bb7b1 100644 --- a/gremlin-python/src/main/jython/gremlin_python/process/traversal.py +++ b/gremlin-python/src/main/jython/gremlin_python/process/traversal.py @@ -319,6 +319,8 @@ class IO(object): reader = "~tinkerpop.io.reader" + registry = "~tinkerpop.io.registry" + writer = "~tinkerpop.io.writer" http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java ---------------------------------------------------------------------- diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java index d503ae8..0ab3b90 100644 --- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java +++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/io/util/CustomId.java @@ -18,9 +18,14 @@ */ package org.apache.tinkerpop.gremlin.structure.io.util; +import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry; +import org.apache.tinkerpop.gremlin.structure.io.Io; +import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; import org.apache.tinkerpop.gremlin.structure.io.graphson.AbstractObjectDeserializer; +import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule; +import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo; import org.apache.tinkerpop.shaded.jackson.core.JsonGenerationException; import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator; import org.apache.tinkerpop.shaded.jackson.core.JsonParser; @@ -31,11 +36,13 @@ import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer; import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer; import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer; import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer; +import org.javatuples.Pair; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.UUID; @@ -66,6 +73,24 @@ public class CustomId { } @Override + public boolean equals(final Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + final CustomId customId = (CustomId) o; + + if (!cluster.equals(customId.cluster)) return false; + return elementId.equals(customId.elementId); + } + + @Override + public int hashCode() { + int result = cluster.hashCode(); + result = 31 * result + elementId.hashCode(); + return result; + } + + @Override public String toString() { return cluster + ":" + elementId; } @@ -219,4 +244,18 @@ public class CustomId { return "simple"; } } + + public static class CustomIdIoRegistry extends AbstractIoRegistry { + + private static final CustomIdIoRegistry INSTANCE = new CustomIdIoRegistry(); + + private CustomIdIoRegistry() { + register(GryoIo.class, CustomId.class, null); + register(GraphSONIo.class, null, new CustomIdTinkerPopJacksonModuleV3d0()); + } + + public static CustomIdIoRegistry instance() { + return INSTANCE; + } + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/ff71c6ab/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java new file mode 100644 index 0000000..06c4db8 --- /dev/null +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/traversal/step/sideEffect/TinkerGraphIoStepTest.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.tinkerpop.gremlin.tinkergraph.process.traversal.step.sideEffect; + +import org.apache.tinkerpop.gremlin.TestHelper; +import org.apache.tinkerpop.gremlin.process.traversal.IO; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Graph; +import org.apache.tinkerpop.gremlin.structure.io.util.CustomId; +import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.util.UUID; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +/** + * It was hard to test the {@link IO#registry} configuration as a generic test. Opted to test it as a bit of a + * standalone test with TinkerGraph. + * + * @author Stephen Mallette (http://stephen.genoprime.com) + */ +public class TinkerGraphIoStepTest { + + private Graph graph; + private GraphTraversalSource g; + + @Before + public void setup() { + graph = TinkerGraph.open(); + g = graph.traversal(); + } + + @Test + public void shouldWriteReadWithCustomIoRegistryGryo() throws Exception { + final UUID uuid = UUID.randomUUID(); + g.addV("person").property("name","stephen").property("custom", new CustomId("a", uuid)).iterate(); + + final File file = TestHelper.generateTempFile(TinkerGraphIoStepTest.class, "shouldWriteReadWithCustomIoRegistryGryo", ".kryo"); + g.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.class.getName()).write().iterate(); + + final Graph emptyGraph = TinkerGraph.open(); + final GraphTraversalSource emptyG = emptyGraph.traversal(); + + try { + emptyG.io(file.getAbsolutePath()).read().iterate(); + fail("Can't read without a registry"); + } catch (Exception ignored) { + // do nothing + } + + emptyG.io(file.getAbsolutePath()).with(IO.registry, CustomId.CustomIdIoRegistry.instance()).read().iterate(); + + assertEquals(1, emptyG.V().has("custom", new CustomId("a", uuid)).count().next().intValue()); + } +}