Improved memory usage when deserializing TinkerGraph from GraphSON Streamed vertices/edges rather than reading them all into memory at once. CTR
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/7ea38751 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/7ea38751 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/7ea38751 Branch: refs/heads/tp32-glv Commit: 7ea38751871f6eddd4c4bcfe8ac82300a08f5a7d Parents: afa3432 Author: Stephen Mallette <[email protected]> Authored: Tue May 23 11:01:55 2017 -0400 Committer: Stephen Mallette <[email protected]> Committed: Tue May 23 11:01:55 2017 -0400 ---------------------------------------------------------------------- CHANGELOG.asciidoc | 1 + .../structure/TinkerIoRegistryV2d0.java | 40 +++++++++----------- 2 files changed, 18 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7ea38751/CHANGELOG.asciidoc ---------------------------------------------------------------------- diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 7057aac..be8059d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima TinkerPop 3.2.5 (Release Date: NOT OFFICIALLY RELEASED YET) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Reduced memory usage for TinkerGraph deserialization in GraphSON by streaming vertices and edges. * Now using Groovy `[...]` map notation in `GroovyTranslator` instead of `new LinkedHashMap(){{ }}`. * Maintained type information on `Traversal.promise()`. * Propagated exception to `Future` instead of calling thread in `RemoteConnection`. http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/7ea38751/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java index 727931c..394216b 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerIoRegistryV2d0.java @@ -26,7 +26,6 @@ import org.apache.tinkerpop.gremlin.structure.io.AbstractIoRegistry; import org.apache.tinkerpop.gremlin.structure.io.IoRegistry; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONUtil; import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoIo; import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoReader; @@ -37,12 +36,11 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex; import org.apache.tinkerpop.shaded.jackson.core.JsonGenerator; import org.apache.tinkerpop.shaded.jackson.core.JsonParser; import org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException; +import org.apache.tinkerpop.shaded.jackson.core.JsonToken; import org.apache.tinkerpop.shaded.jackson.databind.DeserializationContext; import org.apache.tinkerpop.shaded.jackson.databind.SerializerProvider; import org.apache.tinkerpop.shaded.jackson.databind.deser.std.StdDeserializer; -import org.apache.tinkerpop.shaded.jackson.databind.jsontype.TypeSerializer; import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdScalarSerializer; -import org.apache.tinkerpop.shaded.jackson.databind.ser.std.StdSerializer; import org.apache.tinkerpop.shaded.kryo.Kryo; import org.apache.tinkerpop.shaded.kryo.Serializer; import org.apache.tinkerpop.shaded.kryo.io.Input; @@ -53,7 +51,6 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.util.HashMap; import java.util.Iterator; -import java.util.List; import java.util.Map; /** @@ -201,26 +198,23 @@ public final class TinkerIoRegistryV2d0 extends AbstractIoRegistry { conf.setProperty("gremlin.tinkergraph.defaultVertexPropertyCardinality", "list"); final TinkerGraph graph = TinkerGraph.open(conf); - final List<? extends Edge> edges; - final List<? extends Vertex> vertices; - - jsonParser.nextToken(); - final Map<String, Object> graphData = deserializationContext.readValue(jsonParser, Map.class); - vertices = (List<DetachedVertex>) graphData.get(GraphSONTokens.VERTICES); - edges = (List<DetachedEdge>) graphData.get(GraphSONTokens.EDGES); - - - vertices.forEach(e -> { - if (e instanceof DetachedVertex) { - ((DetachedVertex)e).attach(Attachable.Method.getOrCreate(graph)); + while (jsonParser.nextToken() != JsonToken.END_OBJECT) { + if (jsonParser.getCurrentName().equals("vertices")) { + while (jsonParser.nextToken() != JsonToken.END_ARRAY) { + if (jsonParser.currentToken() == JsonToken.START_OBJECT) { + final DetachedVertex v = (DetachedVertex) deserializationContext.readValue(jsonParser, Vertex.class); + v.attach(Attachable.Method.getOrCreate(graph)); + } + } + } else if (jsonParser.getCurrentName().equals("edges")) { + while (jsonParser.nextToken() != JsonToken.END_ARRAY) { + if (jsonParser.currentToken() == JsonToken.START_OBJECT) { + final DetachedEdge e = (DetachedEdge) deserializationContext.readValue(jsonParser, Edge.class); + e.attach(Attachable.Method.getOrCreate(graph)); + } + } } - }); - - edges.forEach(e -> { - if (e instanceof DetachedEdge) { - ((DetachedEdge) e).attach(Attachable.Method.getOrCreate(graph)); - } - }); + } return graph; }
