TINKERPOP-1698 Changed format of property writes Write properties with streams instead of pulling them into a List in memory
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/155e50d8 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/155e50d8 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/155e50d8 Branch: refs/heads/TINKERPOP-1682 Commit: 155e50d8097caa817a68d385ad6192a2df0e36f9 Parents: 0a7f7f6 Author: Stephen Mallette <sp...@genoprime.com> Authored: Tue Jun 27 08:46:35 2017 -0400 Committer: Stephen Mallette <sp...@genoprime.com> Committed: Mon Jul 10 14:14:03 2017 -0400 ---------------------------------------------------------------------- CHANGELOG.asciidoc | 1 + .../structure/io/gryo/GryoSerializersV3d0.java | 75 ++++++++++--------- .../structure/io/gryo/kryoshim/InputShim.java | 2 + .../structure/io/gryo/kryoshim/OutputShim.java | 2 + .../kryoshim/shaded/ShadedInputAdapter.java | 5 ++ .../kryoshim/shaded/ShadedOutputAdapter.java | 7 +- .../structure/io/gryo/_3_3_0/edge-v3d0.kryo | Bin 41 -> 39 bytes .../io/gryo/_3_3_0/standardresult-v3d0.kryo | Bin 235 -> 229 bytes .../io/gryo/_3_3_0/traversalmetrics-v1d0.kryo | Bin 294 -> 306 bytes .../io/gryo/_3_3_0/traverser-v3d0.kryo | Bin 216 -> 210 bytes .../structure/io/gryo/_3_3_0/tree-v3d0.kryo | Bin 293 -> 277 bytes .../structure/io/gryo/_3_3_0/vertex-v3d0.kryo | Bin 207 -> 201 bytes .../io/gryo/_3_3_0/vertexproperty-v3d0.kryo | Bin 26 -> 23 bytes .../kryoshim/unshaded/UnshadedInputAdapter.java | 5 ++ .../unshaded/UnshadedOutputAdapter.java | 5 ++ 15 files changed, 66 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/CHANGELOG.asciidoc ---------------------------------------------------------------------- diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 2fe7094..8c08d5d 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima TinkerPop 3.3.0 (Release Date: NOT OFFICIALLY RELEASED YET) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* Established the Gryo 3.0 format. * `GryoVersion` now includes a default `ClassResolver` to supply to the `GryoMapper`. * `GryoClassResolver` renamed to `GryoClassResolverV1d0` which has an abstract class that for providers to extend in `AbstractGryoClassResolver`. * Removed previously deprecated `TraversalSource.Builder` class. http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializersV3d0.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializersV3d0.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializersV3d0.java index 20e13c4..82fa178 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializersV3d0.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoSerializersV3d0.java @@ -31,10 +31,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics; import org.apache.tinkerpop.gremlin.process.traversal.util.OrP; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMetrics; import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Property; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; -import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.InputShim; import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.KryoShim; import org.apache.tinkerpop.gremlin.structure.io.gryo.kryoshim.OutputShim; @@ -51,6 +51,7 @@ import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; @@ -86,14 +87,18 @@ public final class GryoSerializersV3d0 { } kryo.writeClassAndObject(output, edge.outVertex().id()); - output.writeString(edge.outVertex().label()); - - final List<Property> properties = IteratorUtils.asList(edge.properties()); - output.writeInt(properties.size()); - properties.forEach(p -> { - output.writeString(p.key()); - kryo.writeClassAndObject(output, p.value()); - }); + + // temporary try/catch perhaps? need this to get SparkSingleIterationStrategyTest to work. Trying to grab + // the label of the adjacent vertex ends in error if there's a StarEdge in the ComputerGraph$ComputerEdge. + // maybe this gets fixed up when TINKERPOP-1592 is in play. hopefully this serializer will get better + // with that + try { + output.writeString(edge.outVertex().label()); + } catch (Exception ex) { + output.writeString(Vertex.DEFAULT_LABEL); + } + + writeElementProperties(kryo, output, edge); } @Override @@ -112,8 +117,7 @@ public final class GryoSerializersV3d0 { outV.setLabel(input.readString()); builder.setOutV(outV.create()); - final int numberOfProperties = input.readInt(); - for (int ix = 0; ix < numberOfProperties; ix ++) { + while(input.readBoolean()) { builder.addProperty(new DetachedProperty<>(input.readString(), kryo.readClassAndObject(input))); } @@ -130,24 +134,22 @@ public final class GryoSerializersV3d0 { kryo.writeClassAndObject(output, vertex.id()); output.writeString(vertex.label()); - final List<VertexProperty> properties = IteratorUtils.asList(vertex.properties()); - output.writeInt(properties.size()); - properties.forEach(vp -> { + final Iterator<? extends VertexProperty> properties = vertex.properties(); + output.writeBoolean(properties.hasNext()); + while (properties.hasNext()) { + final VertexProperty vp = properties.next(); kryo.writeClassAndObject(output, vp.id()); output.writeString(vp.label()); kryo.writeClassAndObject(output, vp.value()); if (vp instanceof DetachedVertexProperty || (vertex.graph().features().vertex().supportsMetaProperties())) { - final List<Property> metaProperties = IteratorUtils.asList(vp.properties()); - output.writeInt(metaProperties.size()); - metaProperties.forEach(p -> { - output.writeString(p.key()); - kryo.writeClassAndObject(output, p.value()); - }); + writeElementProperties(kryo, output, vp); } else { - output.writeInt(0); + output.writeBoolean(false); } - }); + + output.writeBoolean(properties.hasNext()); + } } @Override @@ -156,15 +158,13 @@ public final class GryoSerializersV3d0 { builder.setId(kryo.readClassAndObject(input)); builder.setLabel(input.readString()); - final int numberOfProperties = input.readInt(); - for (int ix = 0; ix < numberOfProperties; ix ++) { + while(input.readBoolean()) { final DetachedVertexProperty.Builder vpBuilder = DetachedVertexProperty.build(); vpBuilder.setId(kryo.readClassAndObject(input)); vpBuilder.setLabel(input.readString()); vpBuilder.setValue(kryo.readClassAndObject(input)); - final int numberOfMetaProperties = input.readInt(); - for (int iy = 0; iy < numberOfMetaProperties; iy ++) { + while(input.readBoolean()) { vpBuilder.addProperty(new DetachedProperty<>(input.readString(), kryo.readClassAndObject(input))); } @@ -207,14 +207,9 @@ public final class GryoSerializersV3d0 { output.writeString(vertexProperty.element().label()); if (vertexProperty instanceof DetachedVertexProperty || (vertexProperty.graph().features().vertex().supportsMetaProperties())) { - final List<Property> metaProperties = IteratorUtils.asList(vertexProperty.properties()); - output.writeInt(metaProperties.size()); - metaProperties.forEach(p -> { - output.writeString(p.key()); - kryo.writeClassAndObject(output, p.value()); - }); + writeElementProperties(kryo, output, vertexProperty); } else { - output.writeInt(0); + output.writeBoolean(false); } } @@ -230,8 +225,7 @@ public final class GryoSerializersV3d0 { host.setLabel(input.readString()); vpBuilder.setV(host.create()); - final int numberOfMetaProperties = input.readInt(); - for (int iy = 0; iy < numberOfMetaProperties; iy ++) { + while(input.readBoolean()) { vpBuilder.addProperty(new DetachedProperty<>(input.readString(), kryo.readClassAndObject(input))); } @@ -465,4 +459,15 @@ public final class GryoSerializersV3d0 { return m; } } + + private static void writeElementProperties(final KryoShim kryo, final OutputShim output, final Element element) { + final Iterator<? extends Property> properties = element.properties(); + output.writeBoolean(properties.hasNext()); + while (properties.hasNext()) { + final Property p = properties.next(); + output.writeString(p.key()); + kryo.writeClassAndObject(output, p.value()); + output.writeBoolean(properties.hasNext()); + } + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java index d6ba06d..21bfd1f 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/InputShim.java @@ -36,5 +36,7 @@ public interface InputShim { public int readInt(); + public boolean readBoolean(); + public double readDouble(); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java index 85cba71..abfe8bf 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/OutputShim.java @@ -38,5 +38,7 @@ public interface OutputShim { public void writeShort(final int s); + public void writeBoolean(final boolean b); + public void flush(); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java index 22a45ee..aa7741b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedInputAdapter.java @@ -67,4 +67,9 @@ public class ShadedInputAdapter implements InputShim { public double readDouble() { return shadedInput.readDouble(); } + + @Override + public boolean readBoolean() { + return shadedInput.readBoolean(); + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java index 5ecc0fc..1b1697b 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/kryoshim/shaded/ShadedOutputAdapter.java @@ -60,11 +60,16 @@ public class ShadedOutputAdapter implements OutputShim { } @Override - public void writeShort(int s) { + public void writeShort(final int s) { shadedOutput.writeShort(s); } @Override + public void writeBoolean(final boolean b) { + shadedOutput.writeBoolean(b); + } + + @Override public void flush() { shadedOutput.flush(); } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/edge-v3d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/edge-v3d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/edge-v3d0.kryo index b58bff1..6dfbefb 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/edge-v3d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/edge-v3d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/standardresult-v3d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/standardresult-v3d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/standardresult-v3d0.kryo index b1a2558..06c5d8a 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/standardresult-v3d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/standardresult-v3d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traversalmetrics-v1d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traversalmetrics-v1d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traversalmetrics-v1d0.kryo index 1aee9e9..2d02c43 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traversalmetrics-v1d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traversalmetrics-v1d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traverser-v3d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traverser-v3d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traverser-v3d0.kryo index 9ec67de..7c1fb0d 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traverser-v3d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/traverser-v3d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/tree-v3d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/tree-v3d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/tree-v3d0.kryo index 94d1357..d5a9dd9 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/tree-v3d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/tree-v3d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertex-v3d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertex-v3d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertex-v3d0.kryo index eb46417..4464039 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertex-v3d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertex-v3d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertexproperty-v3d0.kryo ---------------------------------------------------------------------- diff --git a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertexproperty-v3d0.kryo b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertexproperty-v3d0.kryo index 3139408..95f8be0 100644 Binary files a/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertexproperty-v3d0.kryo and b/gremlin-tools/gremlin-io-test/src/test/resources/org/apache/tinkerpop/gremlin/structure/io/gryo/_3_3_0/vertexproperty-v3d0.kryo differ http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java index f8177e3..8e1ea05 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedInputAdapter.java @@ -67,4 +67,9 @@ public class UnshadedInputAdapter implements InputShim { public double readDouble() { return unshadedInput.readDouble(); } + + @Override + public boolean readBoolean() { + return unshadedInput.readBoolean(); + } } http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/155e50d8/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java ---------------------------------------------------------------------- diff --git a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java index 3799ee4..7f53bd1 100644 --- a/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java +++ b/spark-gremlin/src/main/java/org/apache/tinkerpop/gremlin/spark/structure/io/gryo/kryoshim/unshaded/UnshadedOutputAdapter.java @@ -68,6 +68,11 @@ public class UnshadedOutputAdapter implements OutputShim { } @Override + public void writeBoolean(final boolean b) { + unshadedOutput.writeBoolean(b); + } + + @Override public void flush() { unshadedOutput.flush(); }