discovering various synchronization bottlenecks in TinkerGraphComputer. Also, realized some dumb things I was doing in TraversalVertexProgram. Its crazy, for this benchmark that @dkuppitz and i have, if I don't touch vertex.properties that are compute keys: millisecond return times. If I do, seconds return times.... Need to figure out how to partition TinkerGraphView... Perhaps thread local..dah.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/8d961285 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/8d961285 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/8d961285 Branch: refs/heads/TINKERPOP-1565 Commit: 8d961285f687107ffc5c1d8a3bca7c78ea833adf Parents: cef1979 Author: Marko A. Rodriguez <[email protected]> Authored: Wed Jan 4 10:32:21 2017 -0700 Committer: Marko A. Rodriguez <[email protected]> Committed: Thu Jan 5 17:00:05 2017 -0700 ---------------------------------------------------------------------- .../traversal/TraversalVertexProgram.java | 16 +++++++--- .../computer/traversal/WorkerExecutor.java | 2 +- .../computer/TinkerGraphComputerView.java | 32 +++++++++----------- .../groovy/TinkerGraphGroovyPlayTest.groovy | 2 +- 4 files changed, 27 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java index 1a54721..b82e265 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/TraversalVertexProgram.java @@ -32,7 +32,6 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexComputeKey; import org.apache.tinkerpop.gremlin.process.computer.VertexProgram; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.ComputerResultStep; import org.apache.tinkerpop.gremlin.process.computer.traversal.step.map.TraversalVertexProgramStep; -import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy; import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy; import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder; import org.apache.tinkerpop.gremlin.process.computer.util.SingleMessenger; @@ -51,6 +50,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileSideEffectStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.ProfileStep; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy; import org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy; import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet; import org.apache.tinkerpop.gremlin.process.traversal.util.DefaultTraversal; @@ -251,8 +251,14 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet< ((Barrier) this.traversalMatrix.getStepById(stepId)).done(); } // define halted traversers - final TraverserSet<Object> haltedTraversers = vertex.<TraverserSet<Object>>property(HALTED_TRAVERSERS).orElse(new TraverserSet<>()); - vertex.property(VertexProperty.Cardinality.single, HALTED_TRAVERSERS, haltedTraversers); + final VertexProperty<TraverserSet<Object>> property = vertex.property(HALTED_TRAVERSERS); + final TraverserSet<Object> haltedTraversers; + if (property.isPresent()) { + haltedTraversers = property.value(); + } else { + haltedTraversers = new TraverserSet<>(); + vertex.property(VertexProperty.Cardinality.single, HALTED_TRAVERSERS, haltedTraversers); + } ////////////////// if (memory.isInitialIteration()) { // ITERATION 1 final TraverserSet<Object> activeTraversers = new TraverserSet<>(); @@ -282,9 +288,9 @@ public final class TraversalVertexProgram implements VertexProgram<TraverserSet< activeTraversers.add((Traverser.Admin) traverser); }); } - memory.add(VOTE_TO_HALT, activeTraversers.isEmpty() || WorkerExecutor.execute(vertex, new SingleMessenger<>(messenger, activeTraversers), this.traversalMatrix, memory, this.returnHaltedTraversers, this.haltedTraverserStrategy)); + memory.add(VOTE_TO_HALT, activeTraversers.isEmpty() || WorkerExecutor.execute(vertex, new SingleMessenger<>(messenger, activeTraversers), this.traversalMatrix, memory, this.returnHaltedTraversers, haltedTraversers, this.haltedTraverserStrategy)); } else // ITERATION 1+ - memory.add(VOTE_TO_HALT, WorkerExecutor.execute(vertex, messenger, this.traversalMatrix, memory, this.returnHaltedTraversers, this.haltedTraverserStrategy)); + memory.add(VOTE_TO_HALT, WorkerExecutor.execute(vertex, messenger, this.traversalMatrix, memory, this.returnHaltedTraversers, haltedTraversers, this.haltedTraverserStrategy)); // save space by not having an empty halted traversers property if (this.returnHaltedTraversers || haltedTraversers.isEmpty()) vertex.<TraverserSet>property(HALTED_TRAVERSERS).remove(); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java index 2571e7b..e6e73d0 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java @@ -59,10 +59,10 @@ final class WorkerExecutor { final TraversalMatrix<?, ?> traversalMatrix, final Memory memory, final boolean returnHaltedTraversers, + final TraverserSet<Object> haltedTraversers, final HaltedTraverserStrategy haltedTraverserStrategy) { final TraversalSideEffects traversalSideEffects = traversalMatrix.getTraversal().getSideEffects(); final AtomicBoolean voteToHalt = new AtomicBoolean(true); - final TraverserSet<Object> haltedTraversers = vertex.value(TraversalVertexProgram.HALTED_TRAVERSERS); final TraverserSet<Object> activeTraversers = new TraverserSet<>(); final TraverserSet<Object> toProcessTraversers = new TraverserSet<>(); http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java index 43090fe..43998fb 100644 --- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java +++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java @@ -43,8 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * @author Marko A. Rodriguez (http://markorodriguez.com) @@ -104,11 +102,14 @@ public final class TinkerGraphComputerView { } public List<Property> getProperties(final TinkerVertex vertex) { - final Stream<Property> a = TinkerHelper.getProperties(vertex).values().stream().flatMap(list -> list.stream()); - final Stream<Property> b = this.computeProperties.containsKey(vertex) ? - this.computeProperties.get(vertex).values().stream().flatMap(list -> list.stream()) : - Stream.empty(); - return Stream.concat(a, b).collect(Collectors.toList()); + final List<Property> list = new ArrayList<>(); + for (final List<VertexProperty> properties : TinkerHelper.getProperties(vertex).values()) { + list.addAll(properties); + } + for (final List<VertexProperty<?>> properties : this.computeProperties.getOrDefault(vertex, Collections.emptyMap()).values()) { + list.addAll(properties); + } + return list; } public void removeProperty(final TinkerVertex vertex, final String key, final VertexProperty property) { @@ -131,8 +132,9 @@ public final class TinkerGraphComputerView { // remove all transient properties from the vertices for (final VertexComputeKey computeKey : this.computeKeys.values()) { if (computeKey.isTransient()) { - final List<VertexProperty<?>> toRemove = this.computeProperties.values().stream().flatMap(map -> map.getOrDefault(computeKey.getKey(), Collections.emptyList()).stream()).collect(Collectors.toList()); - toRemove.forEach(VertexProperty::remove); + for (final Map<String, List<VertexProperty<?>>> properties : this.computeProperties.values()) { + properties.remove(computeKey.getKey()); + } } } } @@ -211,22 +213,16 @@ public final class TinkerGraphComputerView { } private void addValue(final Vertex vertex, final String key, final VertexProperty property) { - final Map<String, List<VertexProperty<?>>> elementProperties = this.computeProperties.computeIfAbsent(vertex, k -> new ConcurrentHashMap<>()); + final Map<String, List<VertexProperty<?>>> elementProperties = this.computeProperties.computeIfAbsent(vertex, k -> new HashMap<>()); elementProperties.compute(key, (k, v) -> { - if (null == v) v = Collections.synchronizedList(new ArrayList<>()); + if (null == v) v = new ArrayList<>(); v.add(property); return v; }); } private void removeValue(final Vertex vertex, final String key, final VertexProperty property) { - this.computeProperties.computeIfPresent(vertex, (k, v) -> { - v.computeIfPresent(key, (k1, v1) -> { - v1.remove(property); - return v1; - }); - return v; - }); + this.computeProperties.<List<Map<String, VertexProperty<?>>>>getOrDefault(vertex, Collections.emptyMap()).get(key).remove(property); } private List<VertexProperty<?>> getValue(final Vertex vertex, final String key) { http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/8d961285/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy ---------------------------------------------------------------------- diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy index d277977..a736f3d 100644 --- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy +++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/process/groovy/TinkerGraphGroovyPlayTest.groovy @@ -40,7 +40,7 @@ class TinkerGraphGroovyPlayTest { def a = graph.traversal().withComputer(Computer.compute()) def r = new Random(123) - (1..1725403).each { + (1..1000000).each { def vid = ["a", "b", "c", "d"].collectEntries { [it, r.nextInt() % 400000] } graph.addVertex(T.id, vid) }; []
