Repository: tinkerpop
Updated Branches:
  refs/heads/master a8da47f14 -> 173d823a9


added GraphComputing.atMaster(boolean) to allow steps (if the want) to know 
whether they are executing locally at master or distributed across workers. 
This fixes a bug in dedup() on OLAP where non-element traversers were not being 
dedup'd correctly.


Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/0dcf6592
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/0dcf6592
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/0dcf6592

Branch: refs/heads/master
Commit: 0dcf65921b6045d734d1d6230309f2a1de4f4ee5
Parents: 193db1c
Author: Marko A. Rodriguez <okramma...@gmail.com>
Authored: Tue Oct 11 08:20:31 2016 -0600
Committer: Marko A. Rodriguez <okramma...@gmail.com>
Committed: Tue Oct 11 08:20:31 2016 -0600

----------------------------------------------------------------------
 CHANGELOG.asciidoc                              |  2 ++
 .../computer/traversal/MasterExecutor.java      |  4 ++-
 .../computer/traversal/WorkerExecutor.java      |  4 ++-
 .../process/traversal/step/GraphComputing.java  | 30 ++++++++++++++++++++
 .../traversal/step/filter/DedupGlobalStep.java  | 12 ++++++--
 .../step/filter/GroovyDedupTest.groovy          |  6 +++-
 .../traversal/step/filter/DedupTest.java        | 30 +++++++++++++++++---
 7 files changed, 79 insertions(+), 9 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 746d38c..fbb6043 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,8 @@ 
image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
 TinkerPop 3.2.3 (Release Date: NOT OFFICIALLY RELEASED YET)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
+* Added `GraphComputing.atMaster(boolean)` to allow steps to know whether they 
are executing at master or distributed at workers.
+* Fixed a bug in OLAP where `DedupGlobalStep` wasn't de-duping local master 
traversers.
 * Added `HasContainerHolder.removeHasContainer()`-method with default 
`UnsupportedOperationException` implementation.
 * `TraversalSource.withComputer()` is simplified to add a 
`VertexProgramStrategy`. Easier for language variants.
 * Fixed a `Set`, `List`, `Map` bug in the various `Translators` where such 
collections were not being internally translated.

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
----------------------------------------------------------------------
diff --git 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
index f81ca14..48e3a57 100644
--- 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
+++ 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
@@ -24,12 +24,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.Path;
 import org.apache.tinkerpop.gremlin.process.traversal.Step;
 import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
 import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
+import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
 import org.apache.tinkerpop.gremlin.process.traversal.step.LocalBarrier;
 import 
org.apache.tinkerpop.gremlin.process.traversal.step.filter.ConnectiveStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep;
 import 
org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep;
 import 
org.apache.tinkerpop.gremlin.process.traversal.step.filter.TailGlobalStep;
-import 
org.apache.tinkerpop.gremlin.process.traversal.step.filter.WherePredicateStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.IdStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.LabelStep;
 import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesStep;
@@ -105,6 +105,7 @@ final class MasterExecutor {
                 else {
                     currentStep = 
traversalMatrix.getStepById(traverser.getStepId());
                     if (!currentStep.getId().equals(previousStep.getId()) && 
!(previousStep instanceof EmptyStep)) {
+                        GraphComputing.atMaster(previousStep, true);
                         while (previousStep.hasNext()) {
                             final Traverser.Admin<Object> result = 
previousStep.next();
                             if (result.isHalted())
@@ -120,6 +121,7 @@ final class MasterExecutor {
                 }
             }
             if (!(currentStep instanceof EmptyStep)) {
+                GraphComputing.atMaster(currentStep, true);
                 while (currentStep.hasNext()) {
                     final Traverser.Admin<Object> traverser = 
currentStep.next();
                     if (traverser.isHalted())

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
----------------------------------------------------------------------
diff --git 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
index 0a3aad2..2571e7b 100644
--- 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
+++ 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
@@ -21,14 +21,15 @@ package 
org.apache.tinkerpop.gremlin.process.computer.traversal;
 import org.apache.tinkerpop.gremlin.process.computer.Memory;
 import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
 import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import 
org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
 import org.apache.tinkerpop.gremlin.process.traversal.Step;
 import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
 import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
 import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
 import org.apache.tinkerpop.gremlin.process.traversal.step.Bypassing;
+import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
 import org.apache.tinkerpop.gremlin.process.traversal.step.LocalBarrier;
 import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
+import 
org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
 import 
org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
 import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMatrix;
 import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -179,6 +180,7 @@ final class WorkerExecutor {
                                   final Memory memory,
                                   final boolean returnHaltedTraversers,
                                   final HaltedTraverserStrategy 
haltedTraverserStrategy) {
+        GraphComputing.atMaster(step, false);
         if (step instanceof Barrier) {
             if (step instanceof Bypassing)
                 ((Bypassing) step).setBypass(true);

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
----------------------------------------------------------------------
diff --git 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
index 0028eb1..bd9fc42 100644
--- 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
+++ 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
@@ -18,6 +18,9 @@
  */
 package org.apache.tinkerpop.gremlin.process.traversal.step;
 
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+
 /**
  * A {@code GraphComputing} step is one that will change its behavior whether 
its on a {@link org.apache.tinkerpop.gremlin.process.computer.GraphComputer} or 
not.
  * {@link 
org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy}
 is responsible for calling the {@link GraphComputing#onGraphComputer()} method.
@@ -32,4 +35,31 @@ public interface GraphComputing {
      */
     public void onGraphComputer();
 
+    /**
+     * Some steps should behave different whether they are executing at the 
master traversal or distributed across the worker traversals.
+     * The default implementation does nothing.
+     *
+     * @param atMaster whether the step is currently executing at master
+     */
+    public default void atMaster(boolean atMaster) {
+
+    }
+
+    public static void atMaster(final Step<?, ?> step, boolean atMaster) {
+        if (step instanceof GraphComputing)
+            ((GraphComputing) step).atMaster(atMaster);
+        if (step instanceof TraversalParent) {
+            for (final Traversal.Admin<?, ?> local : ((TraversalParent) 
step).getLocalChildren()) {
+                for (final Step<?, ?> s : local.getSteps()) {
+                    GraphComputing.atMaster(s, atMaster);
+                }
+            }
+            for (final Traversal.Admin<?, ?> global : ((TraversalParent) 
step).getGlobalChildren()) {
+                for (final Step<?, ?> s : global.getSteps()) {
+                    GraphComputing.atMaster(s, atMaster);
+                }
+            }
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
----------------------------------------------------------------------
diff --git 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
index 8ef6455..220805f 100644
--- 
a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
+++ 
b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
@@ -57,6 +57,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> 
implements Traversal
     private boolean onGraphComputer = false;
     private final Set<String> dedupLabels;
     private Set<String> keepLabels;
+    private boolean executingAtMaster = false;
 
     public DedupGlobalStep(final Traversal.Admin traversal, final String... 
dedupLabels) {
         super(traversal);
@@ -65,7 +66,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> 
implements Traversal
 
     @Override
     protected boolean filter(final Traverser.Admin<S> traverser) {
-        if (this.onGraphComputer) return true;
+        if (this.onGraphComputer && !this.executingAtMaster) return true;
         traverser.setBulk(1);
         if (null == this.dedupLabels) {
             return 
this.duplicateSet.add(TraversalUtil.applyNullable(traverser, 
this.dedupTraversal));
@@ -77,6 +78,11 @@ public final class DedupGlobalStep<S> extends FilterStep<S> 
implements Traversal
     }
 
     @Override
+    public void atMaster(final boolean atMaster) {
+        this.executingAtMaster = atMaster;
+    }
+
+    @Override
     public ElementRequirement getMaxRequirement() {
         return null == this.dedupLabels ? ElementRequirement.ID : 
PathProcessor.super.getMaxRequirement();
     }
@@ -206,5 +212,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> 
implements Traversal
     }
 
     @Override
-    public Set<String> getKeepLabels() { return this.keepLabels; }
+    public Set<String> getKeepLabels() {
+        return this.keepLabels;
+    }
 }

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
----------------------------------------------------------------------
diff --git 
a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
 
b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
index d03f526..a091c6f 100644
--- 
a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
+++ 
b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
@@ -19,7 +19,6 @@
 package org.apache.tinkerpop.gremlin.process.traversal.step.filter
 
 import org.apache.tinkerpop.gremlin.process.traversal.Path
-import org.apache.tinkerpop.gremlin.process.traversal.Scope
 import org.apache.tinkerpop.gremlin.process.traversal.Traversal
 import org.apache.tinkerpop.gremlin.process.traversal.util.ScriptTraversal
 import org.apache.tinkerpop.gremlin.structure.Vertex
@@ -95,5 +94,10 @@ public abstract class GroovyDedupTest {
         public Traversal<Vertex, String> 
get_g_V_both_both_dedup_byXoutE_countX_name() {
             new ScriptTraversal<>(g, "gremlin-groovy", 
"g.V.both.both.dedup.by(outE().count).name")
         }
+
+        @Override
+        public Traversal<Vertex, Long> 
get_g_V_groupCount_selectXvaluesX_unfold_dedup() {
+            new ScriptTraversal<>(g, "gremlin-groovy", 
"g.V.groupCount.select(values).unfold.dedup")
+        }
     }
 }

http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
----------------------------------------------------------------------
diff --git 
a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
 
b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
index fa55c8e..970a976 100644
--- 
a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
+++ 
b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
@@ -32,16 +32,18 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
 import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.both;
 import static 
org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.bothE;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.isIn;
+import static 
org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.select;
+import static org.apache.tinkerpop.gremlin.structure.Column.values;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -81,6 +83,8 @@ public abstract class DedupTest extends 
AbstractGremlinProcessTest {
 
     public abstract Traversal<Vertex, Map<String, String>> 
get_g_V_out_asXxX_in_asXyX_selectXx_yX_byXnameX_fold_dedupXlocal_x_yX_unfold();
 
+    public abstract Traversal<Vertex, Long> 
get_g_V_groupCount_selectXvaluesX_unfold_dedup();
+
     @Test
     @LoadGraphWith(MODERN)
     public void g_V_out_in_valuesXnameX_fold_dedupXlocalX_unfold() {
@@ -269,6 +273,14 @@ public abstract class DedupTest extends 
AbstractGremlinProcessTest {
         assertEquals(4, new HashSet<>(names).size());
     }
 
+    @Test
+    @LoadGraphWith(MODERN)
+    public void g_V_groupCount_selectXvaluesX_unfold_dedup() {
+        final Traversal<Vertex, Long> traversal = 
get_g_V_groupCount_selectXvaluesX_unfold_dedup();
+        printTraversalForm(traversal);
+        checkResults(Collections.singletonList(1L), traversal);
+    }
+
     public static class Traversals extends DedupTest {
         @Override
         public Traversal<Vertex, String> 
get_g_V_out_in_valuesXnameX_fold_dedupXlocalX_unfold() {
@@ -277,7 +289,7 @@ public abstract class DedupTest extends 
AbstractGremlinProcessTest {
 
         @Override
         public Traversal<Vertex, Map<String, String>> 
get_g_V_out_asXxX_in_asXyX_selectXx_yX_byXnameX_fold_dedupXlocal_x_yX_unfold() {
-            return 
g.V().out().as("x").in().as("y").select("x","y").by("name").fold().dedup(Scope.local,"x","y").unfold();
+            return g.V().out().as("x").in().as("y").select("x", 
"y").by("name").fold().dedup(Scope.local, "x", "y").unfold();
         }
 
         @Override
@@ -334,5 +346,15 @@ public abstract class DedupTest extends 
AbstractGremlinProcessTest {
         public Traversal<Vertex, String> 
get_g_V_both_both_dedup_byXoutE_countX_name() {
             return 
g.V().both().both().dedup().by(__.outE().count()).values("name");
         }
+
+        @Override
+        public Traversal<Vertex, Long> 
get_g_V_groupCount_selectXvaluesX_unfold_dedup() {
+            return g.V().groupCount().select(values).<Long>unfold().dedup();
+        }
+
+        /*@Override
+        public Traversal<Vertex,Collection<Vertex>> 
get_g_V_asXaX_repeatXbothX_timesX3X_emit_asXbX_group_byXselectXaXX_byXselectXbX_dedup_order_byXidX_foldX_selectXvaluesX_unfold_dedup()
 {
+            return 
g.V().as("a").repeat(both()).times(3).emit().as("b").group().by(select("a")).by(select("b").dedup().order().by(T.id).fold()).select(values).<Collection<Vertex>>unfold().dedup();
+        }*/
     }
 }

Reply via email to