TINKERPOP-1979 Get math() working on spark properly The Expression class was not serializable and Spark was not happy. Wrapped it up in another class that was and now tests work on spark ok.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/a6e0a2d5 Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/a6e0a2d5 Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/a6e0a2d5 Branch: refs/heads/tp33 Commit: a6e0a2d55c6b5bb9a710a53b8dfc208696416fe2 Parents: 9f8f3b6 Author: Stephen Mallette <[email protected]> Authored: Fri Jun 8 16:53:30 2018 -0400 Committer: Stephen Mallette <[email protected]> Committed: Fri Jun 8 16:53:30 2018 -0400 ---------------------------------------------------------------------- .../process/traversal/step/map/MathStep.java | 53 +++++++++++++++----- 1 file changed, 40 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/a6e0a2d5/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MathStep.java ---------------------------------------------------------------------- diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MathStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MathStep.java index e259eaf..0994411 100644 --- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MathStep.java +++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/map/MathStep.java @@ -33,6 +33,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalRing; import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil; import org.apache.tinkerpop.gremlin.structure.util.StringFactory; +import java.io.Serializable; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -47,19 +48,14 @@ public final class MathStep<S> extends MapStep<S, Double> implements ByModulatin private static final String CURRENT = "_"; private final String equation; - private final Set<String> variables; - private final Expression expression; + private final TinkerExpression expression; private TraversalRing<S, Number> traversalRing = new TraversalRing<>(); private Set<String> keepLabels; public MathStep(final Traversal.Admin traversal, final String equation) { super(traversal); this.equation = equation; - this.variables = MathStep.getVariables(this.equation); - this.expression = new ExpressionBuilder(this.equation) - .variables(this.variables) - .implicitMultiplication(false) - .build(); + this.expression = new TinkerExpression(equation, MathStep.getVariables(this.equation)); } @Override @@ -69,8 +65,8 @@ public final class MathStep<S> extends MapStep<S, Double> implements ByModulatin @Override protected Double map(final Traverser.Admin<S> traverser) { - final Expression localExpression = new Expression(this.expression); - for (final String var : this.variables) { + final Expression localExpression = new Expression(this.expression.getExpression()); + for (final String var : this.expression.getVariables()) { localExpression.setVariable(var, var.equals(CURRENT) ? TraversalUtil.applyNullable(traverser, this.traversalRing.next()).doubleValue() : @@ -95,7 +91,8 @@ public final class MathStep<S> extends MapStep<S, Double> implements ByModulatin // but I suppose a better way might be make it more clear when this step is dealing with an actual path and // when it is not and/or adjust ComputerVerificationStrategy to cope with the situation where math() is only // dealing with the local stargraph. - return (variables.contains(CURRENT) && variables.size() == 1) ? ElementRequirement.ID : PathProcessor.super.getMaxRequirement(); + return (this.expression.getVariables().contains(CURRENT) && this.expression.getVariables().size() == 1) ? + ElementRequirement.ID : PathProcessor.super.getMaxRequirement(); } @Override @@ -139,12 +136,12 @@ public final class MathStep<S> extends MapStep<S, Double> implements ByModulatin @Override public Set<String> getScopeKeys() { - if (this.variables.contains(CURRENT)) { - final Set<String> temp = new HashSet<>(this.variables); + if (this.expression.getVariables().contains(CURRENT)) { + final Set<String> temp = new HashSet<>(this.expression.getVariables()); temp.remove(CURRENT); return temp; } else - return this.variables; + return this.expression.getVariables(); } @Override @@ -181,4 +178,34 @@ public final class MathStep<S> extends MapStep<S, Double> implements ByModulatin return variables; } + /** + * A wrapper for the {@code Expression} class. That class is not marked {@code Serializable} and therefore gives + * problems in OLAP specifically with Spark. This wrapper allows the {@code Expression} to be serialized in that + * context with Java serialization. + */ + public static class TinkerExpression implements Serializable { + private transient Expression expression; + private final String equation; + private final Set<String> variables; + + public TinkerExpression(final String equation, final Set<String> variables) { + this.variables = variables; + this.equation = equation; + } + + public Expression getExpression() { + if (null == expression) { + this.expression = new ExpressionBuilder(this.equation) + .variables(this.variables) + .implicitMultiplication(false) + .build(); + } + return expression; + } + + public Set<String> getVariables() { + return variables; + } + } + }
