This is an automated email from the ASF dual-hosted git repository. bertty pushed a commit to branch WAYANG-35 in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 4ca6c3347b3c225e7c78944b9622ad79430bfc22 Author: Bertty Contreras-Rojas <[email protected]> AuthorDate: Fri Sep 3 12:45:05 2021 +0200 [WAYANG-35] the library Trove4j was removed It's important to focus on the TODO added because they may affect the performance of some code Signed-off-by: bertty <[email protected]> --- wayang-commons/pom.xml | 8 --- .../optimizer/costs/SimpleEstimationContext.java | 11 ++-- .../costs/NestableLoadProfileEstimatorTest.java | 11 ++-- .../java/operators/graph/JavaPageRankOperator.java | 67 +++++++++++----------- .../wayang/profiler/log/GeneticOptimizer.java | 29 +++++++--- 5 files changed, 64 insertions(+), 62 deletions(-) diff --git a/wayang-commons/pom.xml b/wayang-commons/pom.xml index e355bc9..5bb233b 100644 --- a/wayang-commons/pom.xml +++ b/wayang-commons/pom.xml @@ -38,14 +38,6 @@ other modules </description> - <dependencies> - <dependency> - <groupId>net.sf.trove4j</groupId> - <artifactId>trove4j</artifactId> - <version>3.0.3</version> - </dependency> - </dependencies> - <dependencyManagement> <dependencies> <!-- Apache Hadoop --> diff --git a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java index bd2367b..c0d0d9c 100644 --- a/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java +++ b/wayang-commons/wayang-core/src/main/java/org/apache/wayang/core/optimizer/costs/SimpleEstimationContext.java @@ -18,8 +18,7 @@ package org.apache.wayang.core.optimizer.costs; -import gnu.trove.map.TObjectDoubleMap; -import gnu.trove.map.hash.TObjectDoubleHashMap; +import java.util.HashMap; import org.json.JSONObject; import org.apache.wayang.core.optimizer.cardinality.CardinalityEstimate; import org.apache.wayang.core.util.JsonSerializables; @@ -34,8 +33,8 @@ import java.util.List; public class SimpleEstimationContext implements EstimationContext { private final CardinalityEstimate[] inputCardinalities, outputCardinalities; - - private final TObjectDoubleMap<String> doubleProperties; + //TODO: change a for efficient Hashmap + private final HashMap<String, Double> doubleProperties; private final int numExecutions; @@ -44,7 +43,7 @@ public class SimpleEstimationContext implements EstimationContext { */ public SimpleEstimationContext(CardinalityEstimate[] inputCardinalities, CardinalityEstimate[] outputCardinalities, - TObjectDoubleMap<String> doubleProperties, + HashMap<String, Double> doubleProperties, int numExecutions) { this.inputCardinalities = inputCardinalities; this.outputCardinalities = outputCardinalities; @@ -106,7 +105,7 @@ public class SimpleEstimationContext implements EstimationContext { CardinalityEstimate.class ); - final TObjectDoubleHashMap<String> doubleProperties = new TObjectDoubleHashMap<>(); + final HashMap<String, Double> doubleProperties = new HashMap<String, Double>(); final JSONObject doublePropertiesJson = json.optJSONObject("properties"); if (doublePropertiesJson != null) { for (String key : doublePropertiesJson.keySet()) { diff --git a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java index 707b9e5..6295021 100644 --- a/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java +++ b/wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/optimizer/costs/NestableLoadProfileEstimatorTest.java @@ -18,8 +18,7 @@ package org.apache.wayang.core.optimizer.costs; -import gnu.trove.map.TObjectDoubleMap; -import gnu.trove.map.hash.TObjectDoubleHashMap; +import java.util.HashMap; import org.junit.Assert; import org.junit.Test; import org.apache.wayang.core.optimizer.OptimizationUtils; @@ -56,7 +55,7 @@ public class NestableLoadProfileEstimatorTest { new CardinalityEstimate(10, 10, 1d), new CardinalityEstimate(100, 100, 1d) }, new CardinalityEstimate[]{new CardinalityEstimate(200, 300, 1d)}, - new TObjectDoubleHashMap<>(), + new HashMap<String, Double>(), 1 )); @@ -89,7 +88,7 @@ public class NestableLoadProfileEstimatorTest { new CardinalityEstimate(10, 10, 1d), new CardinalityEstimate(100, 100, 1d) }, new CardinalityEstimate[]{new CardinalityEstimate(200, 300, 1d)}, - new TObjectDoubleHashMap<>(), + new HashMap<String, Double>(), 1 )); @@ -118,7 +117,7 @@ public class NestableLoadProfileEstimatorTest { final NestableLoadProfileEstimator estimator = LoadProfileEstimators.createFromSpecification(null, specification); SomeExecutionOperator execOp = new SomeExecutionOperator(); - TObjectDoubleMap<String> properties = new TObjectDoubleHashMap<>(); + HashMap<String, Double> properties = new HashMap<String, Double>(); properties.put("numIterations", 2d); final LoadProfile estimate = estimator.estimate(new SimpleEstimationContext( new CardinalityEstimate[]{ @@ -155,7 +154,7 @@ public class NestableLoadProfileEstimatorTest { final NestableLoadProfileEstimator estimator = LoadProfileEstimators.createFromSpecification(null, specification); SomeExecutionOperator execOp = new SomeExecutionOperator(); - TObjectDoubleMap<String> properties = new TObjectDoubleHashMap<>(); + HashMap<String, Double> properties = new HashMap<String, Double>(); properties.put("numIterations", 2d); final LoadProfile estimate = estimator.estimate(new SimpleEstimationContext( new CardinalityEstimate[]{ diff --git a/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java b/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java index e4633a5..d06c1b5 100644 --- a/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java +++ b/wayang-platforms/wayang-java/src/main/java/org/apache/wayang/java/operators/graph/JavaPageRankOperator.java @@ -18,11 +18,9 @@ package org.apache.wayang.java.operators.graph; -import gnu.trove.iterator.TLongFloatIterator; -import gnu.trove.map.TLongFloatMap; -import gnu.trove.map.TLongIntMap; -import gnu.trove.map.hash.TLongFloatHashMap; -import gnu.trove.map.hash.TLongIntHashMap; +import java.util.HashMap; +import java.util.Map; +import java.util.function.BiFunction; import org.apache.wayang.basic.data.Tuple2; import org.apache.wayang.basic.operators.PageRankOperator; import org.apache.wayang.core.optimizer.OptimizationContext; @@ -67,8 +65,8 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut StreamChannel.Instance output = (StreamChannel.Instance) outputs[0]; final Collection<Tuple2<Long, Long>> edges = input.provideCollection(); - final TLongFloatMap pageRanks = this.pageRank(edges); - final Stream<Tuple2<Long, Float>> pageRankStream = this.stream(pageRanks); + final Map<Long, Float> pageRanks = this.pageRank(edges); + final Stream<Tuple2<Long, Float>> pageRankStream = pageRanks.entrySet().stream().map(entry -> new Tuple2<>(entry.getKey(), entry.getValue())); output.accept(pageRankStream); @@ -81,31 +79,33 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut * @param edgeDataSet edges of a graph * @return the page ranks */ - private TLongFloatMap pageRank(Collection<Tuple2<Long, Long>> edgeDataSet) { + //TODO: change for efficient map + private Map<Long, Float> pageRank(Collection<Tuple2<Long, Long>> edgeDataSet) { // Get the degress of all vertices and make sure we collect *all* vertices. - TLongIntMap degrees = new TLongIntHashMap(); + //TODO: change for efficient map + HashMap<Long, Integer> degrees = new HashMap<>(); for (Tuple2<Long, Long> edge : edgeDataSet) { - degrees.adjustOrPutValue(edge.field0, 1, 1); - degrees.adjustOrPutValue(edge.field0, 0, 0); + this.adjustOrPutValue(degrees, edge.field0, 1, 1, Integer::sum); + this.adjustOrPutValue(degrees, edge.field0, 0, 0, Integer::sum); } int numVertices = degrees.size(); float initialRank = 1f / numVertices; float dampingRank = (1 - this.dampingFactor) / numVertices; // Initialize the rank map. - TLongFloatMap initialRanks = new TLongFloatHashMap(); - degrees.forEachKey(k -> { + //TODO: change for efficient map + HashMap<Long, Float> initialRanks = new HashMap<>(); + degrees.forEach( (k, v) -> { initialRanks.putIfAbsent(k, initialRank); - return true; }); - TLongFloatMap currentRanks = initialRanks; + HashMap<Long, Float> currentRanks = initialRanks; for (int iteration = 0; iteration < this.getNumIterations(); iteration++) { // Add the damping first. - TLongFloatMap newRanks = new TLongFloatHashMap(currentRanks.size()); - degrees.forEachKey(k -> { + //TODO: change for efficient map + HashMap<Long, Float> newRanks = new HashMap<Long, Float>(currentRanks.size()); + degrees.forEach( (k, v) -> { newRanks.putIfAbsent(k, dampingRank); - return true; }); // Now add the other ranks. @@ -115,7 +115,7 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut final int degree = degrees.get(sourceVertex); final float currentRank = currentRanks.get(sourceVertex); final float partialRank = this.dampingFactor * currentRank / degree; - newRanks.adjustOrPutValue(targetVertex, partialRank, partialRank); + this.adjustOrPutValue(newRanks, targetVertex, partialRank, partialRank, Float::sum); } currentRanks = newRanks; @@ -124,24 +124,21 @@ public class JavaPageRankOperator extends PageRankOperator implements JavaExecut return currentRanks; } - private Stream<Tuple2<Long, Float>> stream(TLongFloatMap map) { - final TLongFloatIterator tLongFloatIterator = map.iterator(); - Iterator<Tuple2<Long, Float>> iterator = new Iterator<Tuple2<Long, Float>>() { - @Override - public boolean hasNext() { - return tLongFloatIterator.hasNext(); - } - - @Override - public Tuple2<Long, Float> next() { - tLongFloatIterator.advance(); - return new Tuple2<>(tLongFloatIterator.key(), tLongFloatIterator.value()); - } - }; - return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iterator, 0), false); + /** + * simulate the process on the Trove4j library + * @param key key to modify on the map + * @param default_value default value in the case of not key + * @param correction element to add the array in the case of the key exist + */ + private <T> void adjustOrPutValue(Map<Long, T> map, Long key, T default_value, T correction, BiFunction<T, T, T> update){ + if(map.containsKey(key)){ + T value = map.get(key); + map.replace(key, update.apply(value, correction) ); + }else{ + map.put(key, default_value); + } } - @Override public String getLoadProfileEstimatorConfigurationKey() { return "wayang.java.pagerank.load"; diff --git a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java index 33149fd..03645f9 100644 --- a/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java +++ b/wayang-profiler/src/main/java/org/apache/wayang/profiler/log/GeneticOptimizer.java @@ -18,8 +18,7 @@ package org.apache.wayang.profiler.log; -import gnu.trove.map.TObjectIntMap; -import gnu.trove.map.hash.TObjectIntHashMap; +import java.util.HashMap; import org.apache.wayang.core.api.Configuration; import org.apache.wayang.core.optimizer.costs.LoadProfileEstimator; import org.apache.wayang.core.platform.AtomicExecution; @@ -61,7 +60,8 @@ public class GeneticOptimizer { /** * Counts observation instances, such as an operator or a platform initialization, in the training data. */ - private final TObjectIntMap<Object> numObservations; + //TODO: change for efficient map + private final HashMap<Object, Integer> numObservations; /** * {@link Variable}s to learn the overhead of {@link Platform} initialization. @@ -164,20 +164,35 @@ public class GeneticOptimizer { } // Count the distinct elements in the PartialExecutions. - this.numObservations = new TObjectIntHashMap<>(); + this.numObservations = new HashMap<>(); this.runtimeSum = 0L; for (PartialExecution observation : this.observations) { for (String key : getLoadProfileEstimatorKeys(observation)) { - this.numObservations.adjustOrPutValue(key, 1, 1); + this.adjustOrPutValue(key, 1, 1); } for (Platform platform : observation.getInitializedPlatforms()) { - this.numObservations.adjustOrPutValue(platform, 1, 1); + this.adjustOrPutValue(platform, 1, 1); } this.runtimeSum += observation.getMeasuredExecutionTime(); } } /** + * simulate the process on the Trove4j library + * @param key key to modify on the map + * @param default_value default value in the case of not key + * @param correction element to add the array in the case of the key exist + */ + private void adjustOrPutValue(Object key, int default_value, int correction){ + if(this.numObservations.containsKey(key)){ + Integer value = this.numObservations.get(key); + this.numObservations.replace(key, value + correction ); + }else{ + this.numObservations.put(key, default_value); + } + } + + /** * Creates a population of random {@link Individual}s. * * @return the {@link Individual}s ordered by their fitness @@ -283,7 +298,7 @@ public class GeneticOptimizer { return observations; } - public TObjectIntMap<Object> getNumObservations() { + public HashMap<Object, Integer> getNumObservations() { return numObservations; }
