Repository: incubator-systemml Updated Branches: refs/heads/master 7ae794c93 -> f38140baf
[SYSTEMML-698] Remove Guava dependency Remove compile-scope Guava dependency and replace Guava code with equivalent non-library version. Closes #174. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f38140ba Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f38140ba Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f38140ba Branch: refs/heads/master Commit: f38140baf29f12d4c73bd1df1ce4d7f127254012 Parents: 7ae794c Author: Deron Eriksson <[email protected]> Authored: Fri Jun 10 11:22:32 2016 -0700 Committer: Deron Eriksson <[email protected]> Committed: Fri Jun 10 11:22:32 2016 -0700 ---------------------------------------------------------------------- pom.xml | 7 -- src/assembly/distrib/LICENSE | 1 - src/assembly/inmemory/LICENSE | 1 - src/assembly/jar/LICENSE | 1 - src/assembly/standalone-jar/LICENSE | 1 - src/assembly/standalone.xml | 1 - .../api/monitoring/SparkMonitoringUtil.java | 67 +++++++++++++++++--- .../hops/globalopt/InterestingProperties.java | 15 ++--- .../sysml/runtime/io/IOUtilFunctions.java | 2 - .../sysml/runtime/transform/DummycodeAgent.java | 58 ++++++++++------- .../sysml/runtime/transform/RecodeAgent.java | 14 ++-- 11 files changed, 104 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 7467295..9982d41 100644 --- a/pom.xml +++ b/pom.xml @@ -806,13 +806,6 @@ <scope>provided</scope> </dependency> - <!-- Adding Gauva version 14.0.1 to workaround conflict between spark and hadoop dependency --> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava</artifactId> - <version>14.0.1</version> - </dependency> - <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/assembly/distrib/LICENSE ---------------------------------------------------------------------- diff --git a/src/assembly/distrib/LICENSE b/src/assembly/distrib/LICENSE index 0779d38..be745d4 100644 --- a/src/assembly/distrib/LICENSE +++ b/src/assembly/distrib/LICENSE @@ -204,7 +204,6 @@ The following compile-scope dependencies come under the Apache Software License 2.0. -Guava: Google Core Libraries for Java (http://code.google.com/p/guava-libraries/guava) com.google.guava:guava:14.0.1 Apache Wink :: JSON4J (http://www.apache.org/wink/wink-json4j/) org.apache.wink:wink-json4j:1.4 =============================================================================== http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/assembly/inmemory/LICENSE ---------------------------------------------------------------------- diff --git a/src/assembly/inmemory/LICENSE b/src/assembly/inmemory/LICENSE index 1e5f51c..c9b68d6 100644 --- a/src/assembly/inmemory/LICENSE +++ b/src/assembly/inmemory/LICENSE @@ -215,7 +215,6 @@ org.apache.hadoop:hadoop-common:2.4.1 org.apache.hadoop:hadoop-mapreduce-client-core:2.4.1 Compile-scope dependencies: -com.google.guava:guava:14.0.1 org.apache.wink:wink-json4j:1.4 =============================================================================== http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/assembly/jar/LICENSE ---------------------------------------------------------------------- diff --git a/src/assembly/jar/LICENSE b/src/assembly/jar/LICENSE index 0779d38..be745d4 100644 --- a/src/assembly/jar/LICENSE +++ b/src/assembly/jar/LICENSE @@ -204,7 +204,6 @@ The following compile-scope dependencies come under the Apache Software License 2.0. -Guava: Google Core Libraries for Java (http://code.google.com/p/guava-libraries/guava) com.google.guava:guava:14.0.1 Apache Wink :: JSON4J (http://www.apache.org/wink/wink-json4j/) org.apache.wink:wink-json4j:1.4 =============================================================================== http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/assembly/standalone-jar/LICENSE ---------------------------------------------------------------------- diff --git a/src/assembly/standalone-jar/LICENSE b/src/assembly/standalone-jar/LICENSE index 44fd886..5078879 100644 --- a/src/assembly/standalone-jar/LICENSE +++ b/src/assembly/standalone-jar/LICENSE @@ -232,7 +232,6 @@ org.codehaus.jackson:jackson-core-asl:1.8.8 org.codehaus.jackson:jackson-mapper-asl:1.8.8 Compile-scope dependencies: -com.google.guava:guava:14.0.1 org.apache.wink:wink-json4j:1.4 =============================================================================== http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/assembly/standalone.xml ---------------------------------------------------------------------- diff --git a/src/assembly/standalone.xml b/src/assembly/standalone.xml index 8c88cec..bd835d0 100644 --- a/src/assembly/standalone.xml +++ b/src/assembly/standalone.xml @@ -109,7 +109,6 @@ <!-- Exclude compile-scoped dependencies since they are in main artifact jar --> <exclude>*:antlr4-annotations*</exclude> <exclude>*:antlr4-runtime*</exclude> - <exclude>*:guava*</exclude> <exclude>*:org.abego.treelayout.core*</exclude> <exclude>*:wink-json4j*</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java index e5326d6..352eeba 100644 --- a/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java +++ b/src/main/java/org/apache/sysml/api/monitoring/SparkMonitoringUtil.java @@ -21,24 +21,30 @@ package org.apache.sysml.api.monitoring; import java.io.BufferedWriter; import java.io.FileWriter; import java.io.IOException; +import java.util.AbstractMap.SimpleEntry; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; -import scala.collection.Seq; -import scala.xml.Node; - -import com.google.common.collect.Multimap; -import com.google.common.collect.TreeMultimap; import org.apache.sysml.lops.Lop; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.instructions.Instruction; import org.apache.sysml.runtime.instructions.spark.SPInstruction; import org.apache.sysml.runtime.instructions.spark.functions.SparkListener; +import scala.collection.Seq; +import scala.xml.Node; + +import com.google.common.collect.Multimap; +import com.google.common.collect.TreeMultimap; + /** * Usage guide: * MLContext mlCtx = new MLContext(sc, true); @@ -47,13 +53,11 @@ import org.apache.sysml.runtime.instructions.spark.functions.SparkListener; * mlCtx.getMonitoringUtil().getRuntimeInfoInHTML("runtime.html"); */ public class SparkMonitoringUtil { - // ---------------------------------------------------- - // For VLDB Demo: - private Multimap<Location, String> instructions = TreeMultimap.create(); - private Multimap<String, Integer> stageIDs = TreeMultimap.create(); // instruction -> stageIds - private Multimap<String, Integer> jobIDs = TreeMultimap.create(); // instruction -> jobIds private HashMap<String, String> lineageInfo = new HashMap<String, String>(); // instruction -> lineageInfo private HashMap<String, Long> instructionCreationTime = new HashMap<String, Long>(); + private MultiMap<Location, String> instructions = new MultiMap<Location, String>(); + private MultiMap<String, Integer> stageIDs = new MultiMap<String, Integer>(); + private MultiMap<String, Integer> jobIDs = new MultiMap<String, Integer>(); private Multimap<Integer, String> rddInstructionMapping = TreeMultimap.create(); @@ -597,4 +601,47 @@ public class SparkMonitoringUtil { tmp = tmp.replaceAll(Lop.INSTRUCTION_DELIMITOR, ", "); return tmp; } + + public class MultiMap<K, V extends Comparable<V>> { + private SortedMap<K, List<V>> m = new TreeMap<K, List<V>>(); + + public MultiMap(){ + } + + public void put(K key, V value) { + List<V> list; + if (!m.containsKey(key)) { + list = new ArrayList<V>(); + m.put(key, list); + } else { + list = m.get(key); + } + list.add(value); + Collections.sort(list); + } + + public Collection<Entry<K, V>> entries() { + // the treemap is sorted and the lists are sorted, so can traverse + // to generate a key/value ordered list of all entries. + Collection<Entry<K, V>> allEntries = new ArrayList<Entry<K, V>>(); + + for (K key : m.keySet()) { + List<V> list = m.get(key); + for (V value : list) { + Entry<K, V> listEntry = new SimpleEntry<K, V>(key, value); + allEntries.add(listEntry); + } + } + + return allEntries; + } + + public List<V> get(K key) { + return m.get(key); + } + + public Set<K> keySet() { + return m.keySet(); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java b/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java index b81308d..f1bcbe0 100644 --- a/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java +++ b/src/main/java/org/apache/sysml/hops/globalopt/InterestingProperties.java @@ -19,7 +19,7 @@ package org.apache.sysml.hops.globalopt; -import com.google.common.base.Objects; +import java.util.Arrays; /** * An instance of this class represents one 'interesting property set' defined by the instances @@ -101,15 +101,10 @@ public class InterestingProperties @Override public int hashCode() { - //relies on google's guava library - return Objects.hashCode( - _blocksize, - (_format!=null)?_format.ordinal():-1, - (_location!=null)?_location.ordinal():-1, - (_pformat!=null)?_pformat.ordinal():-1, - _replication, - _emptyblocks - ); + Object[] array = new Object[] { _blocksize, (_format != null) ? _format.ordinal() : -1, + (_location != null) ? _location.ordinal() : -1, (_pformat != null) ? _pformat.ordinal() : -1, + _replication, _emptyblocks }; + return Arrays.hashCode(array); } @Override http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 93a8818..48787b1 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -126,8 +126,6 @@ public class IOUtilFunctions */ public static String[] split(String str, String delim) { - //note: split via stringutils faster than precompiled pattern / guava splitter - //split by whole separator required for multi-character delimiters, preserve //all tokens required for empty cells and in order to keep cell alignment return StringUtils.splitByWholeSeparatorPreserveAllTokens(str, delim); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java index ecd1e4d..a25444d 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java +++ b/src/main/java/org/apache/sysml/runtime/transform/DummycodeAgent.java @@ -23,10 +23,15 @@ import java.io.BufferedWriter; import java.io.File; import java.io.IOException; import java.io.OutputStreamWriter; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.regex.Pattern; import org.apache.hadoop.fs.FileSystem; @@ -34,17 +39,13 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.wink.json4j.JSONArray; -import org.apache.wink.json4j.JSONException; -import org.apache.wink.json4j.JSONObject; - -import com.google.common.base.Functions; -import com.google.common.collect.Ordering; - import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.transform.encode.Encoder; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.wink.json4j.JSONArray; +import org.apache.wink.json4j.JSONException; +import org.apache.wink.json4j.JSONObject; public class DummycodeAgent extends Encoder { @@ -228,8 +229,20 @@ public class DummycodeAgent extends Encoder if ( map != null ) { // order map entries by their recodeID - Ordering<String> valueComparator = Ordering.natural().onResultOf(Functions.forMap(map)); - newNames = valueComparator.sortedCopy(map.keySet()); + List<Map.Entry<String, Long>> entryList = new ArrayList<Map.Entry<String, Long>>(map.entrySet()); + Comparator<Map.Entry<String, Long>> comp = new Comparator<Map.Entry<String, Long>>() { + @Override + public int compare(Entry<String, Long> entry1, Entry<String, Long> entry2) { + Long value1 = entry1.getValue(); + Long value2 = entry2.getValue(); + return (int) (value1 - value2); + } + }; + Collections.sort(entryList, comp); + newNames = new ArrayList<String>(); + for (Entry<String, Long> entry : entryList) { + newNames.add(entry.getKey()); + } // construct concatenated string of map entries sb.setLength(0); @@ -252,15 +265,23 @@ public class DummycodeAgent extends Encoder if ( map != null ) { + // order map entries by their recodeID (represented as Strings .. "1", "2", etc.) - Ordering<String> orderByID = new Ordering<String>() - { - public int compare(String s1, String s2) { - return (Integer.parseInt(s1) - Integer.parseInt(s2)); - } + List<Map.Entry<String, String>> entryList = new ArrayList<Map.Entry<String, String>>(map.entrySet()); + Comparator<Map.Entry<String, String>> comp = new Comparator<Map.Entry<String, String>>() { + @Override + public int compare(Entry<String, String> entry1, Entry<String, String> entry2) { + String value1 = entry1.getValue(); + String value2 = entry2.getValue(); + return (Integer.parseInt(value1) - Integer.parseInt(value2)); + } }; + Collections.sort(entryList, comp); + newNames = new ArrayList<String>(); + for (Entry<String, String> entry : entryList) { + newNames.add(entry.getKey()); + } - newNames = orderByID.onResultOf(Functions.forMap(map)).sortedCopy(map.keySet()); // construct concatenated string of map entries sb.setLength(0); for(int idx=0; idx < newNames.size(); idx++) @@ -403,31 +424,24 @@ public class DummycodeAgent extends Encoder @Override public double[] encode(String[] in, double[] out) { - // TODO Auto-generated method stub return null; } @Override public MatrixBlock encode(FrameBlock in, MatrixBlock out) { - // TODO Auto-generated method stub return null; } @Override public void build(String[] in) { - // TODO Auto-generated method stub - } @Override public void build(FrameBlock in) { - // TODO Auto-generated method stub - } @Override public FrameBlock getMetaData(FrameBlock out) { - // TODO Auto-generated method stub return null; } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f38140ba/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java index c4c7e66..6170412 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java +++ b/src/main/java/org/apache/sysml/runtime/transform/RecodeAgent.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -35,12 +36,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; -import org.apache.wink.json4j.JSONArray; -import org.apache.wink.json4j.JSONException; -import org.apache.wink.json4j.JSONObject; - -import com.google.common.collect.Ordering; - import org.apache.sysml.lops.Lop; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -49,6 +44,9 @@ import org.apache.sysml.runtime.transform.MVImputeAgent.MVMethod; import org.apache.sysml.runtime.transform.decode.DecoderRecode; import org.apache.sysml.runtime.transform.encode.Encoder; import org.apache.sysml.runtime.util.UtilFunctions; +import org.apache.wink.json4j.JSONArray; +import org.apache.wink.json4j.JSONException; +import org.apache.wink.json4j.JSONObject; public class RecodeAgent extends Encoder { @@ -261,8 +259,8 @@ public class RecodeAgent extends Encoder throw new RuntimeException("Can not proceed since \"" + agents.getName(colID) + "\" (id=" + colID + ") contains only the missing values, and not a single valid value -- set imputation method to \"constant\"."); // Order entries by category (string) value - Ordering<String> valueComparator = Ordering.natural(); - List<String> newNames = valueComparator.sortedCopy(map.keySet()); + List<String> newNames = new ArrayList<String>(map.keySet()); + Collections.sort(newNames); for(String w : newNames) { //map.keySet()) { count = map.get(w);
