Repository: systemml Updated Branches: refs/heads/master a3ab19768 -> b02599c6d
[MINOR] Cleanups of unchecked generics and improved list concatenation Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8cb55328 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8cb55328 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8cb55328 Branch: refs/heads/master Commit: 8cb5532879a7d2171a1ff9c9f85a7de7fa1e6a23 Parents: a3ab197 Author: Matthias Boehm <[email protected]> Authored: Thu Jan 25 13:14:47 2018 -0800 Committer: Matthias Boehm <[email protected]> Committed: Thu Jan 25 13:14:47 2018 -0800 ---------------------------------------------------------------------- .../context/SparkExecutionContext.java | 2 +- .../controlprogram/parfor/CachedReuseVariables.java | 3 ++- .../parfor/RemoteParForSparkWorker.java | 4 ++-- .../parfor/ResultMergeRemoteGrouping.java | 1 + .../parfor/ResultMergeRemoteSorting.java | 1 + .../apache/sysml/runtime/io/IOUtilFunctions.java | 2 +- .../matrix/data/AddDummyWeightConverter.java | 1 + .../sysml/runtime/matrix/data/MatrixBlock.java | 2 +- .../sysml/runtime/matrix/mapred/CSVWriteMapper.java | 1 + .../mapred/CollectMultipleConvertedOutputs.java | 5 ++--- .../sysml/runtime/matrix/sort/ValueSortMapper.java | 6 +++--- .../runtime/transform/decode/DecoderFactory.java | 1 + .../runtime/transform/encode/EncoderFactory.java | 1 + .../sysml/runtime/transform/meta/TfMetaUtils.java | 1 + .../apache/sysml/runtime/util/UtilFunctions.java | 16 ++++++++++++++++ .../java/org/apache/sysml/utils/Statistics.java | 1 + .../functions/frame/FrameConverterTest.java | 13 +++++-------- 17 files changed, 41 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index 00710c3..098eb37 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -1137,7 +1137,7 @@ public class SparkExecutionContext extends ExecutionContext } } - @SuppressWarnings({ "rawtypes" }) + @SuppressWarnings({ "rawtypes", "unchecked" }) private void rCleanupLineageObject(LineageObject lob) throws IOException { http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/CachedReuseVariables.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/CachedReuseVariables.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/CachedReuseVariables.java index 48ef883..2532cbd 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/CachedReuseVariables.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/CachedReuseVariables.java @@ -43,7 +43,8 @@ public class CachedReuseVariables //build reuse map if not created yet or evicted if( tmp == null ) { tmp = new LocalVariableMap(vars); - tmp.removeAllIn(new HashSet<>(blacklist)); + tmp.removeAllIn((blacklist instanceof HashSet) ? + (HashSet<String>)blacklist : new HashSet<>(blacklist)); _data.put(pfid, new SoftReference<>(tmp)); } //reuse existing reuse map http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java index 9753635..52a19f0 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java @@ -26,7 +26,6 @@ import java.util.HashMap; import java.util.Iterator; import java.util.Map.Entry; -import org.apache.commons.collections.CollectionUtils; import org.apache.spark.TaskContext; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.spark.util.LongAccumulator; @@ -36,6 +35,7 @@ import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysml.runtime.controlprogram.parfor.util.IDHandler; import org.apache.sysml.runtime.util.LocalFileUtils; +import org.apache.sysml.runtime.util.UtilFunctions; import scala.Tuple2; @@ -114,7 +114,7 @@ public class RemoteParForSparkWorker extends ParWorker implements PairFlatMapFun //reuse shared inputs (to read shared inputs once per process instead of once per core; //we reuse everything except result variables and partitioned input matrices) _ec.pinVariables(_ec.getVarList()); //avoid cleanup of shared inputs - Collection<String> blacklist = CollectionUtils.union(_resultVars, _ec.getVarListPartitioned()); + Collection<String> blacklist = UtilFunctions.asSet(_resultVars, _ec.getVarListPartitioned()); reuseVars.reuseVariables(_jobid, _ec.getVariables(), blacklist); //init and register-cleanup of buffer pool (in parfor spark, multiple tasks might http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java index 1f63f85..f543788 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteGrouping.java @@ -30,6 +30,7 @@ public class ResultMergeRemoteGrouping extends WritableComparator } @Override + @SuppressWarnings("rawtypes") public int compare(WritableComparable k1, WritableComparable k2) { ResultMergeTaggedMatrixIndexes key1 = (ResultMergeTaggedMatrixIndexes)k1; http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java index 1ed431e..850a190 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSorting.java @@ -29,6 +29,7 @@ public class ResultMergeRemoteSorting extends WritableComparator } @Override + @SuppressWarnings("rawtypes") public int compare(WritableComparable k1, WritableComparable k2) { ResultMergeTaggedMatrixIndexes key1 = (ResultMergeTaggedMatrixIndexes)k1; http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 9900c45..8d55eab 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -420,7 +420,7 @@ public class IOUtilFunctions * @return the number of columns in the collection of csv file splits * @throws IOException if IOException occurs */ - @SuppressWarnings({ "rawtypes" }) + @SuppressWarnings({ "rawtypes", "unchecked" }) public static int countNumColumnsCSV(InputSplit[] splits, InputFormat informat, JobConf job, String delim ) throws IOException { http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java index 1f1fad6..4c73402 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java @@ -60,6 +60,7 @@ public class AddDummyWeightConverter implements Converter<Writable, Writable, Ma } @Override + @SuppressWarnings("unchecked") public Pair<MatrixIndexes, WeightedPair> next() { Pair<MatrixIndexes, MatrixCell> temp=toCellConverter.next(); pair.setKey(temp.getKey()); http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 56a3f50..432adec 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -559,7 +559,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab //check for existing sparse block: return empty list if( sparseBlock==null ) - return Collections.EMPTY_LIST.iterator(); + return Collections.emptyListIterator(); //get iterator over sparse block return sparseBlock.getIterator(rl, ru); http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java index 3702d69..64a5763 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java @@ -41,6 +41,7 @@ public class CSVWriteMapper extends MapperBase implements Mapper<Writable, Writa TaggedFirstSecondIndexes outIndexes=new TaggedFirstSecondIndexes(); @Override + @SuppressWarnings("unchecked") public void map(Writable rawKey, Writable rawValue, OutputCollector<TaggedFirstSecondIndexes, MatrixBlock> out, Reporter reporter) throws IOException http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java index e630029..e5cc699 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CollectMultipleConvertedOutputs.java @@ -59,9 +59,8 @@ public class CollectMultipleConvertedOutputs } } - public void directOutput(Writable key, Writable value, int output, Reporter reporter) - throws IOException - { + @SuppressWarnings("unchecked") + public void directOutput(Writable key, Writable value, int output, Reporter reporter) throws IOException { multipleOutputs.getCollector(Integer.toString(output), reporter).collect(key, value); } http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java index 344e4f5..12f41b0 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/ValueSortMapper.java @@ -77,9 +77,9 @@ public class ValueSortMapper<KIN extends WritableComparable, VIN extends Writabl } @Override - public void configure(JobConf job) - { - try { + @SuppressWarnings("unchecked") + public void configure(JobConf job) { + try { brlen = MRJobConfiguration.getNumRowsPerBlock(job, (byte) 0); bclen = MRJobConfiguration.getNumColumnsPerBlock(job, (byte) 0); String str=job.get(SortMR.COMBINE_INSTRUCTION, null); http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java index 5a351cb..19aca0f 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java +++ b/src/main/java/org/apache/sysml/runtime/transform/decode/DecoderFactory.java @@ -36,6 +36,7 @@ import org.apache.wink.json4j.JSONObject; public class DecoderFactory { + @SuppressWarnings("unchecked") public static Decoder createDecoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta) throws DMLRuntimeException { http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java index a776cb6..a1cc0d0 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java +++ b/src/main/java/org/apache/sysml/runtime/transform/encode/EncoderFactory.java @@ -45,6 +45,7 @@ public class EncoderFactory return createEncoder(spec, colnames, lschema, meta); } + @SuppressWarnings("unchecked") public static Encoder createEncoder(String spec, String[] colnames, ValueType[] schema, FrameBlock meta) throws DMLRuntimeException { http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java index 3d4353d..e8d512e 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java +++ b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java @@ -337,6 +337,7 @@ public class TfMetaUtils * @return list of column ids * @throws IOException if IOException occurs */ + @SuppressWarnings("unchecked") private static List<Integer> parseRecodeColIDs(String spec, String[] colnames) throws IOException { http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java index 200b2eb..e7a3bdc 100644 --- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java @@ -599,6 +599,22 @@ public class UtilFunctions } @SafeVarargs + public static <T> List<T> asList(List<T>... inputs) { + List<T> ret = new ArrayList<>(); + for( List<T> list : inputs ) + ret.addAll(list); + return ret; + } + + @SafeVarargs + public static <T> Set<T> asSet(List<T>... inputs) { + Set<T> ret = new HashSet<>(); + for( List<T> list : inputs ) + ret.addAll(list); + return ret; + } + + @SafeVarargs public static <T> Set<T> asSet(T[]... inputs) { Set<T> ret = new HashSet<>(); for( T[] input : inputs ) http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/main/java/org/apache/sysml/utils/Statistics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java index c003f30..0371966 100644 --- a/src/main/java/org/apache/sysml/utils/Statistics.java +++ b/src/main/java/org/apache/sysml/utils/Statistics.java @@ -565,6 +565,7 @@ public class Statistics * @return string representing the heavy hitter instructions in tabular * format */ + @SuppressWarnings("unchecked") public static String getHeavyHitters(int num) { int len = _instStats.size(); if (num <= 0 || len <= 0) http://git-wip-us.apache.org/repos/asf/systemml/blob/8cb55328/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java index 3cd14c7..ff539ab 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java @@ -20,12 +20,10 @@ package org.apache.sysml.test.integration.functions.frame; import java.io.IOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.Text; import org.apache.spark.api.java.JavaPairRDD; @@ -87,14 +85,13 @@ public class FrameConverterTest extends AutomatedTestBase private final static List<ValueType> schemaMixedLargeListInt = Collections.nCopies(200, ValueType.INT); private final static List<ValueType> schemaMixedLargeListBool = Collections.nCopies(200, ValueType.BOOLEAN); - private static final List<ValueType> schemaMixedLargeList = new ArrayList<ValueType>(CollectionUtils.union( - CollectionUtils.union(schemaMixedLargeListStr, schemaMixedLargeListDble), - CollectionUtils.union(schemaMixedLargeListInt, schemaMixedLargeListBool))); + private static final List<ValueType> schemaMixedLargeList = UtilFunctions.asList( + schemaMixedLargeListStr, schemaMixedLargeListDble, schemaMixedLargeListInt, schemaMixedLargeListBool); private static final ValueType[] schemaMixedLarge = schemaMixedLargeList.toArray(new ValueType[0]); - private static final List<ValueType> schemaMixedLargeListDFrame = new ArrayList<ValueType>(CollectionUtils.union( - CollectionUtils.union(schemaMixedLargeListStr.subList(0, 100), schemaMixedLargeListDble.subList(0, 100)), - CollectionUtils.union(schemaMixedLargeListInt.subList(0, 100), schemaMixedLargeListBool.subList(0, 100)))); + private static final List<ValueType> schemaMixedLargeListDFrame = UtilFunctions.asList( + schemaMixedLargeListStr.subList(0, 100), schemaMixedLargeListDble.subList(0, 100), + schemaMixedLargeListInt.subList(0, 100), schemaMixedLargeListBool.subList(0, 100)); private static final ValueType[] schemaMixedLargeDFrame = schemaMixedLargeListDFrame.toArray(new ValueType[0]); //NOTE: moderate number of columns to workaround https://issues.apache.org/jira/browse/SPARK-16845
