http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java index 9c59d4e..ee1db4c 100644 --- a/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java +++ b/src/main/java/org/apache/sysml/runtime/io/MatrixReader.java @@ -119,7 +119,7 @@ public abstract class MatrixReader throws InterruptedException, ExecutionException { //create sort tasks (increase number of tasks for better load balance) - ArrayList<SortRowsTask> tasks = new ArrayList<SortRowsTask>(); + ArrayList<SortRowsTask> tasks = new ArrayList<>(); int k2 = (int) Math.min(8*k, rlen); int blklen = (int)(Math.ceil((double)rlen/k2)); for( int i=0; i<k2 & i*blklen<rlen; i++ )
http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java index 5e7df99..98edac4 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java @@ -87,7 +87,7 @@ public class ReaderBinaryBlock extends MatrixReader throws IOException, DMLRuntimeException { //allocate output matrix block collection - ArrayList<IndexedMatrixValue> ret = new ArrayList<IndexedMatrixValue>(); + ArrayList<IndexedMatrixValue> ret = new ArrayList<>(); //prepare file access JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java index 374e795..e5c53dd 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlockParallel.java @@ -88,7 +88,7 @@ public class ReaderBinaryBlockParallel extends ReaderBinaryBlock { //create read tasks for all files ExecutorService pool = Executors.newFixedThreadPool(_numThreads); - ArrayList<ReadFileTask> tasks = new ArrayList<ReadFileTask>(); + ArrayList<ReadFileTask> tasks = new ArrayList<>(); for( Path lpath : IOUtilFunctions.getSequenceFilePaths(fs, path) ){ ReadFileTask t = new ReadFileTask(lpath, job, dest, rlen, clen, brlen, bclen); tasks.add(t); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java index 8d2ef72..9ad97e2 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java @@ -101,7 +101,7 @@ public class ReaderTextCSV extends MatrixReader throws IOException, DMLRuntimeException { //prepare file paths in alphanumeric order - ArrayList<Path> files=new ArrayList<Path>(); + ArrayList<Path> files=new ArrayList<>(); if(fs.isDirectory(path)) { for(FileStatus stat: fs.listStatus(path, CSVReblockMR.hiddenFileFilter)) files.add(stat.getPath()); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java index 2fbe32f..eaf5d7f 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java @@ -133,7 +133,7 @@ public class ReaderTextCSVParallel extends MatrixReader try { // create read tasks for all splits - ArrayList<CSVReadTask> tasks = new ArrayList<CSVReadTask>(); + ArrayList<CSVReadTask> tasks = new ArrayList<>(); int splitCount = 0; for (InputSplit split : splits) { tasks.add( new CSVReadTask(split, _offsets, informat, job, dest, @@ -188,7 +188,7 @@ public class ReaderTextCSVParallel extends MatrixReader try { ExecutorService pool = Executors.newFixedThreadPool(_numThreads); - ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>(); + ArrayList<CountRowsTask> tasks = new ArrayList<>(); for (InputSplit split : splits) { tasks.add(new CountRowsTask(split, informat, job, hasHeader)); hasHeader = false; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java index 5e0e043..b2a9608 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCellParallel.java @@ -130,7 +130,7 @@ public class ReaderTextCellParallel extends MatrixReader //create read tasks for all splits ExecutorService pool = Executors.newFixedThreadPool(par); InputSplit[] splits = informat.getSplits(job, par); - ArrayList<ReadTask> tasks = new ArrayList<ReadTask>(); + ArrayList<ReadTask> tasks = new ArrayList<>(); for( InputSplit split : splits ){ ReadTask t = new ReadTask(split, informat, job, dest, rlen, clen, matrixMarket); tasks.add(t); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java index b5aec06..7f83abe 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlockParallel.java @@ -70,7 +70,7 @@ public class WriterBinaryBlockParallel extends WriterBinaryBlock try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>(); + ArrayList<WriteFileTask> tasks = new ArrayList<>(); int blklen = (int)Math.ceil((double)rlen / brlen / numThreads) * brlen; for(int i=0; i<numThreads & i*blklen<rlen; i++) { Path newPath = new Path(path, IOUtilFunctions.getPartFileName(i)); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java index afc1bdb..b8f9815 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarketParallel.java @@ -68,7 +68,7 @@ public class WriterMatrixMarketParallel extends WriterMatrixMarket try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteMMTask> tasks = new ArrayList<WriteMMTask>(); + ArrayList<WriteMMTask> tasks = new ArrayList<>(); int blklen = (int)Math.ceil((double)rlen / numThreads); for(int i=0; i<numThreads & i*blklen<rlen; i++) { Path newPath = new Path(path, IOUtilFunctions.getPartFileName(i)); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java index f5c62ac..7b39eb3 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java @@ -285,7 +285,7 @@ public class WriterTextCSV extends MatrixWriter if (fs.isDirectory(srcFilePath)) { // compute sorted order among part files - ArrayList<Path> files=new ArrayList<Path>(); + ArrayList<Path> files=new ArrayList<>(); for(FileStatus stat: fs.listStatus(srcFilePath, CSVReblockMR.hiddenFileFilter)) files.add(stat.getPath()); Collections.sort(files); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java index 173f602..2bef173 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSVParallel.java @@ -45,6 +45,7 @@ public class WriterTextCSVParallel extends WriterTextCSV super( props ); } + @Override protected void writeCSVMatrixToHDFS(Path path, JobConf job, FileSystem fs, MatrixBlock src, CSVFileFormatProperties csvprops) throws IOException { @@ -70,7 +71,7 @@ public class WriterTextCSVParallel extends WriterTextCSV try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteCSVTask> tasks = new ArrayList<WriteCSVTask>(); + ArrayList<WriteCSVTask> tasks = new ArrayList<>(); int rlen = src.getNumRows(); int blklen = (int)Math.ceil((double)rlen / numThreads); for(int i=0; i<numThreads & i*blklen<rlen; i++) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java index ec4b042..63f36c2 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCellParallel.java @@ -67,7 +67,7 @@ public class WriterTextCellParallel extends WriterTextCell try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteTextTask> tasks = new ArrayList<WriteTextTask>(); + ArrayList<WriteTextTask> tasks = new ArrayList<>(); int blklen = (int)Math.ceil((double)rlen / numThreads); for(int i=0; i<numThreads & i*blklen<rlen; i++) { Path newPath = new Path(path, IOUtilFunctions.getPartFileName(i)); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java index 3bd9758..a7df822 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java @@ -26,7 +26,7 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.HashSet; -import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -118,8 +118,8 @@ public class CSVReblockMR out.writeLong(count); } - public String toString() - { + @Override + public String toString() { return filename+", "+fileOffset+", "+count; } @@ -180,8 +180,8 @@ public class CSVReblockMR public long[] rlens=null; public long[] clens=null; - public String toString() - { + @Override + public String toString() { StringBuilder sb = new StringBuilder(); sb.append(counterFile.toString()); sb.append("\n"); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java index f18a811..b4e6eda 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java @@ -111,6 +111,7 @@ public class CombineMR } + @Override public void configure(JobConf job) { super.configure(job); @@ -123,7 +124,7 @@ public class CombineMR for(int i=0; i<resultIndexes.length; i++) { MatrixCharacteristics stat=MRJobConfiguration.getMatrixCharacteristicsForOutput(job, resultIndexes[i]); - outputBlockSizes.put(resultIndexes[i], new Pair<Integer, Integer>(stat.getRowsPerBlock(), stat.getColsPerBlock())); + outputBlockSizes.put(resultIndexes[i], new Pair<>(stat.getRowsPerBlock(), stat.getColsPerBlock())); } for(MRInstruction ins: comb_instructions) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java index 7af295f..cb7ca57 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java @@ -153,8 +153,8 @@ public class MatrixCharacteristics implements Serializable return (long) Math.ceil((double)getCols() / getColsPerBlock()); } - public String toString() - { + @Override + public String toString() { return "["+numRows+" x "+numColumns+", nnz="+nonZero +", blocks ("+numRowsPerBlock+" x "+numColumnsPerBlock+")]"; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java index 7a3b474..99d028c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java @@ -27,7 +27,7 @@ import java.util.Arrays; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.filecache.DistributedCache; +import org.apache.hadoop.mapreduce.filecache.DistributedCache; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.DoubleWritable; @@ -117,7 +117,7 @@ public class SortMR private static ArrayList<WritableComparable> readPartitions(FileSystem fs, Path p, JobConf job) throws IOException { - ArrayList<WritableComparable> parts = new ArrayList<WritableComparable>(); + ArrayList<WritableComparable> parts = new ArrayList<>(); SequenceFile.Reader reader = null; try { @@ -189,7 +189,7 @@ public class SortMR //setup input/output paths Path inputDir = new Path(input); inputDir = inputDir.makeQualified(inputDir.getFileSystem(job)); - SamplingSortMRInputFormat.setInputPaths(job, inputDir); + FileInputFormat.setInputPaths(job, inputDir); Path outpath = new Path(tmpOutput); FileOutputFormat.setOutputPath(job, outpath); MapReduceTool.deleteFileIfExistOnHDFS(outpath, job); @@ -336,58 +336,58 @@ public class SortMR private static boolean runStitchupJob(String input, long rlen, long clen, int brlen, int bclen, long[] counts, int numReducers, int replication, String output) - throws Exception - { - JobConf job = new JobConf(SortMR.class); - job.setJobName("SortIndexesMR"); - - //setup input/output paths - Path inpath = new Path(input); - Path outpath = new Path(output); - FileInputFormat.setInputPaths(job, inpath); - FileOutputFormat.setOutputPath(job, outpath); - MapReduceTool.deleteFileIfExistOnHDFS(outpath, job); - - //set number of reducers (1 if local mode) - if( InfrastructureAnalyzer.isLocalMode(job) ) - job.setNumReduceTasks(1); - else - MRJobConfiguration.setNumReducers(job, numReducers, numReducers); - - //setup input/output format - InputInfo iinfo = InputInfo.BinaryBlockInputInfo; - OutputInfo oinfo = OutputInfo.BinaryBlockOutputInfo; - job.setInputFormat(iinfo.inputFormatClass); - job.setOutputFormat(oinfo.outputFormatClass); - CompactInputFormat.setKeyValueClasses(job, MatrixIndexes.class, MatrixBlock.class); - - //setup mapper/reducer/output classes - MRJobConfiguration.setInputInfo(job, (byte)0, InputInfo.BinaryBlockInputInfo, brlen, bclen, ConvertTarget.BLOCK); - job.setMapperClass(IndexSortStitchupMapper.class); + throws Exception + { + JobConf job = new JobConf(SortMR.class); + job.setJobName("SortIndexesMR"); + + //setup input/output paths + Path inpath = new Path(input); + Path outpath = new Path(output); + FileInputFormat.setInputPaths(job, inpath); + FileOutputFormat.setOutputPath(job, outpath); + MapReduceTool.deleteFileIfExistOnHDFS(outpath, job); + + //set number of reducers (1 if local mode) + if( InfrastructureAnalyzer.isLocalMode(job) ) + job.setNumReduceTasks(1); + else + MRJobConfiguration.setNumReducers(job, numReducers, numReducers); + + //setup input/output format + InputInfo iinfo = InputInfo.BinaryBlockInputInfo; + OutputInfo oinfo = OutputInfo.BinaryBlockOutputInfo; + job.setInputFormat(iinfo.inputFormatClass); + job.setOutputFormat(oinfo.outputFormatClass); + CompactInputFormat.setKeyValueClasses(job, MatrixIndexes.class, MatrixBlock.class); + + //setup mapper/reducer/output classes + MRJobConfiguration.setInputInfo(job, (byte)0, InputInfo.BinaryBlockInputInfo, brlen, bclen, ConvertTarget.BLOCK); + job.setMapperClass(IndexSortStitchupMapper.class); job.setReducerClass(IndexSortStitchupReducer.class); job.setOutputKeyClass(oinfo.outputKeyClass); job.setOutputValueClass(oinfo.outputValueClass); - MRJobConfiguration.setBlockSize(job, (byte)0, brlen, bclen); - MRJobConfiguration.setMatricesDimensions(job, new byte[]{0}, new long[]{rlen}, new long[]{clen}); - - //compute shifted prefix sum of offsets and put into configuration - long[] cumsumCounts = new long[counts.length]; - long sum = 0; - for( int i=0; i<counts.length; i++ ) { - cumsumCounts[i] = sum; - sum += counts[i]; - } - job.set(SORT_INDEXES_OFFSETS, Arrays.toString(cumsumCounts)); - - //setup replication factor - job.setInt(MRConfigurationNames.DFS_REPLICATION, replication); - + MRJobConfiguration.setBlockSize(job, (byte)0, brlen, bclen); + MRJobConfiguration.setMatricesDimensions(job, new byte[]{0}, new long[]{rlen}, new long[]{clen}); + + //compute shifted prefix sum of offsets and put into configuration + long[] cumsumCounts = new long[counts.length]; + long sum = 0; + for( int i=0; i<counts.length; i++ ) { + cumsumCounts[i] = sum; + sum += counts[i]; + } + job.set(SORT_INDEXES_OFFSETS, Arrays.toString(cumsumCounts)); + + //setup replication factor + job.setInt(MRConfigurationNames.DFS_REPLICATION, replication); + //set unique working dir MRJobConfiguration.setUniqueWorkingDir(job); //run mr job - RunningJob runJob = JobClient.runJob(job); - - return runJob.isSuccessful(); + RunningJob runJob = JobClient.runJob(job); + + return runJob.isSuccessful(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java index 2dd29af..1f1fad6 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/AddDummyWeightConverter.java @@ -26,10 +26,9 @@ import org.apache.hadoop.io.Writable; @SuppressWarnings("rawtypes") public class AddDummyWeightConverter implements Converter<Writable, Writable, MatrixIndexes, WeightedPair> { - private Converter toCellConverter=null; private WeightedPair outValue=new WeightedPair(); - private Pair<MatrixIndexes, WeightedPair> pair=new Pair<MatrixIndexes, WeightedPair>(); + private Pair<MatrixIndexes, WeightedPair> pair=new Pair<>(); private int rlen; private int clen; public AddDummyWeightConverter() http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java index 63e0a32..9c5d0d2 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToBinaryCellConverter.java @@ -29,7 +29,6 @@ import org.apache.sysml.runtime.util.UtilFunctions; public class BinaryBlockToBinaryCellConverter implements Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixCell> { - private Iterator<IJV> sparseIterator=null; private double[] denseArray=null; private int denseArraySize=0; @@ -43,7 +42,7 @@ Converter<MatrixIndexes, MatrixBlock, MatrixIndexes, MatrixCell> private MatrixIndexes returnIndexes=new MatrixIndexes(); private MatrixCell cell=new MatrixCell(); - private Pair<MatrixIndexes, MatrixCell> pair=new Pair<MatrixIndexes, MatrixCell>(returnIndexes, cell); + private Pair<MatrixIndexes, MatrixCell> pair=new Pair<>(returnIndexes, cell); private void reset() { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java index bfbd5bd..68bf038 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToRowBlockConverter.java @@ -43,7 +43,7 @@ public class BinaryBlockToRowBlockConverter implements Converter<MatrixIndexes, { _destIx = new MatrixIndexes(); _destBlock = new MatrixBlock(); - _pair=new Pair<MatrixIndexes, MatrixBlock>(_destIx, _destBlock); + _pair=new Pair<>(_destIx, _destBlock); } private void reset() http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java index 47c03fc..98396d2 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryBlockToTextCellConverter.java @@ -44,7 +44,7 @@ Converter<MatrixIndexes, MatrixBlock, NullWritable, Text> private int bcolumn; private Text value=new Text(); - private Pair<NullWritable, Text> pair=new Pair<NullWritable, Text>(NullWritable.get(), value); + private Pair<NullWritable, Text> pair=new Pair<>(NullWritable.get(), value); private void reset() { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java index 469e8ac..875514c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToRowBlockConverter.java @@ -26,7 +26,7 @@ public class BinaryCellToRowBlockConverter implements Converter<MatrixIndexes, M private MatrixIndexes returnIndexes=new MatrixIndexes(); private MatrixBlock rowBlock = new MatrixBlock(); - private Pair<MatrixIndexes, MatrixBlock> pair=new Pair<MatrixIndexes, MatrixBlock>(returnIndexes, rowBlock); + private Pair<MatrixIndexes, MatrixBlock> pair=new Pair<>(returnIndexes, rowBlock); private boolean hasValue=false; @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java index 9293d16..bd3486f 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/BinaryCellToTextConverter.java @@ -29,7 +29,7 @@ implements Converter<MatrixIndexes, MatrixCell, NullWritable, Text> { private Text value=new Text(); - private Pair<NullWritable, Text> pair=new Pair<NullWritable, Text>(NullWritable.get(), value); + private Pair<NullWritable, Text> pair=new Pair<>(NullWritable.get(), value); private boolean hasValue=false; @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java index 7da4979..cd9a995 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/CM_N_COVCell.java @@ -43,11 +43,11 @@ import org.apache.sysml.runtime.util.IndexRange; @SuppressWarnings("rawtypes") public class CM_N_COVCell extends MatrixValue implements WritableComparable -{ +{ private CM_COV_Object cm=new CM_COV_Object(); - public String toString() - { + @Override + public String toString() { return cm.toString(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java b/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java index 2f180b0..c66b5c6 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java @@ -59,6 +59,7 @@ public class ConvolutionParameters implements Serializable { return false; } + @Override public String toString() { return "(NCHW=[" + N + " " + C + " " + H + " " + W + "], KCRS=[" + K + " " + R + " " + S + "], stride=[" + stride_h + "," + stride_w + "], pad=[" + pad_h + "," + pad_w + "])"; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index 23dbd8b..a79f6c2 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -233,7 +233,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable * @return map of column name keys and id values */ public Map<String,Integer> getColumnNameIDMap() { - Map<String, Integer> ret = new HashMap<String, Integer>(); + Map<String, Integer> ret = new HashMap<>(); for( int j=0; j<getNumColumns(); j++ ) ret.put(getColumnName(j), j+1); return ret; @@ -1058,7 +1058,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable } //construct recode map - HashMap<String,Long> map = new HashMap<String,Long>(); + HashMap<String,Long> map = new HashMap<>(); Array ldata = _coldata[col]; for( int i=0; i<getNumRows(); i++ ) { Object val = ldata.get(i); @@ -1292,6 +1292,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public abstract void setNz(int rl, int ru, Array value); public abstract void append(String value); public abstract void append(T value); + @Override public abstract Array clone(); public abstract Array slice(int rl, int ru); public abstract void reset(int size); @@ -1303,25 +1304,31 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public StringArray(String[] data) { _data = data; _size = _data.length; - } + } + @Override public String get(int index) { return _data[index]; } + @Override public void set(int index, String value) { _data[index] = value; } + @Override public void set(int rl, int ru, Array value) { set(rl, ru, value, 0); } + @Override public void set(int rl, int ru, Array value, int rlSrc) { System.arraycopy(((StringArray)value)._data, rlSrc, _data, rl, ru-rl+1); } + @Override public void setNz(int rl, int ru, Array value) { String[] data2 = ((StringArray)value)._data; for( int i=rl; i<ru+1; i++ ) if( data2[i]!=null ) _data[i] = data2[i]; } + @Override public void append(String value) { if( _data.length <= _size ) _data = Arrays.copyOf(_data, newSize()); @@ -1338,12 +1345,15 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable _data[i] = (!tmp.isEmpty()) ? tmp : null; } } + @Override public Array clone() { return new StringArray(Arrays.copyOf(_data, _size)); } + @Override public Array slice(int rl, int ru) { return new StringArray(Arrays.copyOfRange(_data,rl,ru+1)); } + @Override public void reset(int size) { if( _data.length < size ) _data = new String[size]; @@ -1357,28 +1367,35 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public BooleanArray(boolean[] data) { _data = data; _size = _data.length; - } + } + @Override public Boolean get(int index) { return _data[index]; } + @Override public void set(int index, Boolean value) { _data[index] = (value!=null) ? value : false; } + @Override public void set(int rl, int ru, Array value) { set(rl, ru, value, 0); } + @Override public void set(int rl, int ru, Array value, int rlSrc) { System.arraycopy(((BooleanArray)value)._data, rlSrc, _data, rl, ru-rl+1); } + @Override public void setNz(int rl, int ru, Array value) { boolean[] data2 = ((BooleanArray)value)._data; for( int i=rl; i<ru+1; i++ ) if( data2[i] ) _data[i] = data2[i]; } + @Override public void append(String value) { append(Boolean.parseBoolean(value)); } + @Override public void append(Boolean value) { if( _data.length <= _size ) _data = Arrays.copyOf(_data, newSize()); @@ -1393,12 +1410,15 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable for( int i=0; i<_size; i++ ) _data[i] = in.readBoolean(); } + @Override public Array clone() { return new BooleanArray(Arrays.copyOf(_data, _size)); } + @Override public Array slice(int rl, int ru) { return new BooleanArray(Arrays.copyOfRange(_data,rl,ru+1)); } + @Override public void reset(int size) { if( _data.length < size ) _data = new boolean[size]; @@ -1412,28 +1432,35 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public LongArray(long[] data) { _data = data; _size = _data.length; - } + } + @Override public Long get(int index) { return _data[index]; } + @Override public void set(int index, Long value) { _data[index] = (value!=null) ? value : 0L; } + @Override public void set(int rl, int ru, Array value) { set(rl, ru, value, 0); } + @Override public void set(int rl, int ru, Array value, int rlSrc) { System.arraycopy(((LongArray)value)._data, rlSrc, _data, rl, ru-rl+1); } + @Override public void setNz(int rl, int ru, Array value) { long[] data2 = ((LongArray)value)._data; for( int i=rl; i<ru+1; i++ ) if( data2[i]!=0 ) _data[i] = data2[i]; } + @Override public void append(String value) { append((value!=null)?Long.parseLong(value):null); } + @Override public void append(Long value) { if( _data.length <= _size ) _data = Arrays.copyOf(_data, newSize()); @@ -1448,12 +1475,15 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable for( int i=0; i<_size; i++ ) _data[i] = in.readLong(); } + @Override public Array clone() { return new LongArray(Arrays.copyOf(_data, _size)); } + @Override public Array slice(int rl, int ru) { return new LongArray(Arrays.copyOfRange(_data,rl,ru+1)); } + @Override public void reset(int size) { if( _data.length < size ) _data = new long[size]; @@ -1467,28 +1497,35 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable public DoubleArray(double[] data) { _data = data; _size = _data.length; - } + } + @Override public Double get(int index) { return _data[index]; } + @Override public void set(int index, Double value) { _data[index] = (value!=null) ? value : 0d; } + @Override public void set(int rl, int ru, Array value) { set(rl,ru, value, 0); } + @Override public void set(int rl, int ru, Array value, int rlSrc) { System.arraycopy(((DoubleArray)value)._data, rlSrc, _data, rl, ru-rl+1); } + @Override public void setNz(int rl, int ru, Array value) { double[] data2 = ((DoubleArray)value)._data; for( int i=rl; i<ru+1; i++ ) if( data2[i]!=0 ) _data[i] = data2[i]; } + @Override public void append(String value) { append((value!=null)?Double.parseDouble(value):null); } + @Override public void append(Double value) { if( _data.length <= _size ) _data = Arrays.copyOf(_data, newSize()); @@ -1503,12 +1540,15 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable for( int i=0; i<_size; i++ ) _data[i] = in.readDouble(); } + @Override public Array clone() { return new DoubleArray(Arrays.copyOf(_data, _size)); } + @Override public Array slice(int rl, int ru) { return new DoubleArray(Arrays.copyOfRange(_data,rl,ru+1)); } + @Override public void reset(int size) { if( _data.length < size ) _data = new double[size]; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java index f19fedd..f4c6c7a 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/IJV.java @@ -51,6 +51,7 @@ public class IJV return _v; } + @Override public String toString() { return "("+_i+", "+_j+"): "+_v; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java index de4b425..3290609 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/IdenticalConverter.java @@ -24,8 +24,7 @@ import org.apache.hadoop.io.Writable; public class IdenticalConverter implements Converter<Writable, Writable, Writable, Writable> { - - private Pair<Writable, Writable> pair=new Pair<Writable, Writable>(); + private Pair<Writable, Writable> pair=new Pair<>(); private boolean hasValue=false; @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java index 1ec5290..4a99adf 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixAgg.java @@ -248,7 +248,7 @@ public class LibMatrixAgg //(currently: always parallelization over number of rows) try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<AggTask> tasks = new ArrayList<AggTask>(); + ArrayList<AggTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)m/k)); for( int i=0; i<k & i*blklen<m; i++ ) { tasks.add( (uaop.indexFn instanceof ReduceCol) ? @@ -346,10 +346,10 @@ public class LibMatrixAgg //step 1: compute aggregates per row partition AggType uaoptype = getAggType(uaop); - ArrayList<PartialAggTask> tasks = new ArrayList<PartialAggTask>(); + ArrayList<PartialAggTask> tasks = new ArrayList<>(); for( int i=0; i<k & i*blklen<m; i++ ) tasks.add( new PartialAggTask(in, new MatrixBlock(mk,n2,false), uaoptype, uaop, i*blklen, Math.min((i+1)*blklen, m)) ); - List<Future<Object>> taskret = pool.invokeAll(tasks); + List<Future<Object>> taskret = pool.invokeAll(tasks); for( Future<Object> task : taskret ) task.get(); //check for errors @@ -364,7 +364,7 @@ public class LibMatrixAgg MatrixBlock tmp2 = cumaggregateUnaryMatrix(tmp, new MatrixBlock(tasks.size(), n2, false), uop); //step 3: compute final cumulative aggregate - ArrayList<CumAggTask> tasks2 = new ArrayList<CumAggTask>(); + ArrayList<CumAggTask> tasks2 = new ArrayList<>(); for( int i=0; i<k & i*blklen<m; i++ ) { double[] agg = (i==0)? null : DataConverter.convertToDoubleVector(tmp2.sliceOperations(i-1, i-1, 0, n2-1, new MatrixBlock()), false); @@ -437,7 +437,7 @@ public class LibMatrixAgg try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<AggTernaryTask> tasks = new ArrayList<AggTernaryTask>(); + ArrayList<AggTernaryTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)in1.rlen/k)); IndexFunction ixFn = op.indexFn; for( int i=0; i<k & i*blklen<in1.rlen; i++ ) @@ -511,7 +511,7 @@ public class LibMatrixAgg //(currently: parallelization over columns to avoid additional memory requirements) try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<GrpAggTask> tasks = new ArrayList<GrpAggTask>(); + ArrayList<GrpAggTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)target.clen/k)); for( int i=0; i<k & i*blklen<target.clen; i++ ) tasks.add( new GrpAggTask(groups, target, weights, result, numGroups, op, i*blklen, Math.min((i+1)*blklen, target.clen)) ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCuDNNInputRowFetcher.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCuDNNInputRowFetcher.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCuDNNInputRowFetcher.java index a21dae0..581607e 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCuDNNInputRowFetcher.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCuDNNInputRowFetcher.java @@ -47,9 +47,9 @@ public class LibMatrixCuDNNInputRowFetcher implements java.lang.AutoCloseable { */ public LibMatrixCuDNNInputRowFetcher(GPUContext gCtx, String instName, MatrixObject image) throws DMLRuntimeException { this.gCtx = gCtx; this.instName = instName; - numColumns = LibMatrixCuDNN.toInt(image.getNumColumns()); - isInputInSparseFormat = LibMatrixCuDNN.isInSparseFormat(gCtx, image); - inPointer = isInputInSparseFormat ? LibMatrixCuDNN.getSparsePointer(gCtx, image, instName) : LibMatrixCuDNN.getDensePointerForCuDNN(gCtx, image, instName); + numColumns = LibMatrixCUDA.toInt(image.getNumColumns()); + isInputInSparseFormat = LibMatrixCUDA.isInSparseFormat(gCtx, image); + inPointer = isInputInSparseFormat ? LibMatrixCUDA.getSparsePointer(gCtx, image, instName) : LibMatrixCuDNN.getDensePointerForCuDNN(gCtx, image, instName); outPointer = gCtx.allocate(numColumns*Sizeof.DOUBLE); } /** @@ -65,10 +65,10 @@ public class LibMatrixCuDNNInputRowFetcher implements java.lang.AutoCloseable { cudaMemset(outPointer, 0, numColumns*Sizeof.DOUBLE); jcuda.runtime.JCuda.cudaDeviceSynchronize(); if(GPUStatistics.DISPLAY_STATISTICS) GPUStatistics.maintainCPMiscTimes(instName, GPUInstruction.MISC_TIMER_SET_ZERO, System.nanoTime() - t0); - LibMatrixCuDNN.sliceSparseDense(gCtx, instName, (CSRPointer)inPointer, outPointer, n, n, 0, LibMatrixCuDNN.toInt(numColumns-1), numColumns); + LibMatrixCUDA.sliceSparseDense(gCtx, instName, (CSRPointer)inPointer, outPointer, n, n, 0, LibMatrixCUDA.toInt(numColumns-1), numColumns); } else { - LibMatrixCuDNN.sliceDenseDense(gCtx, instName, (Pointer)inPointer, outPointer, n, n, 0, LibMatrixCuDNN.toInt(numColumns-1), numColumns); + LibMatrixCUDA.sliceDenseDense(gCtx, instName, (Pointer)inPointer, outPointer, n, n, 0, LibMatrixCUDA.toInt(numColumns-1), numColumns); } return outPointer; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java index 4dc9460..90dda05 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java @@ -342,7 +342,7 @@ public class LibMatrixDatagen try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<RandTask> tasks = new ArrayList<RandTask>(); + ArrayList<RandTask> tasks = new ArrayList<>(); int blklen = ((int)(Math.ceil((double)parnb/k))); for( int i=0; i<k & i*blklen<parnb; i++ ) { int rl = parcol ? 0 : i*blklen; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java index 190d807..dbb65fe 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java @@ -208,7 +208,7 @@ public class LibMatrixMult //(currently: always parallelization over number of rows) try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<MatrixMultTask> tasks = new ArrayList<MatrixMultTask>(); + ArrayList<MatrixMultTask> tasks = new ArrayList<>(); int nk = (pm2r||pm2c) ? k : UtilFunctions.roundToNext(Math.min(8*k,num/32), k); ArrayList<Integer> blklens = getBalancedBlockSizes(num, nk); for( int i=0, lb=0; i<blklens.size(); lb+=blklens.get(i), i++ ) @@ -321,7 +321,7 @@ public class LibMatrixMult //(currently: always parallelization over number of rows) try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<MatrixMultChainTask> tasks = new ArrayList<MatrixMultChainTask>(); + ArrayList<MatrixMultChainTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)mX.rlen/k)); blklen += (blklen%24 != 0)?24-blklen%24:0; for( int i=0; i<k & i*blklen<mX.rlen; i++ ) @@ -403,7 +403,7 @@ public class LibMatrixMult //core multi-threaded matrix mult computation try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<MatrixMultTransposeTask> tasks = new ArrayList<MatrixMultTransposeTask>(); + ArrayList<MatrixMultTransposeTask> tasks = new ArrayList<>(); //load balance via #tasks=2k due to triangular shape int blklen = (int)(Math.ceil((double)ret.rlen/(2*k))); for( int i=0; i<2*k & i*blklen<ret.rlen; i++ ) @@ -483,7 +483,7 @@ public class LibMatrixMult try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<MatrixMultPermuteTask> tasks = new ArrayList<MatrixMultPermuteTask>(); + ArrayList<MatrixMultPermuteTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)pm1.rlen/k)); for( int i=0; i<k & i*blklen<pm1.rlen; i++ ) tasks.add(new MatrixMultPermuteTask(pm1, m2, ret1, ret2, i*blklen, Math.min((i+1)*blklen, pm1.rlen))); @@ -558,7 +558,7 @@ public class LibMatrixMult try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<MatrixMultWSLossTask> tasks = new ArrayList<MatrixMultWSLossTask>(); + ArrayList<MatrixMultWSLossTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)mX.rlen/k)); for( int i=0; i<k & i*blklen<mX.rlen; i++ ) tasks.add(new MatrixMultWSLossTask(mX, mU, mV, mW, wt, i*blklen, Math.min((i+1)*blklen, mX.rlen))); @@ -634,7 +634,7 @@ public class LibMatrixMult try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<MatrixMultWSigmoidTask> tasks = new ArrayList<MatrixMultWSigmoidTask>(); + ArrayList<MatrixMultWSigmoidTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)mW.rlen/k)); for( int i=0; i<k & i*blklen<mW.rlen; i++ ) tasks.add(new MatrixMultWSigmoidTask(mW, mU, mV, ret, wt, i*blklen, Math.min((i+1)*blklen, mW.rlen))); @@ -749,7 +749,7 @@ public class LibMatrixMult try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<MatrixMultWDivTask> tasks = new ArrayList<MatrixMultWDivTask>(); + ArrayList<MatrixMultWDivTask> tasks = new ArrayList<>(); //create tasks (for wdivmm-left, parallelization over columns; //for wdivmm-right, parallelization over rows; both ensure disjoint results) if( wt.isLeft() ) { @@ -826,7 +826,7 @@ public class LibMatrixMult try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<MatrixMultWCeTask> tasks = new ArrayList<MatrixMultWCeTask>(); + ArrayList<MatrixMultWCeTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)mW.rlen/k)); for( int i=0; i<k & i*blklen<mW.rlen; i++ ) tasks.add(new MatrixMultWCeTask(mW, mU, mV, eps, wt, i*blklen, Math.min((i+1)*blklen, mW.rlen))); @@ -898,7 +898,7 @@ public class LibMatrixMult try { ExecutorService pool = Executors.newFixedThreadPool(k); - ArrayList<MatrixMultWuTask> tasks = new ArrayList<MatrixMultWuTask>(); + ArrayList<MatrixMultWuTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)mW.rlen/k)); for( int i=0; i<k & i*blklen<mW.rlen; i++ ) tasks.add(new MatrixMultWuTask(mW, mU, mV, ret, wt, fn, i*blklen, Math.min((i+1)*blklen, mW.rlen))); @@ -3646,7 +3646,7 @@ public class LibMatrixMult } private static ArrayList<Integer> getBalancedBlockSizes(int len, int k) { - ArrayList<Integer> ret = new ArrayList<Integer>(); + ArrayList<Integer> ret = new ArrayList<>(); int base = len / k; int rest = len % k; for( int i=0; i<k; i++ ) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java index d2d3d86..76002fe 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java @@ -191,7 +191,7 @@ public class LibMatrixReorg //pre-processing (compute nnz per column once for sparse) int[] cnt = null; if( in.sparse && out.sparse ) { - ArrayList<CountNnzTask> tasks = new ArrayList<CountNnzTask>(); + ArrayList<CountNnzTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)in.rlen/k)); for( int i=0; i<k & i*blklen<in.rlen; i++ ) tasks.add(new CountNnzTask(in, i*blklen, Math.min((i+1)*blklen, in.rlen))); @@ -200,7 +200,7 @@ public class LibMatrixReorg cnt = mergeNnzCounts(cnt, rtask.get()); } //compute actual transpose and check for errors - ArrayList<TransposeTask> tasks = new ArrayList<TransposeTask>(); + ArrayList<TransposeTask> tasks = new ArrayList<>(); boolean row = (in.sparse || in.rlen >= in.clen) && !out.sparse; int len = row ? in.rlen : in.clen; int blklen = (int)(Math.ceil((double)len/k)); @@ -504,7 +504,7 @@ public class LibMatrixReorg reshapeDense(mbIn, row_offset, col_offset, rblk, mcIn, mcOut, rowwise); //prepare output - out = new ArrayList<IndexedMatrixValue>(); + out = new ArrayList<>(); for( Entry<MatrixIndexes, MatrixBlock> e : rblk.entrySet() ) out.add(new IndexedMatrixValue(e.getKey(),e.getValue())); @@ -567,7 +567,7 @@ public class LibMatrixReorg } //compute outputs (at most two output blocks) - HashMap<MatrixIndexes,IndexedMatrixValue> out = new HashMap<MatrixIndexes,IndexedMatrixValue>(); + HashMap<MatrixIndexes,IndexedMatrixValue> out = new HashMap<>(); MatrixBlock linData = (MatrixBlock) data.getValue(); MatrixBlock linOffset = (MatrixBlock) offset.getValue(); MatrixIndexes tmpIx = new MatrixIndexes(-1,-1); @@ -1429,7 +1429,7 @@ public class LibMatrixReorg private static Collection<MatrixIndexes> computeAllResultBlockIndexes( MatrixIndexes ixin, MatrixCharacteristics mcIn, MatrixCharacteristics mcOut, boolean rowwise ) { - HashSet<MatrixIndexes> ret = new HashSet<MatrixIndexes>(); + HashSet<MatrixIndexes> ret = new HashSet<>(); long row_offset = (ixin.getRowIndex()-1)*mcOut.getRowsPerBlock(); long col_offset = (ixin.getColumnIndex()-1)*mcOut.getColsPerBlock(); @@ -1661,7 +1661,7 @@ public class LibMatrixReorg if( in.sparse ) //SPARSE { - SparseBlock a = in.sparseBlock; + SparseBlock a = in.sparseBlock; for ( int i=0; i < m; i++ ) rlen2 += (flags[i] = !a.isEmpty(i)) ? 1 : 0; } @@ -1931,7 +1931,7 @@ public class LibMatrixReorg else { try { ExecutorService pool = Executors.newFixedThreadPool( k ); - ArrayList<RExpandColsTask> tasks = new ArrayList<RExpandColsTask>(); + ArrayList<RExpandColsTask> tasks = new ArrayList<>(); int blklen = (int)(Math.ceil((double)rlen/k/8)); for( int i=0; i<8*k & i*blklen<rlen; i++ ) tasks.add(new RExpandColsTask(in, ret, http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index 432b18a..66a5d9a 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -437,6 +437,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab //////// // Metadata information + @Override public int getNumRows() { return rlen; } @@ -451,6 +452,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab rlen = r; } + @Override public int getNumColumns() { return clen; } @@ -459,6 +461,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab clen = c; } + @Override public long getNonZeros() { return nonZeros; } @@ -916,6 +919,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab * * @return true if sparse */ + @Override public boolean isInSparseFormat() { return sparse; } @@ -2898,6 +2902,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab } } + @Override public MatrixValue binaryOperations(BinaryOperator op, MatrixValue thatValue, MatrixValue result) throws DMLRuntimeException { @@ -2927,6 +2932,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab return ret; } + @Override public void binaryOperationsInPlace(BinaryOperator op, MatrixValue thatValue) throws DMLRuntimeException { @@ -2946,9 +2952,8 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab //core binary cell operation LibMatrixBincell.bincellOpInPlace(this, that, op); } - - + @Override public void incrementalAggregate(AggregateOperator aggOp, MatrixValue correction, MatrixValue newWithCorrection) throws DMLRuntimeException @@ -3193,6 +3198,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab throw new DMLRuntimeException("unrecognized correctionLocation: "+aggOp.correctionLocation); } + @Override public void incrementalAggregate(AggregateOperator aggOp, MatrixValue newWithCorrection) throws DMLRuntimeException { @@ -3940,6 +3946,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab dest.recomputeNonZeros(); } + @Override public void sliceOperations(ArrayList<IndexedMatrixValue> outlist, IndexRange range, int rowCut, int colCut, int normalBlockRowFactor, int normalBlockColFactor, int boundaryRlen, int boundaryClen) { @@ -4095,6 +4102,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab } } + @Override public MatrixValue zeroOutOperations(MatrixValue result, IndexRange range, boolean complementary) throws DMLRuntimeException { @@ -4203,6 +4211,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab return result; } + @Override public MatrixValue aggregateUnaryOperations(AggregateUnaryOperator op, MatrixValue result, int blockingFactorRow, int blockingFactorCol, MatrixIndexes indexesIn) throws DMLRuntimeException @@ -4211,7 +4220,7 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab blockingFactorRow, blockingFactorCol, indexesIn, false); } - + @Override public MatrixValue aggregateUnaryOperations(AggregateUnaryOperator op, MatrixValue result, int blockingFactorRow, int blockingFactorCol, MatrixIndexes indexesIn, boolean inCP) throws DMLRuntimeException @@ -4835,12 +4844,14 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab return sum_wt; } + @Override public MatrixValue aggregateBinaryOperations(MatrixIndexes m1Index, MatrixValue m1Value, MatrixIndexes m2Index, MatrixValue m2Value, MatrixValue result, AggregateBinaryOperator op ) throws DMLRuntimeException { return aggregateBinaryOperations(m1Value, m2Value, result, op); } + @Override public MatrixValue aggregateBinaryOperations(MatrixValue m1Value, MatrixValue m2Value, MatrixValue result, AggregateBinaryOperator op) throws DMLRuntimeException { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java index 62db808..7e82c43 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixCell.java @@ -149,7 +149,8 @@ public class MatrixCell extends MatrixValue implements WritableComparable, Seria value=0; } - public void reset(int rl, int cl, boolean sp, long nnzs) { + @Override + public void reset(int rl, int cl, boolean sp, long nnzs) { value=0; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java index 6e9986a..e0a9579 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixValue.java @@ -61,6 +61,7 @@ public abstract class MatrixValue implements WritableComparable return (this.row == cthat.row && this.column == cthat.column); } + @Override public int hashCode() { return UtilFunctions.longHashCode((row << 16) + column); } @@ -69,8 +70,9 @@ public abstract class MatrixValue implements WritableComparable row = r; column = c; } - public String toString() - { + + @Override + public String toString() { return "("+row+","+column+")"; } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java index 73a35be..1f72692 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/OperationsOnMatrixValues.java @@ -299,7 +299,7 @@ public class OperationsOnMatrixValues throws DMLRuntimeException { IndexedMatrixValue imv = new IndexedMatrixValue(new MatrixIndexes(iix, jix), (MatrixBlock)in); - ArrayList<IndexedMatrixValue> outlist = new ArrayList<IndexedMatrixValue>(); + ArrayList<IndexedMatrixValue> outlist = new ArrayList<>(); performSlice(imv, ixrange, brlen, bclen, outlist); return SparkUtils.fromIndexedMatrixBlockToPair(outlist); @@ -452,8 +452,8 @@ public class OperationsOnMatrixValues public static ArrayList performSlice(IndexRange ixrange, int brlen, int bclen, int iix, int jix, FrameBlock in) throws DMLRuntimeException { - Pair<Long, FrameBlock> lfp = new Pair<Long, FrameBlock>(new Long(((iix-1)*brlen)+1), in); - ArrayList<Pair<Long, FrameBlock>> outlist = new ArrayList<Pair<Long, FrameBlock>>(); + Pair<Long, FrameBlock> lfp = new Pair<>(new Long(((iix-1)*brlen)+1), in); + ArrayList<Pair<Long, FrameBlock>> outlist = new ArrayList<>(); performSlice(lfp, ixrange, brlen, bclen, outlist); return outlist; @@ -512,7 +512,7 @@ public class OperationsOnMatrixValues { ValueType[] schema = Arrays.copyOfRange(block.getSchema(), (int)tmpRange.colStart, (int)tmpRange.colEnd+1); long iResultIndex = Math.max(((r-1)*brlen - ixrange.rowStart + 1), 0); - Pair<Long,FrameBlock> out=new Pair<Long,FrameBlock>(new Long(iResultIndex+1), new FrameBlock(schema)); + Pair<Long,FrameBlock> out=new Pair<>(new Long(iResultIndex+1), new FrameBlock(schema)); outlist.add(out); } @@ -571,7 +571,7 @@ public class OperationsOnMatrixValues resultBlock.ensureAllocatedColumns(iRHSRows); resultBlock = resultBlock.leftIndexingOperations(slicedRHSBlk, lhs_lrl, lhs_lru, lhs_lcl, lhs_lcu, new FrameBlock()); - outlist.add(new Pair<Long, FrameBlock>((leftRowIndex-1)*brlenLeft+1, resultBlock)); + outlist.add(new Pair<>((leftRowIndex-1)*brlenLeft+1, resultBlock)); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java index 443644a..759546f 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/PartialBlock.java @@ -75,8 +75,8 @@ public class PartialBlock implements WritableComparable<PartialBlock> out.writeDouble(value); } - public String toString() - { + @Override + public String toString() { return "["+row+", "+column+"]:"+value; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java index 9f527d4..63b8071 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseBlockCOO.java @@ -234,6 +234,7 @@ public class SparseBlockCOO extends SparseBlock return pos(ru) - pos(rl); } + @Override public long size(int rl, int ru, int cl, int cu) { long nnz = 0; for(int i=rl; i<ru; i++) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowScalar.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowScalar.java b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowScalar.java index a86eee9..fa95e4c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowScalar.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/SparseRowScalar.java @@ -54,6 +54,7 @@ public final class SparseRowScalar extends SparseRow implements Serializable return new double[]{value}; } + @Override public int[] indexes() { return new int[]{index}; } @@ -86,10 +87,12 @@ public final class SparseRowScalar extends SparseRow implements Serializable value = v; } + @Override public double get(int col) { return (index==col) ? value : 0; } + @Override public void sort() { //do nothing } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java b/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java index 5310669..b7ac903 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/Tagged.java @@ -72,8 +72,8 @@ public class Tagged<T extends WritableComparable> implements WritableComparable< base.write(out); } - public String toString() - { + @Override + public String toString() { return base.toString()+" ~~ tag: "+tag; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java index bea0c48..ac10f2e 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedFirstSecondIndexes.java @@ -61,27 +61,23 @@ public class TaggedFirstSecondIndexes implements WritableComparable<TaggedFirstS setTag(other.tag); } - public String toString() - { + @Override + public String toString() { return "("+first+", "+second+") tag: "+tag; } - public byte getTag() - { + public byte getTag() { return tag; } - public long getFirstIndex() - { + public long getFirstIndex() { return first; } - public long getSecondIndex() - { + public long getSecondIndex() { return second; } - public void setIndexes(long i1, long i2) - { + public void setIndexes(long i1, long i2) { first=i1; second=i2; } @@ -91,7 +87,6 @@ public class TaggedFirstSecondIndexes implements WritableComparable<TaggedFirstS first=in.readLong(); tag=in.readByte(); second=in.readLong(); - } @Override @@ -99,7 +94,6 @@ public class TaggedFirstSecondIndexes implements WritableComparable<TaggedFirstS out.writeLong(first); out.writeByte(tag); out.writeLong(second); - } public int compareTo(TaggedFirstSecondIndexes other) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixIndexes.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixIndexes.java index 47e114e..1c23f7c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixIndexes.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedMatrixIndexes.java @@ -42,6 +42,7 @@ public class TaggedMatrixIndexes extends Tagged<MatrixIndexes> return "k: "+base+", tag: "+tag; } + @Override public void readFields(DataInput in) throws IOException { if( base == null ){ base = new MatrixIndexes(); @@ -50,6 +51,7 @@ public class TaggedMatrixIndexes extends Tagged<MatrixIndexes> tag=in.readByte(); } + @Override public void write(DataOutput out) throws IOException { base.write(out); out.writeByte(tag); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java index 4b298a4..3b3c3a4 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TaggedTripleIndexes.java @@ -51,22 +51,21 @@ public class TaggedTripleIndexes extends TaggedFirstSecondIndexes this.tag=that.tag; } - public String toString() - { + @Override + public String toString() { return "("+first+", "+second+") k: "+third+", tag: "+tag; } - public long getThirdIndex() - { + public long getThirdIndex() { return third; } - public void setIndexes(long i1, long i2, long i3) - { + public void setIndexes(long i1, long i2, long i3) { super.setIndexes(i1, i2); third=i3; } + @Override public void readFields(DataInput in) throws IOException { first=in.readLong(); second=in.readLong(); @@ -74,6 +73,7 @@ public class TaggedTripleIndexes extends TaggedFirstSecondIndexes tag=in.readByte(); } + @Override public void write(DataOutput out) throws IOException { out.writeLong(first); out.writeLong(second); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java index ebe49bd..3568a4f 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TextCellToRowBlockConverter.java @@ -27,10 +27,9 @@ import org.apache.sysml.runtime.util.FastStringTokenizer; public class TextCellToRowBlockConverter implements Converter<LongWritable, Text, MatrixIndexes, MatrixBlock> { - private MatrixIndexes indexes=new MatrixIndexes(); private MatrixBlock rowBlock=new MatrixBlock(); - private Pair<MatrixIndexes, MatrixBlock> pair=new Pair<MatrixIndexes, MatrixBlock>(indexes, rowBlock); + private Pair<MatrixIndexes, MatrixBlock> pair=new Pair<>(indexes, rowBlock); private FastStringTokenizer st = new FastStringTokenizer(' '); private boolean hasValue=false; private boolean toIgnore=false; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java index 93d6cc6..036743f 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TextToBinaryCellConverter.java @@ -29,10 +29,9 @@ import org.apache.sysml.runtime.util.FastStringTokenizer; public class TextToBinaryCellConverter implements Converter<LongWritable, Text, MatrixIndexes, MatrixCell> { - private MatrixIndexes indexes = new MatrixIndexes(); private MatrixCell value = new MatrixCell(); - private Pair<MatrixIndexes, MatrixCell> pair = new Pair<MatrixIndexes, MatrixCell>(indexes, value); + private Pair<MatrixIndexes, MatrixCell> pair = new Pair<>(indexes, value); private FastStringTokenizer st = new FastStringTokenizer(' '); private boolean hasValue = false; private boolean toIgnore = false; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java b/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java index cbcc9f8..a60f1c0 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/TripleIndexes.java @@ -44,26 +44,24 @@ public class TripleIndexes implements WritableComparable<TripleIndexes>, Seriali private long third=-1; public TripleIndexes(){} - public TripleIndexes(long i1, long i2, long i3) - { + + public TripleIndexes(long i1, long i2, long i3) { first=i1; second=i2; third=i3; } - public TripleIndexes(TripleIndexes that) - { + public TripleIndexes(TripleIndexes that) { setIndexes(that); } public void setIndexes(TripleIndexes that) { - this.first=that.first; this.second=that.second; this.third=that.third; } - public String toString() - { + @Override + public String toString() { return "("+first+", "+second+") k: "+third; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java index 83575f0..c8581a7 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/UnPaddedOutputFormat.java @@ -62,6 +62,6 @@ public class UnPaddedOutputFormat<K extends Writable, V extends Writable> extend Path file = FileOutputFormat.getTaskOutputPath(job, name); FileSystem fs = file.getFileSystem(job); FSDataOutputStream fileOut = fs.create(file, true, job.getInt(MRConfigurationNames.IO_FILE_BUFFER_SIZE, 4096), progress); - return new UnpaddedRecordWriter<K, V>(fileOut); + return new UnpaddedRecordWriter<>(fileOut); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java index dac0170..f111111 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCell.java @@ -38,8 +38,8 @@ public class WeightedCell extends MatrixCell protected double weight=0; - public String toString() - { + @Override + public String toString() { return value+": "+weight; } @@ -62,6 +62,8 @@ public class WeightedCell extends MatrixCell throw new DMLRuntimeException("the Matrix Value is not WeightedCell!"); return (WeightedCell) cell; } + + @Override public void copy(MatrixValue that){ WeightedCell c2; try { @@ -102,18 +104,16 @@ public class WeightedCell extends MatrixCell throw new RuntimeException("hashCode() should never be called on instances of this class."); } - public void setWeight(double w) - { + public void setWeight(double w) { weight=w; } - public double getWeight() - { + public double getWeight() { return weight; } - public double getValue() - { + @Override + public double getValue() { return value; } @@ -128,6 +128,7 @@ public class WeightedCell extends MatrixCell } //TODO: how to handle -minus left vs. minus right + @Override public void denseScalarOperationsInPlace(ScalarOperator op) throws DMLRuntimeException { value=op.executeScalar(value); @@ -153,11 +154,13 @@ public class WeightedCell extends MatrixCell return c3; } + @Override public void sparseScalarOperationsInPlace(ScalarOperator op) throws DMLRuntimeException { value=op.executeScalar(value); } + @Override public void sparseUnaryOperationsInPlace(UnaryOperator op) throws DMLRuntimeException { value=op.fn.execute(value); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java index d24fec6..00d4327 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedCellToSortInputConverter.java @@ -26,10 +26,9 @@ import org.apache.hadoop.io.IntWritable; public class WeightedCellToSortInputConverter implements Converter<MatrixIndexes, WeightedCell, DoubleWritable, IntWritable> { - private DoubleWritable outKey=new DoubleWritable(); private IntWritable outValue=new IntWritable(); - private Pair<DoubleWritable, IntWritable> pair=new Pair<DoubleWritable, IntWritable>(outKey, outValue); + private Pair<DoubleWritable, IntWritable> pair=new Pair<>(outKey, outValue); private boolean hasValue=false; @Override public void convert(MatrixIndexes k1, WeightedCell v1) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java index c9a6292..a255272 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/WeightedPair.java @@ -32,8 +32,9 @@ public class WeightedPair extends WeightedCell private static final long serialVersionUID = 8772815876289553196L; private double other=0; - public String toString() - { + + @Override + public String toString() { return value+", "+other+": "+weight; } @@ -57,6 +58,8 @@ public class WeightedPair extends WeightedCell throw new DMLRuntimeException("the Matrix Value is not WeightedPair!"); return (WeightedPair) cell; } + + @Override public void copy(MatrixValue that){ WeightedPair c2; try { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java index 3a27c0c..1b2cd1a 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/hadoopfix/MultipleInputs.java @@ -66,7 +66,7 @@ public class MultipleInputs { * @return A map of paths to inputformats for the job */ static Map<Path, InputFormat> getInputFormatMap(JobConf conf) { - Map<Path, InputFormat> m = new HashMap<Path, InputFormat>(); + Map<Path, InputFormat> m = new HashMap<>(); String[] pathMappings = conf.get(MRConfigurationNames.MR_INPUT_MULTIPLEINPUTS_DIR_FORMATS).split(","); for (String pathMapping : pathMappings) { String[] split = pathMapping.split(";"); @@ -95,7 +95,7 @@ public class MultipleInputs { if (conf.get(MRConfigurationNames.MR_INPUT_MULTIPLEINPUTS_DIR_MAPPERS) == null) { return Collections.emptyMap(); } - Map<Path, Class<? extends Mapper>> m = new HashMap<Path, Class<? extends Mapper>>(); + Map<Path, Class<? extends Mapper>> m = new HashMap<>(); String[] pathMappings = conf.get(MRConfigurationNames.MR_INPUT_MULTIPLEINPUTS_DIR_MAPPERS).split(","); for (String pathMapping : pathMappings) { String[] split = pathMapping.split(";"); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java index 5f36a40..eb8f6c0 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java @@ -111,6 +111,7 @@ implements Mapper<Writable, Writable, Writable, Writable> } } + @Override public void close() throws IOException { if(cachedCollector!=null) @@ -131,6 +132,7 @@ implements Mapper<Writable, Writable, Writable, Writable> } } + @Override public void configure(JobConf job) { super.configure(job); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java index 4194b08..2fd0ac1 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java @@ -129,6 +129,7 @@ implements Reducer<TaggedFirstSecondIndexes, MatrixValue, MatrixIndexes, MatrixV } } + @Override public void configure(JobConf job) { super.configure(job); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java index aedd2b0..f04fba8 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVReblockMapper.java @@ -48,7 +48,7 @@ public class CSVReblockMapper extends MapperBase implements Mapper<LongWritable, private long rowOffset=0; private boolean first=true; private long num=0; - private HashMap<Long, Long> offsetMap=new HashMap<Long, Long>(); + private HashMap<Long, Long> offsetMap=new HashMap<>(); private String _delim=" "; private boolean ignoreFirstLine=false; private boolean headerFile=false; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java index 0c1bf2c..3702d69 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CSVWriteMapper.java @@ -77,6 +77,7 @@ public class CSVWriteMapper extends MapperBase implements Mapper<Writable, Writa reporter.incrCounter(Counters.MAP_TIME, System.currentTimeMillis()-start); } + @Override public void configure(JobConf job) { super.configure(job); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java index b856774..272a48d 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CachedValueMap.java @@ -103,7 +103,7 @@ public class CachedValueMap //extends CachedMap<IndexedMatrixValue> ArrayList<IndexedMatrixValue> list = map.get(tag); if( list == null ){ - list = new ArrayList<IndexedMatrixValue>(); + list = new ArrayList<>(); map.put(tag, list); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java index 7a0b8c2..82a9873 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/FrameReblockBuffer.java @@ -141,7 +141,7 @@ public class FrameReblockBuffer return; //output block - out.add(new Pair<Long, FrameBlock>(new Long(key), value)); + out.add(new Pair<>(new Long(key), value)); } private static class FrameCell http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java index 50a83fd..c25b798 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRCombiner.java @@ -56,7 +56,8 @@ implements Reducer<MatrixIndexes, TaggedMatrixValue, MatrixIndexes, TaggedMatrix report.incrCounter(Counters.COMBINE_OR_REDUCE_TIME, System.currentTimeMillis()-start); } - + + @Override public void configure(JobConf job) { super.configure(job); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java index 0c30023..3e83272 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java @@ -157,6 +157,7 @@ implements Mapper<Writable, Writable, Writable, Writable> } + @Override public void configure(JobConf job) { super.configure(job); @@ -199,7 +200,7 @@ implements Mapper<Writable, Writable, Writable, Writable> ArrayList<Integer> vec=tagMapping.get(output); if(vec==null) { - vec=new ArrayList<Integer>(); + vec=new ArrayList<>(); tagMapping.put(output, vec); } vec.add(i); @@ -208,6 +209,7 @@ implements Mapper<Writable, Writable, Writable, Writable> collectFinalMultipleOutputs=MRJobConfiguration.getMultipleConvertedOutputs(job); } + @Override public void close() throws IOException { if( cachedReporter!=null && mapOnlyJob ) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java index 454093d..88152f5 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GroupedAggMRMapper.java @@ -142,8 +142,7 @@ public class GroupedAggMRMapper extends MapperBase GroupedAggregateInstruction[] grpaggIns = MRJobConfiguration.getGroupedAggregateInstructions(job); if( grpaggIns == null ) throw new RuntimeException("no GroupAggregate Instructions found!"); - - ArrayList<GroupedAggregateInstruction> vec = new ArrayList<GroupedAggregateInstruction>(); + ArrayList<GroupedAggregateInstruction> vec = new ArrayList<>(); for(int i=0; i<representativeMatrixes.size(); i++) { byte input=representativeMatrixes.get(i); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java index 047f084..39bee87 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/IndexedMatrixValue.java @@ -80,8 +80,8 @@ public class IndexedMatrixValue implements Serializable _value = block2; } - public String toString() - { + @Override + public String toString() { return "("+_indexes.getRowIndex()+", "+_indexes.getColumnIndex()+"): \n"+_value; } }
