http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java index 850d4ea..cfd2456 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/PmmSPInstruction.java @@ -131,7 +131,7 @@ public class PmmSPInstruction extends BinarySPInstruction { public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> arg0 ) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ixIn = arg0._1(); MatrixBlock mb2 = arg0._2(); @@ -163,9 +163,9 @@ public class PmmSPInstruction extends BinarySPInstruction { //hence we do a meta data correction afterwards) mb1.permutationMatrixMultOperations(mb2, out1, out2); out1.setNumRows(UtilFunctions.computeBlockSize(_rlen, rowIX1, _brlen)); - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(rowIX1, ixIn.getColumnIndex()), out1)); + ret.add(new Tuple2<>(new MatrixIndexes(rowIX1, ixIn.getColumnIndex()), out1)); if( out2 != null ) - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(rowIX2, ixIn.getColumnIndex()), out2)); + ret.add(new Tuple2<>(new MatrixIndexes(rowIX2, ixIn.getColumnIndex()), out2)); } return ret.iterator();
http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java index 907fa6f..a4d3b2e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuaternarySPInstruction.java @@ -200,8 +200,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { QuaternaryOperator qop = (QuaternaryOperator) _optr; //tracking of rdds and broadcasts (for lineage maintenance) - ArrayList<String> rddVars = new ArrayList<String>(); - ArrayList<String> bcVars = new ArrayList<String>(); + ArrayList<String> rddVars = new ArrayList<>(); + ArrayList<String> bcVars = new ArrayList<>(); JavaPairRDD<MatrixIndexes,MatrixBlock> in = sec.getBinaryBlockRDDHandleForVariable( input1.getName() ); JavaPairRDD<MatrixIndexes, MatrixBlock> out = null; @@ -399,8 +399,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { //create return tuple MatrixIndexes ixOut = createOutputIndexes(ixIn); - return new Tuple2<MatrixIndexes,MatrixBlock>(ixOut, blkOut); - } + return new Tuple2<>(ixOut, blkOut); + } } } @@ -433,8 +433,8 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { blkIn1.quaternaryOperations(_qop, mbU, mbV, mbW, blkOut); //create return tuple - MatrixIndexes ixOut = createOutputIndexes(ixIn); - return new Tuple2<MatrixIndexes,MatrixBlock>(ixOut, blkOut); + MatrixIndexes ixOut = createOutputIndexes(ixIn); + return new Tuple2<>(ixOut, blkOut); } } @@ -470,7 +470,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { //create return tuple MatrixIndexes ixOut = createOutputIndexes(ixIn); - return new Tuple2<MatrixIndexes,MatrixBlock>(ixOut, blkOut); + return new Tuple2<>(ixOut, blkOut); } } @@ -504,7 +504,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { //create return tuple MatrixIndexes ixOut = createOutputIndexes(ixIn1); - return new Tuple2<MatrixIndexes,MatrixBlock>(ixOut, blkOut); + return new Tuple2<>(ixOut, blkOut); } } @@ -524,7 +524,7 @@ public class QuaternarySPInstruction extends ComputationSPInstruction { MatrixBlock blkOut = new MatrixBlock(blkIn); //output new tuple - return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut,blkOut); + return new Tuple2<>(ixOut,blkOut); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java index 668a7d6..b50bf73 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RandSPInstruction.java @@ -354,26 +354,25 @@ public class RandSPInstruction extends UnarySPInstruction { double hdfsBlkSize = InfrastructureAnalyzer.getHDFSBlockSize(); long numBlocks = new MatrixCharacteristics(rows, cols, rowsInBlock, colsInBlock).getNumBlocks(); long numColBlocks = (long)Math.ceil((double)cols/(double)colsInBlock); - + //a) in-memory seed rdd construction if( numBlocks < INMEMORY_NUMBLOCKS_THRESHOLD ) { ArrayList<Tuple2<MatrixIndexes, Tuple2<Long, Long>>> seeds = - new ArrayList<Tuple2<MatrixIndexes, Tuple2<Long, Long>>>(); + new ArrayList<>(); for( long i=0; i<numBlocks; i++ ) { long r = 1 + i/numColBlocks; long c = 1 + i%numColBlocks; MatrixIndexes indx = new MatrixIndexes(r, c); Long seedForBlock = bigrand.nextLong(); - seeds.add(new Tuple2<MatrixIndexes, Tuple2<Long, Long>>(indx, - new Tuple2<Long, Long>(seedForBlock, nnzIter.nextLong()))); + seeds.add(new Tuple2<>(indx, new Tuple2<>(seedForBlock, nnzIter.nextLong()))); } //for load balancing: degree of parallelism such that ~128MB per partition int numPartitions = (int) Math.max(Math.min(totalSize/hdfsBlkSize, numBlocks), 1); //create seeds rdd - seedsRDD = sec.getSparkContext().parallelizePairs(seeds, numPartitions); + seedsRDD = sec.getSparkContext().parallelizePairs(seeds, numPartitions); } //b) file-based seed rdd construction (for robustness wrt large number of blocks) else @@ -454,7 +453,7 @@ public class RandSPInstruction extends UnarySPInstruction { //a) in-memory offset rdd construction if( numBlocks < INMEMORY_NUMBLOCKS_THRESHOLD ) { - ArrayList<Double> offsets = new ArrayList<Double>(); + ArrayList<Double> offsets = new ArrayList<>(); for( long i=0; i<numBlocks; i++ ) { double off = seq_from + seq_incr*i*rowsInBlock; offsets.add(off); @@ -539,13 +538,12 @@ public class RandSPInstruction extends UnarySPInstruction { int numPartitions = (int) Math.ceil((double)outputSize/hdfsBlockSize); long partitionSize = (long) Math.ceil(maxValue/numPartitions); - ArrayList<SampleTask> offsets = new ArrayList<SampleTask>(); + ArrayList<SampleTask> offsets = new ArrayList<>(); long st = 1; while ( st <= maxValue ) { SampleTask s = new SampleTask(); s.range_start = st; s.seed = bigrand.nextLong(); - offsets.add(s); st = st + partitionSize; } @@ -585,7 +583,7 @@ public class RandSPInstruction extends UnarySPInstruction { private static final long serialVersionUID = -725284524434342939L; long seed; long range_start; - + @Override public String toString() { return "(" + seed + "," + range_start +")"; } } @@ -630,7 +628,7 @@ public class RandSPInstruction extends UnarySPInstruction { long st = t.range_start; long end = Math.min(t.range_start+_partitionSize, _maxValue); - ArrayList<Double> retList = new ArrayList<Double>(); + ArrayList<Double> retList = new ArrayList<>(); if ( _frac == 1.0 ) { @@ -696,8 +694,7 @@ public class RandSPInstruction extends UnarySPInstruction { long rowID = t._2()+1; MatrixIndexes mi = new MatrixIndexes(rowID, 1); MatrixCell mc = new MatrixCell(t._1()); - - return new Tuple2<MatrixIndexes, MatrixCell>(mi, mc); + return new Tuple2<>(mi, mc); } } @@ -714,7 +711,7 @@ public class RandSPInstruction extends UnarySPInstruction { } @Override public Tuple2<Double, Double> call(Double t) throws Exception { - return new Tuple2<Double,Double>( r.nextDouble(), t ); + return new Tuple2<>( r.nextDouble(), t ); } } @@ -728,10 +725,9 @@ public class RandSPInstruction extends UnarySPInstruction { String[] parts = IOUtilFunctions.split(arg, ","); MatrixIndexes ix = new MatrixIndexes( Long.parseLong(parts[0]), Long.parseLong(parts[1])); - Tuple2<Long,Long> seed = new Tuple2<Long,Long>( + Tuple2<Long,Long> seed = new Tuple2<>( Long.parseLong(parts[2]), Long.parseLong(parts[3])); - - return new Tuple2<MatrixIndexes, Tuple2<Long, Long>>(ix,seed); + return new Tuple2<>(ix,seed); } } @@ -780,19 +776,15 @@ public class RandSPInstruction extends UnarySPInstruction { long blockColIndex = ix.getColumnIndex(); int lrlen = UtilFunctions.computeBlockSize(_rlen, blockRowIndex, _brlen); int lclen = UtilFunctions.computeBlockSize(_clen, blockColIndex, _bclen); - long seed = kv._2._1; long blockNNZ = kv._2._2; MatrixBlock blk = new MatrixBlock(); - RandomMatrixGenerator rgen = LibMatrixDatagen.createRandomMatrixGenerator( _pdf, lrlen, lclen, lrlen, lclen, _sparsity, _min, _max, _pdfParams ); - blk.randOperationsInPlace(rgen, LongStream.of(blockNNZ), null, seed); - - return new Tuple2<MatrixIndexes, MatrixBlock>(kv._1, blk); + return new Tuple2<>(kv._1, blk); } } @@ -819,12 +811,12 @@ public class RandSPInstruction extends UnarySPInstruction { double seq_to = (_seq_incr > 0) ? Math.min(_global_seq_end, seq_from + _seq_incr*(_brlen-1)) : Math.max(_global_seq_end, seq_from + _seq_incr*(_brlen+1)); - long globalRow = (long)Math.round((seq_from-_global_seq_start)/_seq_incr)+1; + long globalRow = (long)Math.round((seq_from-_global_seq_start)/_seq_incr)+1; long rowIndex = UtilFunctions.computeBlockIndex(globalRow, _brlen); MatrixIndexes indx = new MatrixIndexes(rowIndex, 1); MatrixBlock blk = MatrixBlock.seqOperations(seq_from, seq_to, _seq_incr); - return new Tuple2<MatrixIndexes, MatrixBlock>(indx, blk); + return new Tuple2<>(indx, blk); } } @@ -841,7 +833,7 @@ public class RandSPInstruction extends UnarySPInstruction { private boolean isMemAvail(long lRows, long lCols, double sparsity, double min, double max) { double size = (min == 0 && max == 0) ? OptimizerUtils.estimateSizeEmptyBlock(rows, cols): - OptimizerUtils.estimateSizeExactSparsity(rows, cols, sparsity); + OptimizerUtils.estimateSizeExactSparsity(rows, cols, sparsity); return ( OptimizerUtils.isValidCPDimensions(rows, cols) && OptimizerUtils.isValidCPMatrixSize(rows, cols, sparsity) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java index b547310..f8b92ac 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ReorgSPInstruction.java @@ -242,7 +242,7 @@ public class ReorgSPInstruction extends UnarySPInstruction { public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> arg0 ) throws Exception { - ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); @@ -251,16 +251,16 @@ public class ReorgSPInstruction extends UnarySPInstruction { long rix = ixIn.getRowIndex(); MatrixIndexes ixOut = new MatrixIndexes(rix, rix); MatrixBlock blkOut = (MatrixBlock) blkIn.reorgOperations(_reorgOp, new MatrixBlock(), -1, -1, -1); - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixOut,blkOut)); + ret.add(new Tuple2<>(ixOut,blkOut)); // insert newly created empty blocks for entire row int numBlocks = (int) Math.ceil((double)_mcIn.getRows()/_mcIn.getRowsPerBlock()); for(int i = 1; i <= numBlocks; i++) { if(i != ixOut.getColumnIndex()) { int lrlen = UtilFunctions.computeBlockSize(_mcIn.getRows(), rix, _mcIn.getRowsPerBlock()); - int lclen = UtilFunctions.computeBlockSize(_mcIn.getRows(), i, _mcIn.getRowsPerBlock()); - MatrixBlock emptyBlk = new MatrixBlock(lrlen, lclen, true); - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(rix, i), emptyBlk)); + int lclen = UtilFunctions.computeBlockSize(_mcIn.getRows(), i, _mcIn.getRowsPerBlock()); + MatrixBlock emptyBlk = new MatrixBlock(lrlen, lclen, true); + ret.add(new Tuple2<>(new MatrixIndexes(rix, i), emptyBlk)); } } @@ -288,7 +288,7 @@ public class ReorgSPInstruction extends UnarySPInstruction { IndexedMatrixValue in = SparkUtils.toIndexedMatrixBlock(arg0); //execute reverse operation - ArrayList<IndexedMatrixValue> out = new ArrayList<IndexedMatrixValue>(); + ArrayList<IndexedMatrixValue> out = new ArrayList<>(); LibMatrixReorg.rev(in, _mcIn.getRows(), _mcIn.getRowsPerBlock(), out); //construct output http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java index 43ba34a..608200f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/RmmSPInstruction.java @@ -140,7 +140,7 @@ public class RmmSPInstruction extends BinarySPInstruction { public Iterator<Tuple2<TripleIndexes, MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> arg0 ) throws Exception { - LinkedList<Tuple2<TripleIndexes, MatrixBlock>> ret = new LinkedList<Tuple2<TripleIndexes, MatrixBlock>>(); + LinkedList<Tuple2<TripleIndexes, MatrixBlock>> ret = new LinkedList<>(); MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); @@ -153,7 +153,7 @@ public class RmmSPInstruction extends BinarySPInstruction { long k = ixIn.getColumnIndex(); for( long j=1; j<=numBlocks; j++ ) { TripleIndexes tmptix = new TripleIndexes(i, j, k); - ret.add( new Tuple2<TripleIndexes, MatrixBlock>(tmptix, blkIn) ); + ret.add( new Tuple2<>(tmptix, blkIn) ); } } else // RHS MATRIX @@ -163,7 +163,7 @@ public class RmmSPInstruction extends BinarySPInstruction { long j = ixIn.getColumnIndex(); for( long i=1; i<=numBlocks; i++ ) { TripleIndexes tmptix = new TripleIndexes(i, j, k); - ret.add( new Tuple2<TripleIndexes, MatrixBlock>(tmptix, blkIn) ); + ret.add( new Tuple2<>(tmptix, blkIn) ); } } @@ -197,9 +197,9 @@ public class RmmSPInstruction extends BinarySPInstruction { //core block matrix multiplication blkIn1.aggregateBinaryOperations(blkIn1, blkIn2, blkOut, _op); - + //output new tuple - return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut, blkOut); + return new Tuple2<>(ixOut, blkOut); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java index b4e48df..4ef5109 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java @@ -263,7 +263,7 @@ public class TernarySPInstruction extends ComputationSPInstruction { MatrixBlock mb = arg0._2(); //col-vector //create an output cell per matrix block row (aligned w/ original source position) - ArrayList<Tuple2<MatrixIndexes, Double>> retVal = new ArrayList<Tuple2<MatrixIndexes,Double>>(); + ArrayList<Tuple2<MatrixIndexes, Double>> retVal = new ArrayList<>(); CTable ctab = CTable.getCTableFnObject(); for( int i=0; i<mb.getNumRows(); i++ ) { @@ -274,7 +274,7 @@ public class TernarySPInstruction extends ComputationSPInstruction { //indirect construction over pair to avoid tuple2 dependency in general ctable obj if( p.getKey().getRowIndex() >= 1 ) //filter rejected entries - retVal.add(new Tuple2<MatrixIndexes,Double>(p.getKey(), p.getValue())); + retVal.add(new Tuple2<>(p.getKey(), p.getValue())); } return retVal.iterator(); @@ -306,9 +306,9 @@ public class TernarySPInstruction extends ComputationSPInstruction { in1 = extractBlock(kv._2._1, in1); in2 = extractBlock(kv._2._2, in2); // Now return unflatten AL - ArrayList<MatrixBlock> inputs = new ArrayList<MatrixBlock>(); + ArrayList<MatrixBlock> inputs = new ArrayList<>(); inputs.add(in1); inputs.add(in2); - return new Tuple2<MatrixIndexes, ArrayList<MatrixBlock>>(kv._1, inputs); + return new Tuple2<>(kv._1, inputs); } } @@ -343,9 +343,9 @@ public class TernarySPInstruction extends ComputationSPInstruction { in3 = extractBlock(kv._2._2, in3); // Now return unflatten AL - ArrayList<MatrixBlock> inputs = new ArrayList<MatrixBlock>(); + ArrayList<MatrixBlock> inputs = new ArrayList<>(); inputs.add(in1); inputs.add(in2); inputs.add(in3); - return new Tuple2<MatrixIndexes, ArrayList<MatrixBlock>>(kv._1, inputs); + return new Tuple2<>(kv._1, inputs); } } @@ -430,10 +430,9 @@ public class TernarySPInstruction extends ComputationSPInstruction { break; } default: - throw new DMLRuntimeException("Unrecognized opcode in Tertiary Instruction: " + instString); + throw new DMLRuntimeException("Unrecognized opcode in Tertiary Instruction: " + instString); } - - return new Tuple2<MatrixIndexes, CTableMap>(kv._1, ctableResult); + return new Tuple2<>(kv._1, ctableResult); } } @@ -445,9 +444,9 @@ public class TernarySPInstruction extends ComputationSPInstruction { @Override public Tuple2<MatrixIndexes, ArrayList<MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> kv) throws Exception { - ArrayList<MatrixBlock> retVal = new ArrayList<MatrixBlock>(); + ArrayList<MatrixBlock> retVal = new ArrayList<>(); retVal.add(kv._2); - return new Tuple2<MatrixIndexes, ArrayList<MatrixBlock>>(kv._1, retVal); + return new Tuple2<>(kv._1, retVal); } } @@ -459,14 +458,14 @@ public class TernarySPInstruction extends ComputationSPInstruction { @Override public Iterator<Tuple2<MatrixIndexes, Double>> call(CTableMap ctableMap) throws Exception { - ArrayList<Tuple2<MatrixIndexes, Double>> retVal = new ArrayList<Tuple2<MatrixIndexes, Double>>(); + ArrayList<Tuple2<MatrixIndexes, Double>> retVal = new ArrayList<>(); Iterator<LLDoubleEntry> iter = ctableMap.getIterator(); while( iter.hasNext() ) { LLDoubleEntry ijv = iter.next(); long i = ijv.key1; long j = ijv.key2; double v = ijv.value; - retVal.add(new Tuple2<MatrixIndexes, Double>(new MatrixIndexes(i, j), v)); + retVal.add(new Tuple2<>(new MatrixIndexes(i, j), v)); } return retVal.iterator(); } @@ -482,7 +481,7 @@ public class TernarySPInstruction extends ComputationSPInstruction { Tuple2<MatrixIndexes, Double> kv) throws Exception { MatrixCell cell = new MatrixCell(kv._2().doubleValue()); - return new Tuple2<MatrixIndexes, MatrixCell>(kv._1(), cell); + return new Tuple2<>(kv._1(), cell); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java index 5147b8f..e52fbdc 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java @@ -147,14 +147,14 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { - List<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + List<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ixin = arg0._1(); MatrixBlock mbin = arg0._2(); //execute block tsmm operation MatrixBlock out1 = mbin.transposeSelfMatrixMultOperations(new MatrixBlock(), _type); long ixout = _type.isLeft() ? ixin.getColumnIndex() : ixin.getRowIndex(); - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(ixout, ixout), out1)); + ret.add(new Tuple2<>(new MatrixIndexes(ixout, ixout), out1)); if( _type.isLeft() ? ixin.getColumnIndex() == 1 : ixin.getRowIndex() == 1 ) { //execute block mapmm operation for full block only (output two blocks, due to symmetry) @@ -166,11 +166,11 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { MatrixBlock out2 = (MatrixBlock) OperationsOnMatrixValues.performAggregateBinaryIgnoreIndexes( //mm _type.isLeft() ? mbin2t : mbin, _type.isLeft() ? mbin : mbin2t, new MatrixBlock(), _op); MatrixIndexes ixout2 = _type.isLeft() ? new MatrixIndexes(2,1) : new MatrixIndexes(1,2); - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ixout2, out2)); + ret.add(new Tuple2<>(ixout2, out2)); MatrixBlock out3 = transpose(out2, new MatrixBlock()); MatrixIndexes ixout3 = _type.isLeft() ? new MatrixIndexes(1,2) : new MatrixIndexes(2,1); - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ixout3, out3)); + ret.add(new Tuple2<>(ixout3, out3)); } return ret.iterator(); @@ -254,9 +254,9 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { throws Exception { if( _type.isLeft() ) - return new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(arg0._1().getRowIndex(), 1), arg0._2()); + return new Tuple2<>(new MatrixIndexes(arg0._1().getRowIndex(), 1), arg0._2()); else - return new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(1, arg0._1().getColumnIndex()), arg0._2()); + return new Tuple2<>(new MatrixIndexes(1, arg0._1().getColumnIndex()), arg0._2()); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java index 51ea00b..9c7aefc 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/UaggOuterChainSPInstruction.java @@ -265,21 +265,18 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction { { MatrixIndexes in1Ix = arg._1(); MatrixBlock in1Val = arg._2(); - MatrixIndexes outIx = new MatrixIndexes(); MatrixBlock outVal = new MatrixBlock(); int [] bvi = null; if((LibMatrixOuterAgg.isRowIndexMax(_uaggOp)) || (LibMatrixOuterAgg.isRowIndexMin(_uaggOp))) - { bvi = _bvi.getValue(); - } LibMatrixOuterAgg.resetOutputMatix(in1Ix, in1Val, outIx, outVal, _uaggOp); LibMatrixOuterAgg.aggregateMatrix(in1Val, outVal, _bv.value(), bvi, _bOp, _uaggOp); - return new Tuple2<MatrixIndexes, MatrixBlock>(outIx, outVal); - } + return new Tuple2<>(outIx, outVal); + } } } @@ -366,9 +363,8 @@ public class UaggOuterChainSPInstruction extends BinarySPInstruction { else OperationsOnMatrixValues.incrementalAggregation(outVal, null, _tmpVal2, _aggOp, true); } - - return new Tuple2<MatrixIndexes, MatrixBlock>(outIx, outVal); - } - } + return new Tuple2<>(outIx, outVal); + } + } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java index 046abf5..2fb47f9 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java @@ -168,9 +168,9 @@ public class WriteSPInstruction extends SPInstruction { in1 = in1.mapValues(new ComputeBinaryBlockNnzFunction(aNnz)); } - JavaRDD<String> header = null; + JavaRDD<String> header = null; if( oi == OutputInfo.MatrixMarketOutputInfo ) { - ArrayList<String> headerContainer = new ArrayList<String>(1); + ArrayList<String> headerContainer = new ArrayList<>(1); // First output MM header String headerStr = "%%MatrixMarket matrix coordinate real general\n" + // output number of rows, number of columns and number of nnz http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java index bdd34e4..53b9d34 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/BroadcastObject.java @@ -32,7 +32,7 @@ public class BroadcastObject<T extends CacheBlock> extends LineageObject public BroadcastObject( PartitionedBroadcast<T> bvar, String varName, long size ) { super(varName); - _bcHandle = new SoftReference<PartitionedBroadcast<T>>(bvar); + _bcHandle = new SoftReference<>(bvar); _size = size; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java index 58629e6..4d6f0c6 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/LineageObject.java @@ -37,7 +37,7 @@ public abstract class LineageObject protected LineageObject(String varName) { _varName = varName; _numRef = 0; - _childs = new ArrayList<LineageObject>(); + _childs = new ArrayList<>(); } public String getVarName() { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java index 08257f6..ed5167b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/data/PartitionedBlock.java @@ -109,7 +109,7 @@ public class PartitionedBlock<T extends CacheBlock> implements Externalizable public PartitionedBlock<T> createPartition( int offset, int numBlks, T block ) { - PartitionedBlock<T> ret = new PartitionedBlock<T>(); + PartitionedBlock<T> ret = new PartitionedBlock<>(); ret._rlen = _rlen; ret._clen = _clen; ret._brlen = _brlen; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java index cd6e58d..32c86c4 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertFrameBlockToIJVLines.java @@ -37,8 +37,7 @@ public class ConvertFrameBlockToIJVLines implements FlatMapFunction<Tuple2<Long, { long rowoffset = kv._1; FrameBlock block = kv._2; - - ArrayList<String> cells = new ArrayList<String>(); + ArrayList<String> cells = new ArrayList<>(); //write frame meta data if( rowoffset == 1 ) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java index d496b27..26dbf5e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ConvertStringToLongTextPair.java @@ -30,7 +30,7 @@ public class ConvertStringToLongTextPair implements PairFunction<String, LongWri @Override public Tuple2<LongWritable, Text> call(String arg0) throws Exception { - return new Tuple2<LongWritable, Text>(new LongWritable(1), new Text(arg0)); + return new Tuple2<>(new LongWritable(1), new Text(arg0)); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java index b7742cc..957e526 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBinaryCellFunction.java @@ -40,6 +40,6 @@ public class CopyBinaryCellFunction implements PairFunction<Tuple2<MatrixIndexes MatrixIndexes ix = new MatrixIndexes(arg0._1()); MatrixCell cell = new MatrixCell(); cell.copy(arg0._2()); - return new Tuple2<MatrixIndexes,MatrixCell>(ix,cell); + return new Tuple2<>(ix,cell); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java index 42d4843..5423a8f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyBlockPairFunction.java @@ -76,7 +76,7 @@ public class CopyBlockPairFunction implements PairFlatMapFunction<Iterator<Tuple block = new MatrixBlock(arg._2, SparseBlock.Type.CSR, true); else block = new MatrixBlock(arg._2()); - return new Tuple2<MatrixIndexes,MatrixBlock>(ix,block); + return new Tuple2<>(ix,block); } else { return arg; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java index df891f7..81d6739 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyFrameBlockPairFunction.java @@ -51,10 +51,10 @@ public class CopyFrameBlockPairFunction implements PairFunction<Tuple2<LongWrita { if( _deepCopy ) { FrameBlock block = new FrameBlock(arg0._2()); - return new Tuple2<Long,FrameBlock>(arg0._1().get(), block); + return new Tuple2<>(arg0._1().get(), block); } else { - return new Tuple2<Long,FrameBlock>(arg0._1().get(), arg0._2()); + return new Tuple2<>(arg0._1().get(), arg0._2()); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java index 3a2c38d..bb01ca6 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/CopyTextInputFunction.java @@ -38,6 +38,6 @@ public class CopyTextInputFunction implements PairFunction<Tuple2<LongWritable, Tuple2<LongWritable, Text> arg0) throws Exception { LongWritable lw = new LongWritable(arg0._1().get()); Text txt = new Text(arg0._2()); - return new Tuple2<LongWritable,Text>(lw, txt); + return new Tuple2<>(lw, txt); } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java index 61b4385..e6042ed 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractBlockForBinaryReblock.java @@ -80,7 +80,7 @@ public class ExtractBlockForBinaryReblock implements PairFlatMapFunction<Tuple2< long out_endColBlockIndex = UtilFunctions.computeBlockIndex(endColGlobalCellIndex, out_bclen); assert(out_startRowBlockIndex <= out_endRowBlockIndex && out_startColBlockIndex <= out_endColBlockIndex); - ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> retVal = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> retVal = new ArrayList<>(); for(long i = out_startRowBlockIndex; i <= out_endRowBlockIndex; i++) { for(long j = out_startColBlockIndex; j <= out_endColBlockIndex; j++) { @@ -105,7 +105,7 @@ public class ExtractBlockForBinaryReblock implements PairFlatMapFunction<Tuple2< blk.appendValue(out_i1, out_j1, val); } } - retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(indx, blk)); + retVal.add(new Tuple2<>(indx, blk)); } } return retVal.iterator(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java index 56c1c46..3e6908b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroup.java @@ -58,7 +58,7 @@ public abstract class ExtractGroup implements Serializable } //output weighted cells - ArrayList<Tuple2<MatrixIndexes, WeightedCell>> groupValuePairs = new ArrayList<Tuple2<MatrixIndexes, WeightedCell>>(); + ArrayList<Tuple2<MatrixIndexes, WeightedCell>> groupValuePairs = new ArrayList<>(); long coloff = (ix.getColumnIndex()-1)*_bclen; //local pre-aggregation for sum w/ known output dimensions @@ -75,7 +75,7 @@ public abstract class ExtractGroup implements Serializable weightedCell.setValue(tmpval); weightedCell.setWeight(1); MatrixIndexes ixout = new MatrixIndexes(i+1,coloff+j+1); - groupValuePairs.add(new Tuple2<MatrixIndexes, WeightedCell>(ixout, weightedCell)); + groupValuePairs.add(new Tuple2<>(ixout, weightedCell)); } } } @@ -93,7 +93,7 @@ public abstract class ExtractGroup implements Serializable weightedCell.setValue(target.quickGetValue(i, j)); weightedCell.setWeight(1); MatrixIndexes ixout = new MatrixIndexes(groupVal,coloff+j+1); - groupValuePairs.add(new Tuple2<MatrixIndexes, WeightedCell>(ixout, weightedCell)); + groupValuePairs.add(new Tuple2<>(ixout, weightedCell)); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java index d040dbc..c3900a5 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ExtractGroupNWeights.java @@ -50,7 +50,7 @@ public class ExtractGroupNWeights implements PairFlatMapFunction<Tuple2<MatrixIn } //output weighted cells - ArrayList<Tuple2<MatrixIndexes, WeightedCell>> groupValuePairs = new ArrayList<Tuple2<MatrixIndexes, WeightedCell>>(); + ArrayList<Tuple2<MatrixIndexes, WeightedCell>> groupValuePairs = new ArrayList<>(); for(int i = 0; i < group.getNumRows(); i++) { WeightedCell weightedCell = new WeightedCell(); weightedCell.setValue(target.quickGetValue(i, 0)); @@ -60,7 +60,7 @@ public class ExtractGroupNWeights implements PairFlatMapFunction<Tuple2<MatrixIn throw new Exception("Expected group values to be greater than equal to 1 but found " + groupVal); } MatrixIndexes ix = new MatrixIndexes(groupVal, 1); - groupValuePairs.add(new Tuple2<MatrixIndexes, WeightedCell>(ix, weightedCell)); + groupValuePairs.add(new Tuple2<>(ix, weightedCell)); } return groupValuePairs.iterator(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java index 6197b75..430af80 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/GetMIMBFromRow.java @@ -33,14 +33,9 @@ public class GetMIMBFromRow implements PairFunction<Row, MatrixIndexes, MatrixBl @Override public Tuple2<MatrixIndexes, MatrixBlock> call(Row row) throws Exception { -// try { MatrixIndexes indx = (MatrixIndexes) row.apply(0); MatrixBlock blk = (MatrixBlock) row.apply(1); - return new Tuple2<MatrixIndexes, MatrixBlock>(indx, blk); -// } -// catch(Exception e) { -// throw new Exception("Incorrect type of DataFrame passed to MLMatrix:" + e.getMessage()); -// } + return new Tuple2<>(indx, blk); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java index 70f3318..afd2e1a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/MatrixVectorBinaryOpPartitionFunction.java @@ -80,7 +80,7 @@ public class MatrixVectorBinaryOpPartitionFunction implements PairFlatMapFunctio //execute the binary operation MatrixBlock ret = (MatrixBlock) (in1.binaryOperations (_op, in2, new MatrixBlock())); - return new Tuple2<MatrixIndexes, MatrixBlock>(ix, ret); + return new Tuple2<>(ix, ret); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java index ddb77e8..3a2cd15 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/OuterVectorBinaryOpFunction.java @@ -84,8 +84,7 @@ public class OuterVectorBinaryOpFunction implements PairFlatMapFunction<Tuple2<M MatrixBlock in2 = _pmV.getBlock(1, _currPos); MatrixBlock resultBlk = (MatrixBlock)in1.binaryOperations (_op, in2, new MatrixBlock()); resultBlk.examSparsity(); - ret = new Tuple2<MatrixIndexes,MatrixBlock>( - new MatrixIndexes(ix.getRowIndex(), _currPos), resultBlk); + ret = new Tuple2<>(new MatrixIndexes(ix.getRowIndex(), _currPos), resultBlk); _currPos ++; } catch(Exception ex) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java index 79b9460..111e3b9 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReorgMapFunction.java @@ -60,16 +60,13 @@ public class ReorgMapFunction implements PairFunction<Tuple2<MatrixIndexes, Matr { MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); - //swap the matrix indexes MatrixIndexes ixOut = new MatrixIndexes(ixIn); _indexFnObject.execute(ixIn, ixOut); - //swap the matrix block data MatrixBlock blkOut = (MatrixBlock) blkIn.reorgOperations(_reorgOp, new MatrixBlock(), -1, -1, -1); - //output new tuple - return new Tuple2<MatrixIndexes, MatrixBlock>(ixOut,blkOut); + return new Tuple2<>(ixOut,blkOut); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateBlockFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateBlockFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateBlockFunction.java index 22ea33f..00f67cf 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateBlockFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateBlockFunction.java @@ -19,8 +19,8 @@ package org.apache.sysml.runtime.instructions.spark.functions; +import java.util.ArrayList; import java.util.Iterator; -import java.util.LinkedList; import org.apache.spark.api.java.function.PairFlatMapFunction; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -53,7 +53,7 @@ public class ReplicateBlockFunction implements PairFlatMapFunction<Tuple2<Matrix public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call( Tuple2<MatrixIndexes, MatrixBlock> arg0 ) throws Exception { - LinkedList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new LinkedList<Tuple2<MatrixIndexes, MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ixIn = arg0._1(); MatrixBlock blkIn = arg0._2(); @@ -66,7 +66,7 @@ public class ReplicateBlockFunction implements PairFlatMapFunction<Tuple2<Matrix for( long j=1; j<=numBlocks; j++ ) { MatrixIndexes tmpix = new MatrixIndexes(i, j); MatrixBlock tmpblk = _deep ? new MatrixBlock(blkIn) : blkIn; - ret.add( new Tuple2<MatrixIndexes, MatrixBlock>(tmpix, tmpblk) ); + ret.add( new Tuple2<>(tmpix, tmpblk) ); } } else // RHS MATRIX @@ -76,7 +76,7 @@ public class ReplicateBlockFunction implements PairFlatMapFunction<Tuple2<Matrix for( long i=1; i<=numBlocks; i++ ) { MatrixIndexes tmpix = new MatrixIndexes(i, j); MatrixBlock tmpblk = _deep ? new MatrixBlock(blkIn) : blkIn; - ret.add( new Tuple2<MatrixIndexes, MatrixBlock>(tmpix, tmpblk) ); + ret.add( new Tuple2<>(tmpix, tmpblk) ); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java index 8c6eb92..a0d1770 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/functions/ReplicateVectorFunction.java @@ -59,14 +59,13 @@ public class ReplicateVectorFunction implements PairFlatMapFunction<Tuple2<Matri throw new Exception("Expected a column vector in ReplicateVector"); } - ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> retVal = new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> retVal = new ArrayList<>(); for(int i = 1; i <= _numReplicas; i++) { if( _byRow ) - retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(i, ix.getColumnIndex()), mb)); + retVal.add(new Tuple2<>(new MatrixIndexes(i, ix.getColumnIndex()), mb)); else - retVal.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(ix.getRowIndex(), i), mb)); + retVal.add(new Tuple2<>(new MatrixIndexes(ix.getRowIndex(), i), mb)); } - return retVal.iterator(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java index 4e836eb..055b33b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java @@ -305,7 +305,7 @@ public class FrameRDDConverterUtils public static StructType convertFrameSchemaToDFSchema(ValueType[] fschema, boolean containsID) { // generate the schema based on the string of schema - List<StructField> fields = new ArrayList<StructField>(); + List<StructField> fields = new ArrayList<>(); // add id column type if( containsID ) @@ -474,9 +474,8 @@ public class FrameRDDConverterUtils while( max >= 0 && arg0.hasNext() ) { long val = arg0.next(); max = (val < max) ? -1 : val; - } - - ArrayList<Long> ret = new ArrayList<Long>(); + } + ArrayList<Long> ret = new ArrayList<>(); ret.add(max); return ret.iterator(); } @@ -492,7 +491,7 @@ public class FrameRDDConverterUtils @Override public Tuple2<LongWritable, Text> call(String arg0) throws Exception { - return new Tuple2<LongWritable,Text>(new SerLongWritable(1L), new SerText(arg0)); + return new Tuple2<>(new SerLongWritable(1L), new SerText(arg0)); } } @@ -502,7 +501,7 @@ public class FrameRDDConverterUtils @Override public Tuple2<LongWritable, FrameBlock> call(Tuple2<LongWritable, FrameBlock> arg0) throws Exception { - return new Tuple2<LongWritable,FrameBlock>(new SerLongWritable(arg0._1.get()), arg0._2); + return new Tuple2<>(new SerLongWritable(arg0._1.get()), arg0._2); } } @@ -512,7 +511,7 @@ public class FrameRDDConverterUtils @Override public Tuple2<Long, Text> call(Tuple2<LongWritable, Text> arg0) throws Exception { - return new Tuple2<Long,Text>(new Long(arg0._1.get()), arg0._2); + return new Tuple2<>(new Long(arg0._1.get()), arg0._2); } } @@ -522,7 +521,7 @@ public class FrameRDDConverterUtils @Override public Tuple2<LongWritable, FrameBlock> call(Tuple2<Long, FrameBlock> arg0) throws Exception { - return new Tuple2<LongWritable, FrameBlock>(new LongWritable(arg0._1), arg0._2); + return new Tuple2<>(new LongWritable(arg0._1), arg0._2); } } @@ -532,7 +531,7 @@ public class FrameRDDConverterUtils @Override public Tuple2<Long, FrameBlock> call(Tuple2<LongWritable, FrameBlock> arg0) throws Exception { - return new Tuple2<Long, FrameBlock>(arg0._1.get(), arg0._2); + return new Tuple2<>(arg0._1.get(), arg0._2); } } @@ -579,7 +578,7 @@ public class FrameRDDConverterUtils public Iterator<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Text,Long>> arg0) throws Exception { - ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<Tuple2<Long,FrameBlock>>(); + ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<>(); long ix = -1; FrameBlock fb = null; @@ -651,7 +650,7 @@ public class FrameRDDConverterUtils throws DMLRuntimeException { if( fb != null && fb.getNumRows()>0 ) - ret.add(new Tuple2<Long,FrameBlock>(ix, fb)); + ret.add(new Tuple2<>(ix, fb)); } } @@ -672,7 +671,7 @@ public class FrameRDDConverterUtils Long ix = arg0._1(); FrameBlock blk = arg0._2(); - ArrayList<String> ret = new ArrayList<String>(); + ArrayList<String> ret = new ArrayList<>(); StringBuilder sb = new StringBuilder(); //handle header information and frame meta data @@ -745,7 +744,7 @@ public class FrameRDDConverterUtils public Iterator<Tuple2<Long, FrameBlock>> call(Iterator<Tuple2<Row, Long>> arg0) throws Exception { - ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<Tuple2<Long,FrameBlock>>(); + ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<>(); long ix = -1; FrameBlock fb = null; @@ -791,7 +790,7 @@ public class FrameRDDConverterUtils throws DMLRuntimeException { if( fb != null && fb.getNumRows()>0 ) - ret.add(new Tuple2<Long,FrameBlock>(ix, fb)); + ret.add(new Tuple2<>(ix, fb)); } } @@ -805,7 +804,7 @@ public class FrameRDDConverterUtils { long rowIndex = arg0._1(); FrameBlock blk = arg0._2(); - ArrayList<Row> ret = new ArrayList<Row>(); + ArrayList<Row> ret = new ArrayList<>(); //handle Frame block data int rows = blk.getNumRows(); @@ -846,7 +845,7 @@ public class FrameRDDConverterUtils throws IOException, DMLRuntimeException { //temporary list of indexed matrix values to prevent library dependencies - ArrayList<Pair<Long, FrameBlock>> rettmp = new ArrayList<Pair<Long, FrameBlock>>(); + ArrayList<Pair<Long, FrameBlock>> rettmp = new ArrayList<>(); rbuff.flushBufferToBinaryBlocks(rettmp); ret.addAll(SparkUtils.fromIndexedFrameBlock(rettmp)); } @@ -866,7 +865,7 @@ public class FrameRDDConverterUtils public Iterator<Tuple2<Long, FrameBlock>> call(Iterator<Text> arg0) throws Exception { - ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<Tuple2<Long,FrameBlock>>(); + ArrayList<Tuple2<Long,FrameBlock>> ret = new ArrayList<>(); FrameReblockBuffer rbuff = new FrameReblockBuffer(_bufflen, _rlen, _clen, _schema ); FastStringTokenizer st = new FastStringTokenizer(' '); @@ -922,7 +921,7 @@ public class FrameRDDConverterUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Tuple2<MatrixIndexes,MatrixBlock> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ix = arg0._1(); MatrixBlock mb = arg0._2(); MatrixBlock mbreuse = new MatrixBlock(); @@ -944,7 +943,7 @@ public class FrameRDDConverterUtils out.copy(0, out.getNumRows()-1, (int)cl, (int)cu, mb.sliceOperations(i, ru, 0, mb.getNumColumns()-1, mbreuse), true); out.examSparsity(); - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(ixout,out)); + ret.add(new Tuple2<>(ixout,out)); } return ret.iterator(); @@ -982,7 +981,7 @@ public class FrameRDDConverterUtils throws Exception { FrameBlock fb = DataConverter.convertToFrameBlock(arg0._2()); - return new Tuple2<Long, FrameBlock>( + return new Tuple2<>( (arg0._1().getRowIndex()-1)*_brlen+1, fb); } } @@ -994,7 +993,7 @@ public class FrameRDDConverterUtils private MatrixCharacteristics _mcIn; private MatrixCharacteristics _mcOut; - public BinaryBlockToMatrixBlockFunction(MatrixCharacteristics mcIn, MatrixCharacteristics mcOut) { + public BinaryBlockToMatrixBlockFunction(MatrixCharacteristics mcIn, MatrixCharacteristics mcOut) { _mcIn = mcIn; //Frame Characteristics _mcOut = mcOut; //Matrix Characteristics } @@ -1006,7 +1005,7 @@ public class FrameRDDConverterUtils long rowIndex = arg0._1(); FrameBlock blk = arg0._2(); - ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes, MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<>(); long rlen = _mcIn.getRows(); long clen = _mcIn.getCols(); int brlen = _mcOut.getRowsPerBlock(); @@ -1030,7 +1029,7 @@ public class FrameRDDConverterUtils FrameBlock frame = blk.sliceOperations(fix, fix2, (int)cpos-1, (int)cpos+lclen-2, new FrameBlock()); MatrixBlock mframe = DataConverter.convertToMatrixBlock(frame); - ret.add(new Tuple2<MatrixIndexes, MatrixBlock>(new MatrixIndexes(rix, cix), + ret.add(new Tuple2<>(new MatrixIndexes(rix, cix), matrix.leftIndexingOperations(mframe, mix, mix2, 0, lclen-1, new MatrixBlock(), UpdateType.INPLACE_PINNED))); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java index 8ec02cc..b5ce814 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java @@ -281,7 +281,7 @@ public class RDDConverterUtils .groupByKey().map(new ConvertRowBlocksToRows((int)mc.getCols(), mc.getColsPerBlock(), toVector)); //create data frame schema - List<StructField> fields = new ArrayList<StructField>(); + List<StructField> fields = new ArrayList<>(); fields.add(DataTypes.createStructField(DF_ID_COLUMN, DataTypes.DoubleType, false)); if( toVector ) fields.add(DataTypes.createStructField("C1", new VectorUDT(), false)); @@ -441,7 +441,7 @@ public class RDDConverterUtils public Iterator<LabeledPoint> call(MatrixBlock arg0) throws Exception { - ArrayList<LabeledPoint> ret = new ArrayList<LabeledPoint>(); + ArrayList<LabeledPoint> ret = new ArrayList<>(); for( int i=0; i<arg0.getNumRows(); i++ ) { MatrixBlock tmp = arg0.sliceOperations(i, i, 0, arg0.getNumColumns()-2, new MatrixBlock()); ret.add(new LabeledPoint(arg0.getValue(i, arg0.getNumColumns()-1), createVector(tmp))); @@ -482,7 +482,7 @@ public class RDDConverterUtils throws IOException, DMLRuntimeException { //temporary list of indexed matrix values to prevent library dependencies - ArrayList<IndexedMatrixValue> rettmp = new ArrayList<IndexedMatrixValue>(); + ArrayList<IndexedMatrixValue> rettmp = new ArrayList<>(); rbuff.flushBufferToBinaryBlocks(rettmp); ret.addAll(SparkUtils.fromIndexedMatrixBlock(rettmp)); } @@ -500,7 +500,7 @@ public class RDDConverterUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Text> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); ReblockBuffer rbuff = new ReblockBuffer(_bufflen, _rlen, _clen, _brlen, _bclen); FastStringTokenizer st = new FastStringTokenizer(' '); @@ -541,8 +541,8 @@ public class RDDConverterUtils throws Exception { SerLongWritable slarg = new SerLongWritable(arg0._1()); - SerText starg = new SerText(arg0._2()); - return new Tuple2<LongWritable,Text>(slarg, starg); + SerText starg = new SerText(arg0._2()); + return new Tuple2<>(slarg, starg); } } @@ -555,8 +555,8 @@ public class RDDConverterUtils throws Exception { SerLongWritable slarg = new SerLongWritable(1L); - SerText starg = new SerText(arg0); - return new Tuple2<LongWritable,Text>(slarg, starg); + SerText starg = new SerText(arg0); + return new Tuple2<>(slarg, starg); } } @@ -575,7 +575,7 @@ public class RDDConverterUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<MatrixIndexes,MatrixCell>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); ReblockBuffer rbuff = new ReblockBuffer(_bufflen, _rlen, _clen, _brlen, _bclen); while( arg0.hasNext() ) @@ -679,7 +679,7 @@ public class RDDConverterUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<Text,Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); int ncblks = (int)Math.ceil((double)_clen/_bclen); MatrixIndexes[] ix = new MatrixIndexes[ncblks]; @@ -759,7 +759,7 @@ public class RDDConverterUtils int len = ix.length; for( int i=0; i<len; i++ ) if( mb[i] != null ) { - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix[i],mb[i])); + ret.add(new Tuple2<>(ix[i],mb[i])); mb[i].examSparsity(); //ensure right representation } } @@ -792,7 +792,7 @@ public class RDDConverterUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<org.apache.spark.mllib.regression.LabeledPoint,Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); int ncblks = (int)Math.ceil((double)_clen/_bclen); MatrixIndexes[] ix = new MatrixIndexes[ncblks]; @@ -873,10 +873,10 @@ public class RDDConverterUtils private static void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) throws DMLRuntimeException { - int len = ix.length; + int len = ix.length; for( int i=0; i<len; i++ ) if( mb[i] != null ) { - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix[i],mb[i])); + ret.add(new Tuple2<>(ix[i],mb[i])); mb[i].examSparsity(); //ensure right representation } } @@ -899,7 +899,7 @@ public class RDDConverterUtils MatrixIndexes ix = arg0._1(); MatrixBlock blk = arg0._2(); - ArrayList<String> ret = new ArrayList<String>(); + ArrayList<String> ret = new ArrayList<>(); //handle header information if(_props.hasHeader() && ix.getRowIndex()==1 ) { @@ -944,19 +944,14 @@ public class RDDConverterUtils public Iterator<Tuple2<Long,Tuple2<Long,MatrixBlock>>> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { - ArrayList<Tuple2<Long,Tuple2<Long,MatrixBlock>>> ret = - new ArrayList<Tuple2<Long,Tuple2<Long,MatrixBlock>>>(); - + ArrayList<Tuple2<Long,Tuple2<Long,MatrixBlock>>> ret = new ArrayList<>(); MatrixIndexes ix = arg0._1(); MatrixBlock blk = arg0._2(); - for( int i=0; i<blk.getNumRows(); i++ ) { MatrixBlock tmpBlk = blk.sliceOperations(i, i, 0, blk.getNumColumns()-1, new MatrixBlock()); long rix = UtilFunctions.computeCellIndex(ix.getRowIndex(), _brlen, i); - ret.add(new Tuple2<Long,Tuple2<Long,MatrixBlock>>(rix, - new Tuple2<Long,MatrixBlock>(ix.getColumnIndex(),tmpBlk))); + ret.add(new Tuple2<>(rix, new Tuple2<>(ix.getColumnIndex(),tmpBlk))); } - return ret.iterator(); } @@ -982,24 +977,20 @@ public class RDDConverterUtils { long rowIndex = arg0._1(); MatrixBlock[] tmpBlks = new MatrixBlock[_ncblks]; - //collect and sort input blocks Iterator<Tuple2<Long, MatrixBlock>> iter = arg0._2().iterator(); while( iter.hasNext() ) { Tuple2<Long, MatrixBlock> entry = iter.next(); tmpBlks[entry._1().intValue()-1] = entry._2(); } - //concatenate blocks MatrixBlock out = new MatrixBlock(1,(int)_clen, tmpBlks[0].isInSparseFormat()); - for( int i=0; i<_ncblks; i++ ) { - out.copy(0, 0, i*_bclen, (int)Math.min((i+1)*_bclen, _clen)-1, tmpBlks[i], false); - } + for( int i=0; i<_ncblks; i++ ) + out.copy(0, 0, i*_bclen, (int)Math.min((i+1)*_bclen, _clen)-1, tmpBlks[i], false); out.recomputeNonZeros(); - //output row block - return new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(rowIndex, 1),out); - } + return new Tuple2<>(new MatrixIndexes(rowIndex, 1),out); + } } ///////////////////////////////// @@ -1033,7 +1024,7 @@ public class RDDConverterUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<Row, Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); int ncblks = (int)Math.ceil((double)_clen/_bclen); MatrixIndexes[] ix = new MatrixIndexes[ncblks]; @@ -1059,7 +1050,7 @@ public class RDDConverterUtils int off = _containsID ? 1: 0; Object obj = _isVector ? tmp._1().get(off) : tmp._1(); for( int cix=1, pix=_isVector?0:off; cix<=ncblks; cix++ ) { - int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); + int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); //allocate sparse row once (avoid re-allocations) if( mb[cix-1].isInSparseFormat() ) { int lnnz = countNnz(obj, _isVector, pix, lclen); @@ -1118,7 +1109,7 @@ public class RDDConverterUtils int len = ix.length; for( int i=0; i<len; i++ ) if( mb[i] != null ) { - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix[i],mb[i])); + ret.add(new Tuple2<>(ix[i],mb[i])); mb[i].examSparsity(); //ensure right representation } } @@ -1169,7 +1160,7 @@ public class RDDConverterUtils throw new DMLRuntimeException("ID Column '" + DF_ID_COLUMN + "' expected to be 1-based, but found value: "+id); } - return new Tuple2<Row,Long>(arg0, id-1); + return new Tuple2<>(arg0, id-1); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java index 046c015..2fddce3 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDSortUtils.java @@ -182,10 +182,10 @@ public class RDDSortUtils //flip sort indices from <source ix in target pos> to <target ix in source pos> MatrixBlock sortedIxSrc = new MatrixBlock(sortedIx.getNumRows(), 1, false); for (int i=0; i < sortedIx.getNumRows(); i++) - sortedIxSrc.quickSetValue((int)sortedIx.quickGetValue(i,0)-1, 0, i+1); + sortedIxSrc.quickSetValue((int)sortedIx.quickGetValue(i,0)-1, 0, i+1); //broadcast index vector - PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<MatrixBlock>(sortedIxSrc, brlen, bclen); + PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<>(sortedIxSrc, brlen, bclen); Broadcast<PartitionedBlock<MatrixBlock>> _pmb = sec.getSparkContext().broadcast(pmb); //sort data with broadcast index vector @@ -214,7 +214,7 @@ public class RDDSortUtils public Iterator<DoublePair> call(Tuple2<MatrixBlock,MatrixBlock> arg0) throws Exception { - ArrayList<DoublePair> ret = new ArrayList<DoublePair>(); + ArrayList<DoublePair> ret = new ArrayList<>(); MatrixBlock mb1 = arg0._1(); MatrixBlock mb2 = arg0._2(); @@ -243,19 +243,18 @@ public class RDDSortUtils public Iterator<Tuple2<ValueIndexPair,Double>> call(Tuple2<MatrixIndexes,MatrixBlock> arg0) throws Exception { - ArrayList<Tuple2<ValueIndexPair,Double>> ret = new ArrayList<Tuple2<ValueIndexPair,Double>>(); + ArrayList<Tuple2<ValueIndexPair,Double>> ret = new ArrayList<>(); MatrixIndexes ix = arg0._1(); MatrixBlock mb = arg0._2(); long ixoffset = (ix.getRowIndex()-1)*_brlen; for( int i=0; i<mb.getNumRows(); i++) { double val = mb.quickGetValue(i, 0); - ret.add(new Tuple2<ValueIndexPair,Double>( - new ValueIndexPair(val,ixoffset+i+1), val)); + ret.add(new Tuple2<>(new ValueIndexPair(val,ixoffset+i+1), val)); } return ret.iterator(); - } + } } private static class CreateDoubleKeyFunction implements Function<Double,Double> @@ -290,7 +289,7 @@ public class RDDSortUtils public Tuple2<Long, Long> call(Tuple2<ValueIndexPair,Long> arg0) throws Exception { - return new Tuple2<Long,Long>(arg0._1().ix, arg0._2()); + return new Tuple2<>(arg0._1().ix, arg0._2()); } } @@ -311,8 +310,7 @@ public class RDDSortUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<Double,Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); - + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ix = null; MatrixBlock mb = null; @@ -326,7 +324,7 @@ public class RDDSortUtils if( ix == null || ix.getRowIndex() != rix ) { if( ix !=null ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); mb = new MatrixBlock((int)len, 1, false); @@ -337,8 +335,7 @@ public class RDDSortUtils //flush last block if( mb!=null && mb.getNonZeros() != 0 ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); - + ret.add(new Tuple2<>(ix,mb)); return ret.iterator(); } } @@ -359,7 +356,7 @@ public class RDDSortUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<DoublePair,Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ix = null; MatrixBlock mb = null; @@ -374,10 +371,10 @@ public class RDDSortUtils if( ix == null || ix.getRowIndex() != rix ) { if( ix !=null ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); - mb = new MatrixBlock((int)len, 2, false); + mb = new MatrixBlock((int)len, 2, false); } mb.quickSetValue(pos, 0, val._1.val1); @@ -386,7 +383,7 @@ public class RDDSortUtils //flush last block if( mb!=null && mb.getNonZeros() != 0 ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); return ret.iterator(); } @@ -408,7 +405,7 @@ public class RDDSortUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<ValueIndexPair,Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ix = null; MatrixBlock mb = null; @@ -423,7 +420,7 @@ public class RDDSortUtils if( ix == null || ix.getRowIndex() != rix ) { if( ix !=null ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); mb = new MatrixBlock((int)len, 1, false); @@ -434,7 +431,7 @@ public class RDDSortUtils //flush last block if( mb!=null && mb.getNonZeros() != 0 ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); return ret.iterator(); } @@ -456,7 +453,7 @@ public class RDDSortUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Iterator<Tuple2<Long,Long>> arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); MatrixIndexes ix = null; MatrixBlock mb = null; @@ -471,7 +468,7 @@ public class RDDSortUtils if( ix == null || ix.getRowIndex() != rix ) { if( ix !=null ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); long len = UtilFunctions.computeBlockSize(_rlen, rix, _brlen); ix = new MatrixIndexes(rix,1); mb = new MatrixBlock((int)len, 1, false); @@ -482,7 +479,7 @@ public class RDDSortUtils //flush last block if( mb!=null && mb.getNonZeros() != 0 ) - ret.add(new Tuple2<MatrixIndexes,MatrixBlock>(ix,mb)); + ret.add(new Tuple2<>(ix,mb)); return ret.iterator(); } @@ -560,7 +557,7 @@ public class RDDSortUtils _currBlk = null; } - return new Tuple2<MatrixIndexes,RowMatrixBlock>(lix, new RowMatrixBlock(len, pos, tmp)); + return new Tuple2<>(lix, new RowMatrixBlock(len, pos, tmp)); } catch(Exception ex) { throw new RuntimeException(ex); @@ -649,7 +646,7 @@ public class RDDSortUtils _currBlk = null; } - return new Tuple2<MatrixIndexes,RowMatrixBlock>(lix, new RowMatrixBlock(len, pos, tmp)); + return new Tuple2<>(lix, new RowMatrixBlock(len, pos, tmp)); } catch(Exception ex) { throw new RuntimeException(ex); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java index 4e7866d..977cd10 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/SparkUtils.java @@ -66,47 +66,47 @@ public class SparkUtils } public static Tuple2<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlock( IndexedMatrixValue in ){ - return new Tuple2<MatrixIndexes,MatrixBlock>(in.getIndexes(), (MatrixBlock)in.getValue()); + return new Tuple2<>(in.getIndexes(), (MatrixBlock)in.getValue()); } public static ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlock( ArrayList<IndexedMatrixValue> in ) { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); for( IndexedMatrixValue imv : in ) ret.add(fromIndexedMatrixBlock(imv)); return ret; } public static Pair<MatrixIndexes,MatrixBlock> fromIndexedMatrixBlockToPair( IndexedMatrixValue in ){ - return new Pair<MatrixIndexes,MatrixBlock>(in.getIndexes(), (MatrixBlock)in.getValue()); + return new Pair<>(in.getIndexes(), (MatrixBlock)in.getValue()); } public static ArrayList<Pair<MatrixIndexes,MatrixBlock>> fromIndexedMatrixBlockToPair( ArrayList<IndexedMatrixValue> in ) { - ArrayList<Pair<MatrixIndexes,MatrixBlock>> ret = new ArrayList<Pair<MatrixIndexes,MatrixBlock>>(); + ArrayList<Pair<MatrixIndexes,MatrixBlock>> ret = new ArrayList<>(); for( IndexedMatrixValue imv : in ) ret.add(fromIndexedMatrixBlockToPair(imv)); return ret; } public static Tuple2<Long,FrameBlock> fromIndexedFrameBlock( Pair<Long, FrameBlock> in ){ - return new Tuple2<Long, FrameBlock>(in.getKey(), in.getValue()); + return new Tuple2<>(in.getKey(), in.getValue()); } public static ArrayList<Tuple2<Long,FrameBlock>> fromIndexedFrameBlock( ArrayList<Pair<Long, FrameBlock>> in ) { - ArrayList<Tuple2<Long, FrameBlock>> ret = new ArrayList<Tuple2<Long, FrameBlock>>(); + ArrayList<Tuple2<Long, FrameBlock>> ret = new ArrayList<>(); for( Pair<Long, FrameBlock> ifv : in ) ret.add(fromIndexedFrameBlock(ifv)); return ret; } public static ArrayList<Pair<Long,Long>> toIndexedLong( List<Tuple2<Long, Long>> in ) { - ArrayList<Pair<Long, Long>> ret = new ArrayList<Pair<Long, Long>>(); + ArrayList<Pair<Long, Long>> ret = new ArrayList<>(); for( Tuple2<Long, Long> e : in ) - ret.add(new Pair<Long,Long>(e._1(), e._2())); + ret.add(new Pair<>(e._1(), e._2())); return ret; } public static Pair<Long,FrameBlock> toIndexedFrameBlock( Tuple2<Long,FrameBlock> in ) { - return new Pair<Long,FrameBlock>(in._1(), in._2()); + return new Pair<>(in._1(), in._2()); } /** @@ -281,7 +281,7 @@ public class SparkUtils public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call(Long arg0) throws Exception { - ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> list = new ArrayList<Tuple2<MatrixIndexes,MatrixBlock>>(); + ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> list = new ArrayList<>(); long ncblks = _mc.getNumColBlocks(); long nblocksU = Math.min(arg0+_pNumBlocks, _mc.getNumBlocks()); for( long i=arg0; i<nblocksU; i++ ) { @@ -289,9 +289,8 @@ public class SparkUtils long cix = 1 + i % ncblks; int lrlen = UtilFunctions.computeBlockSize(_mc.getRows(), rix, _mc.getRowsPerBlock()); int lclen = UtilFunctions.computeBlockSize(_mc.getCols(), cix, _mc.getColsPerBlock()); - list.add(new Tuple2<MatrixIndexes,MatrixBlock>( - new MatrixIndexes(rix,cix), - new MatrixBlock(lrlen, lclen, true))); + list.add(new Tuple2<>(new MatrixIndexes(rix,cix), + new MatrixBlock(lrlen, lclen, true))); } return list.iterator(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java index 45718f9..6d162cc 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlockParallel.java @@ -41,7 +41,7 @@ import org.apache.sysml.runtime.matrix.data.FrameBlock; */ public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock { - + @Override protected void readBinaryBlockFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, long rlen, long clen ) throws IOException, DMLRuntimeException { @@ -51,12 +51,12 @@ public class FrameReaderBinaryBlockParallel extends FrameReaderBinaryBlock { //create read tasks for all files ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<ReadFileTask> tasks = new ArrayList<ReadFileTask>(); + ArrayList<ReadFileTask> tasks = new ArrayList<>(); for( Path lpath : IOUtilFunctions.getSequenceFilePaths(fs, path) ) tasks.add(new ReadFileTask(lpath, job, fs, dest)); //wait until all tasks have been executed - List<Future<Object>> rt = pool.invokeAll(tasks); + List<Future<Object>> rt = pool.invokeAll(tasks); pool.shutdown(); //check for exceptions http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java index 76da0de..275d647 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSV.java @@ -225,7 +225,6 @@ public class FrameReaderTextCSV extends FrameReader IOUtilFunctions.closeSilently(reader); } } - - return new Pair<Integer,Integer>(nrow, ncol); + return new Pair<>(nrow, ncol); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java index 0f8cb3a..3819962 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java @@ -71,21 +71,21 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV Math.min(numThreads, splits.length)); //compute num rows per split - ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>(); + ArrayList<CountRowsTask> tasks = new ArrayList<>(); for( int i=0; i<splits.length; i++ ) tasks.add(new CountRowsTask(splits[i], informat, job, _props.hasHeader(), i==0)); List<Future<Long>> cret = pool.invokeAll(tasks); //compute row offset per split via cumsum on row counts long offset = 0; - List<Long> offsets = new ArrayList<Long>(); + List<Long> offsets = new ArrayList<>(); for( Future<Long> count : cret ) { offsets.add(offset); offset += count.get(); } //read individual splits - ArrayList<ReadRowsTask> tasks2 = new ArrayList<ReadRowsTask>(); + ArrayList<ReadRowsTask> tasks2 = new ArrayList<>(); for( int i=0; i<splits.length; i++ ) tasks2.add( new ReadRowsTask(splits[i], informat, job, dest, offsets.get(i).intValue(), i==0)); List<Future<Object>> rret = pool.invokeAll(tasks2); @@ -103,7 +103,7 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV @Override protected Pair<Integer,Integer> computeCSVSize( Path path, JobConf job, FileSystem fs) throws IOException - { + { int numThreads = OptimizerUtils.getParallelTextReadParallelism(); TextInputFormat informat = new TextInputFormat(); @@ -117,7 +117,7 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV int nrow = 0; ExecutorService pool = Executors.newFixedThreadPool(numThreads); try { - ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>(); + ArrayList<CountRowsTask> tasks = new ArrayList<>(); for( int i=0; i<splits.length; i++ ) tasks.add(new CountRowsTask(splits[i], informat, job, _props.hasHeader(), i==0)); List<Future<Long>> cret = pool.invokeAll(tasks); @@ -130,8 +130,7 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV finally { pool.shutdown(); } - - return new Pair<Integer,Integer>(nrow, ncol); + return new Pair<>(nrow, ncol); } private static class CountRowsTask implements Callable<Long> http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java index 96b49d4..7e88030 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCellParallel.java @@ -60,12 +60,12 @@ public class FrameReaderTextCellParallel extends FrameReaderTextCell //create read tasks for all splits ExecutorService pool = Executors.newFixedThreadPool(numThreads); InputSplit[] splits = informat.getSplits(job, numThreads); - ArrayList<ReadTask> tasks = new ArrayList<ReadTask>(); + ArrayList<ReadTask> tasks = new ArrayList<>(); for( InputSplit split : splits ) tasks.add(new ReadTask(split, informat, job, dest)); //wait until all tasks have been executed - List<Future<Object>> rt = pool.invokeAll(tasks); + List<Future<Object>> rt = pool.invokeAll(tasks); pool.shutdown(); //check for exceptions http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java index 52f1ed0..763d8a8 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlockParallel.java @@ -73,7 +73,7 @@ public class FrameWriterBinaryBlockParallel extends FrameWriterBinaryBlock try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>(); + ArrayList<WriteFileTask> tasks = new ArrayList<>(); int blklen = (int)Math.ceil((double)rlen / blen / numThreads) * blen; for(int i=0; i<numThreads & i*blklen<rlen; i++) { Path newPath = new Path(path, IOUtilFunctions.getPartFileName(i)); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java index fe4fd39..4c2f39d 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSVParallel.java @@ -75,7 +75,7 @@ public class FrameWriterTextCSVParallel extends FrameWriterTextCSV try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>(); + ArrayList<WriteFileTask> tasks = new ArrayList<>(); int blklen = (int)Math.ceil((double)rlen / numThreads); for(int i=0; i<numThreads & i*blklen<rlen; i++) { Path newPath = new Path(path, IOUtilFunctions.getPartFileName(i)); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java index f42ca41..ba55454 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCellParallel.java @@ -70,7 +70,7 @@ public class FrameWriterTextCellParallel extends FrameWriterTextCell try { ExecutorService pool = Executors.newFixedThreadPool(numThreads); - ArrayList<WriteFileTask> tasks = new ArrayList<WriteFileTask>(); + ArrayList<WriteFileTask> tasks = new ArrayList<>(); int blklen = (int)Math.ceil((double)rlen / numThreads); for(int i=0; i<numThreads & i*blklen<rlen; i++) { Path newPath = new Path(path, IOUtilFunctions.getPartFileName(i)); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 3ac4dc4..9900c45 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -171,7 +171,7 @@ public class IOUtilFunctions return new String[]{""}; // scan string and create individual tokens - ArrayList<String> tokens = new ArrayList<String>(); + ArrayList<String> tokens = new ArrayList<>(); int from = 0, to = 0; int len = str.length(); int dlen = delim.length(); @@ -464,7 +464,7 @@ public class IOUtilFunctions if( fs.isDirectory(file) || IOUtilFunctions.isObjectStoreFileScheme(file) ) { - LinkedList<Path> tmp = new LinkedList<Path>(); + LinkedList<Path> tmp = new LinkedList<>(); FileStatus[] dStatus = fs.listStatus(file); for( FileStatus fdStatus : dStatus ) if( !fdStatus.getPath().getName().startsWith("_") //skip internal files
