Repository: systemml Updated Branches: refs/heads/master b7fd340fe -> 92ee2cbf8
http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java index 6c0b463..1fbc341 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSVParallel.java @@ -106,7 +106,7 @@ public class ReaderTextCSVParallel extends MatrixReader ret.examSparsity(); // sanity check for parallel row count (since determined internally) - if (rlen > 0 && rlen != ret.getNumRows()) + if (rlen >= 0 && rlen != ret.getNumRows()) throw new DMLRuntimeException("Read matrix inconsistent with given meta data: " + "expected nrow="+ rlen + ", real nrow=" + ret.getNumRows()); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java index ae867bf..a8f31cb 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java @@ -281,10 +281,10 @@ public class WriterBinaryBlock extends MatrixWriter { case ROW_BLOCK_WISE_N: { - long numBlocks = ((rlen-1)/brlen)+1; + long numBlocks = Math.max(((rlen-1)/brlen)+1, 1); long numPartBlocks = (long)Math.ceil(((double)DistributedCacheInput.PARTITION_SIZE)/clen/brlen); - int count = 0; + int count = 0; for( int k = 0; k<numBlocks; k+=numPartBlocks ) { // 1) create sequence file writer, with right replication factor @@ -331,10 +331,10 @@ public class WriterBinaryBlock extends MatrixWriter } case COLUMN_BLOCK_WISE_N: { - long numBlocks = ((clen-1)/bclen)+1; + long numBlocks = Math.max(((clen-1)/bclen)+1, 1); long numPartBlocks = (long)Math.ceil(((double)DistributedCacheInput.PARTITION_SIZE)/rlen/bclen); - int count = 0; + int count = 0; for( int k = 0; k<numBlocks; k+=numPartBlocks ) { // 1) create sequence file writer, with right replication factor http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java index a7df822..89c1bee 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/CSVReblockMR.java @@ -274,7 +274,7 @@ public class CSVReblockMR AssignRowIDMRReturn ret1 = CSVReblockMR.runAssignRowIDMRJob(inputs, inputInfos, brlens, bclens, reblockInstructions, replication, smallestFiles); for(int i=0; i<rlens.length; i++) - if( (rlens[i]>0 && rlens[i]!=ret1.rlens[i]) || (clens[i]>0 && clens[i]!=ret1.clens[i]) ) + if( (rlens[i]>=0 && rlens[i]!=ret1.rlens[i]) || (clens[i]>=0 && clens[i]!=ret1.clens[i]) ) throw new RuntimeException("Dimension doesn't mach for input matrix "+i+", expected ("+rlens[i]+", "+clens[i]+") but real ("+ret1.rlens[i]+", "+ret1.clens[i]+")"); JobReturn ret= CSVReblockMR.runCSVReblockJob(null, inputs, inputInfos, ret1.rlens, ret1.clens, brlens, bclens, reblockInstructions, otherInstructionsInReducer, numReducers, replication, resultIndexes, outputs, outputInfos, ret1.counterFile, smallestFiles); @@ -451,12 +451,9 @@ public class CSVReblockMR MapReduceTool.deleteFileIfExistOnHDFS(counterFile, job); /* Process different counters */ - Group group=runjob.getCounters().getGroup(MRJobConfiguration.NUM_NONZERO_CELLS); for(int i=0; i<resultIndexes.length; i++) { - // number of non-zeros stats[i].setNonZeros(group.getCounter(Integer.toString(i))); - // System.out.println("result #"+resultIndexes[i]+" ===>\n"+stats[i]); } return new JobReturn(stats, outputInfos, runjob.isSuccessful()); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java index b4e6eda..a3d08f3 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/CombineMR.java @@ -204,22 +204,17 @@ public class CombineMR valueBuff.setOtherValue(in2.getValue().getValue(r, c)); valueBuff.setWeight(in3.getValue().getValue(r, c)); for(int i: outputIndexes) - { collectFinalMultipleOutputs.collectOutput(keyBuff, valueBuff, i, reporter); - //System.out.println("output: "+keyBuff+" -- "+valueBuff); - } } } catch (Exception e) { throw new RuntimeException(e); } - } private void processBinaryCombineInstruction(CombineBinaryInstruction ins, Reporter reporter) - throws IOException + throws IOException { - IndexedMatrixValue in1=cachedValues.getFirst(ins.input1); IndexedMatrixValue in2=cachedValues.getFirst(ins.input2); if(in1==null && in2==null) @@ -239,21 +234,14 @@ public class CombineMR in2.getValue().getNumColumns()); } - if(in2==null) - { + if(in2==null) { in2=zeroInput; in2.getValue().reset(in1.getValue().getNumRows(), in1.getValue().getNumColumns()); } - //System.out.println("in1:"+in1); - //System.out.println("in2:"+in2); - //process instruction try { - /*in1.getValue().combineOperations(in2.getValue(), collectFinalMultipleOutputs, - reporter, keyBuff, valueBuff, getOutputIndexes(ins.output));*/ - ArrayList<Integer> outputIndexes = outputIndexesMapping.get(ins.output); for(int r=0; r<in1.getValue().getNumRows(); r++) for(int c=0; c<in1.getValue().getNumColumns(); c++) http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java index f95dbe7..3a669bf 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/DataGenMR.java @@ -154,9 +154,9 @@ public class DataGenMR //seed generation Well1024a bigrand = LibMatrixDatagen.setupSeedsForRand(randInst.getSeed()); - for(long r = 0; r < rlens[i]; r += brlens[i]) { + for(long r = 0; r < Math.max(rlens[i],1); r += brlens[i]) { long curBlockRowSize = Math.min(brlens[i], (rlens[i] - r)); - for(long c = 0; c < clens[i]; c += bclens[i]) { + for(long c = 0; c < Math.max(clens[i],1); c += bclens[i]) { long curBlockColSize = Math.min(bclens[i], (clens[i] - c)); sb.append((r / brlens[i]) + 1); sb.append(','); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java index 0c06424..21c42cd 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java @@ -81,18 +81,15 @@ public class MatrixCharacteristics implements Serializable } - public MatrixCharacteristics(long nr, long nc, int bnr, int bnc) - { + public MatrixCharacteristics(long nr, long nc, int bnr, int bnc) { set(nr, nc, bnr, bnc); } - public MatrixCharacteristics(long nr, long nc, int bnr, int bnc, long nnz) - { + public MatrixCharacteristics(long nr, long nc, int bnr, int bnc, long nnz) { set(nr, nc, bnr, bnc, nnz); } - public MatrixCharacteristics(MatrixCharacteristics that) - { + public MatrixCharacteristics(MatrixCharacteristics that) { set(that.numRows, that.numColumns, that.numRowsPerBlock, that.numColumnsPerBlock, that.nonZero); } @@ -148,11 +145,13 @@ public class MatrixCharacteristics implements Serializable } public long getNumRowBlocks() { - return (long) Math.ceil((double)getRows() / getRowsPerBlock()); + //number of row blocks w/ awareness of zero rows + return Math.max((long) Math.ceil((double)getRows() / getRowsPerBlock()), 1); } public long getNumColBlocks() { - return (long) Math.ceil((double)getCols() / getColsPerBlock()); + //number of column blocks w/ awareness of zero columns + return Math.max((long) Math.ceil((double)getCols() / getColsPerBlock()), 1); } @Override @@ -184,29 +183,28 @@ public class MatrixCharacteristics implements Serializable } public boolean dimsKnown() { - return ( numRows > 0 && numColumns > 0 ); + return ( numRows >= 0 && numColumns >= 0 ); } public boolean dimsKnown(boolean includeNnz) { - return ( numRows > 0 && numColumns > 0 && (!includeNnz || nonZero>=0)); + return ( numRows >= 0 && numColumns >= 0 && (!includeNnz || nonZero >= 0)); } public boolean rowsKnown() { - return ( numRows > 0 ); + return ( numRows >= 0 ); } public boolean colsKnown() { - return ( numColumns > 0 ); + return ( numColumns >= 0 ); } public boolean nnzKnown() { return ( nonZero >= 0 ); } - public boolean mightHaveEmptyBlocks() - { - long singleBlk = Math.min(numRows, numRowsPerBlock) - * Math.min(numColumns, numColumnsPerBlock); + public boolean mightHaveEmptyBlocks() { + long singleBlk = Math.max(Math.min(numRows, numRowsPerBlock),1) + * Math.max(Math.min(numColumns, numColumnsPerBlock),1); return !nnzKnown() || (nonZero < numRows*numColumns - singleBlk); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java index 57d16e1..9fc7e1e 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/ReblockMR.java @@ -184,9 +184,7 @@ public class ReblockMR Group group=runjob.getCounters().getGroup(MRJobConfiguration.NUM_NONZERO_CELLS); for(int i=0; i<resultIndexes.length; i++) { - // number of non-zeros stats[i].setNonZeros( group.getCounter(Integer.toString(i)) ); - // System.out.println("result #"+resultIndexes[i]+" ===>\n"+stats[i]); } return new JobReturn(stats, outputInfos, runjob.isSuccessful()); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java index 0252af3..6df49f2 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDatagen.java @@ -592,25 +592,21 @@ public class LibMatrixDatagen private static void checkMatrixDimensionsAndSparsity(int rows, int cols, double sp) throws DMLRuntimeException { - if( rows <= 0 || cols <= 0 || sp < 0 || sp > 1) + if( rows < 0 || cols < 0 || sp < 0 || sp > 1) throw new DMLRuntimeException("Invalid matrix characteristics: "+rows+"x"+cols+", "+sp); } // modified version of java.util.nextInt - private static long nextLong(Random r, long n) { - if (n <= 0) - throw new IllegalArgumentException("n must be positive"); - - //if ((n & -n) == n) // i.e., n is a power of 2 - // return ((n * (long)r.nextLong()) >> 31); - - long bits, val; - do { - bits = (r.nextLong() << 1) >>> 1; - val = bits % n; - } while (bits - val + (n-1) < 0L); - return val; - } + private static long nextLong(Random r, long n) { + if (n <= 0) + throw new IllegalArgumentException("n must be positive"); + long bits, val; + do { + bits = (r.nextLong() << 1) >>> 1; + val = bits % n; + } while (bits - val + (n-1) < 0L); + return val; + } private static class RandTask implements Callable<Long> { http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java index d7811bf..8767b13 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixBlock.java @@ -3882,11 +3882,9 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab for(; r<=Math.min(range.rowEnd, sparseBlock.numRows()-1); r++) sliceHelp(r, range, colCut, bottomleft, bottomright, -rowCut, normalBlockRowFactor, normalBlockColFactor); - //System.out.println("in: \n"+this); - //System.out.println("outlist: \n"+outlist); } - }else - { + } + else { if(denseBlock!=null) { double[] a = getDenseBlockValues(); @@ -5526,21 +5524,6 @@ public class MatrixBlock extends MatrixValue implements CacheBlock, Externalizab return !sparse || DEFAULT_SPARSEBLOCK == SparseBlock.Type.MCSR; //only MCSR thread-safe } - public void print() - { - System.out.println("sparse = "+sparse); - if(!sparse) - System.out.println("nonzeros = "+nonZeros); - for(int i=0; i<rlen; i++) - { - for(int j=0; j<clen; j++) - { - System.out.print(quickGetValue(i, j)+"\t"); - } - System.out.println(); - } - } - @Override public int compareTo(Object arg0) { throw new RuntimeException("CompareTo should never be called for matrix blocks."); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java index 8565409..4ede878 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/MatrixPackedCell.java @@ -93,8 +93,8 @@ public class MatrixPackedCell extends MatrixCell buffer=(KahanObject) aggOp.increOp.fn.execute(buffer, newWithCor.value, newWithCor.getExtraByPostition(0)); value=buffer._sum; extras[0]=buffer._correction; - // System.out.println("--- "+buffer); - }else if(aggOp.correctionLocation==CorrectionLocationType.LASTROW || aggOp.correctionLocation==CorrectionLocationType.LASTTWOCOLUMNS) + } + else if(aggOp.correctionLocation==CorrectionLocationType.LASTROW || aggOp.correctionLocation==CorrectionLocationType.LASTTWOCOLUMNS) { checkAndAllocateSpace(2); KahanObject buffer=new KahanObject(value, extras[0]); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java index eb8f6c0..f57e9c7 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRMapper.java @@ -55,12 +55,10 @@ implements Mapper<Writable, Writable, Writable, Writable> public void map(Writable index, Writable cell, OutputCollector<Writable, Writable> out, Reporter report) throws IOException { - if(firsttime) - { + if(firsttime) { cachedCollector=out; firsttime=false; } - // System.out.println("input: "+index+" -- "+cell); commonMap(index, cell, out, report); } @@ -81,11 +79,7 @@ implements Mapper<Writable, Writable, Writable, Writable> WeightedPair inputPair=(WeightedPair)input.getValue(); CM_N_COVCell cmValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue(); try { - - // System.out.println("~~~~~\nold: "+cmValue.getCM_N_COVObject()); - // System.out.println("add: "+inputPair); lcmFn.execute(cmValue.getCM_N_COVObject(), inputPair.getValue(), inputPair.getWeight()); - // System.out.println("new: "+cmValue.getCM_N_COVObject()); } catch (DMLRuntimeException e) { throw new IOException(e); } @@ -96,15 +90,10 @@ implements Mapper<Writable, Writable, Writable, Writable> IndexedMatrixValue input = cachedValues.getFirst(tag); if(input==null) continue; - //System.out.println("*** cached Value:\n"+cachedValues); WeightedPair inputPair=(WeightedPair)input.getValue(); CM_N_COVCell comValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue(); try { - - //System.out.println("~~~~~\nold: "+comValue.getCM_N_COVObject()); - // System.out.println("add: "+inputPair); covFn.execute(comValue.getCM_N_COVObject(), inputPair.getValue(), inputPair.getOtherValue(), inputPair.getWeight()); - // System.out.println("new: "+comValue.getCM_N_COVObject()); } catch (DMLRuntimeException e) { throw new IOException(e); } @@ -116,18 +105,14 @@ implements Mapper<Writable, Writable, Writable, Writable> { if(cachedCollector!=null) { - for(byte tag: cmTags) - { + for(byte tag: cmTags) { CM_N_COVCell cmValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue(); cachedCollector.collect(new TaggedFirstSecondIndexes(1, tag, 1), cmValue); - //System.out.println("output to reducer with tag:"+tag+" and value: "+cmValue); } - for(byte tag: covTags) - { + for(byte tag: covTags) { CM_N_COVCell comValue = (CM_N_COVCell) cmNcovCache.getFirst(tag).getValue(); cachedCollector.collect(new TaggedFirstSecondIndexes(1, tag, 1), comValue); - //System.out.println("output to reducer with tag:"+tag+" and value: "+comValue); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java index 2fd0ac1..6a7175c 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/CMCOVMRReducer.java @@ -120,10 +120,8 @@ implements Reducer<TaggedFirstSecondIndexes, MatrixValue, MatrixIndexes, MatrixV } ArrayList<Integer> outputIndexes = outputIndexesMapping.get(in.output); - for(int i: outputIndexes) - { + for(int i: outputIndexes) { collectOutput_N_Increase_Counter(outIndex, outCell, i, report); - // System.out.println("final output: "+outIndex+" -- "+outCell); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java index 82269f7..8122dcd 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/DistributedCacheInput.java @@ -34,8 +34,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.util.DataConverter; public class DistributedCacheInput -{ - +{ //internal partitioning parameter (threshold and partition size) public static final long PARTITION_SIZE = 4000000; //32MB //public static final String PARTITION_SUFFIX = "_dp"; @@ -137,7 +136,7 @@ public class DistributedCacheInput //read matrix partition (or entire vector) try - { + { ReaderBinaryBlock reader = (ReaderBinaryBlock) MatrixReaderFactory.createMatrixReader(InputInfo.BinaryBlockInputInfo); reader.setLocalFS( !MRBaseForCommonInstructions.isJobLocal ); ArrayList<IndexedMatrixValue> tmp = reader.readIndexedMatrixBlocksFromHDFS(fname, _rlen, _clen, _brlen, _bclen); @@ -147,7 +146,6 @@ public class DistributedCacheInput if( dataBlocks==null ) dataBlocks = new IndexedMatrixValue[rowBlocks][colBlocks]; - for (IndexedMatrixValue val : tmp) { MatrixIndexes idx = val.getIndexes(); dataBlocks[(int) idx.getRowIndex() - 1][(int) idx.getColumnIndex() - 1] = val; http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java index 3e83272..605b526 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/GMRMapper.java @@ -169,8 +169,6 @@ implements Mapper<Writable, Writable, Writable, Writable> //assign the temporay vairables try { - // System.out.println(valueClass.getName()); - // System.out.println(MatrixCell.class.getName()); if(job.getMapOutputValueClass().equals(TaggedMatrixPackedCell.class)) taggedValueBuffer=TaggedMatrixValue.createObject(MatrixPackedCell.class); else @@ -232,15 +230,8 @@ implements Mapper<Writable, Writable, Writable, Writable> boolean dimsUnknown = false; for(int i=0; i<resultIndexes.length; i++) { cachedReporter.incrCounter(MRJobConfiguration.NUM_NONZERO_CELLS, Integer.toString(i), resultsNonZeros[i]); - if ( resultDimsUnknown!=null && resultDimsUnknown[i] != (byte) 0 ) { dimsUnknown = true; - // Each counter is of the form: (group, name) - // where group = max_rowdim_resultindex; name = taskid - //System.out.println("--> before i="+i+", row = " + cachedReporter.getCounter("max_rowdim_"+i, ""+taskid).getCounter() + ", col = " + cachedReporter.getCounter("max_coldim_"+i, ""+taskid).getCounter()); - //cachedReporter.getCounter(MRJobConfiguration.MAX_ROW_DIMENSION, Integer.toString(i)).increment(resultsMaxRowDims[i]); - //cachedReporter.getCounter(MRJobConfiguration.MAX_COL_DIMENSION, Integer.toString(i)).increment(resultsMaxColDims[i]); - //System.out.println("--> after i="+i+", row = " + cachedReporter.getCounter("max_rowdim_"+i, ""+taskid).getCounter() + ", col = " + cachedReporter.getCounter("max_coldim_"+i, ""+taskid).getCounter()); } } if ( dimsUnknown ) { http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java index 1b17aca..7dbc899 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRCache.java @@ -67,17 +67,13 @@ public abstract class MMCJMRCache _buffer[i] = new Pair<>(new MatrixIndexes(), valueClass.newInstance()); if( buffMap ) _bufferMap = new HashMap<>(); - - //System.out.println("allocated buffer: "+_bufferCapacity); } - protected void constructLocalFilePrefix(String fname) - { + protected void constructLocalFilePrefix(String fname) { //get random localdir (to spread load across available disks) String[] localDirs = _job.get(MRConfigurationNames.MR_CLUSTER_LOCAL_DIR).split(","); Random rand = new Random(); int randPos = rand.nextInt(localDirs.length); - //construct file prefix String hadoopLocalDir = localDirs[ randPos ]; String uniqueSubDir = MapReduceTool.getGloballyUniqueName(_job); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java index f96e4ef..b6b8dfc 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMCJMRMapper.java @@ -102,15 +102,12 @@ implements Mapper<Writable, Writable, Writable, Writable> ArrayList<IndexedMatrixValue> blkList1 = cachedValues.get(aggBinInstruction.input1); if( blkList1 != null ) for(IndexedMatrixValue result:blkList1) - if(result!=null) - { + if(result!=null) { taggedIndexes.setTag(tagForLeft); taggedIndexes.setIndexes(result.getIndexes().getColumnIndex(), result.getIndexes().getRowIndex()); - if( !((MatrixBlock)result.getValue()).isEmptyBlock() ) out.collect(taggedIndexes, result.getValue()); - //System.out.println("In Mapper: output "+taggedIndexes+" "+ result.getValue().getNumRows()+"x"+result.getValue().getNumColumns()); } //output the key value pair for the right matrix @@ -118,15 +115,12 @@ implements Mapper<Writable, Writable, Writable, Writable> ArrayList<IndexedMatrixValue> blkList2 = cachedValues.get(aggBinInstruction.input2); if( blkList2 != null ) for(IndexedMatrixValue result:blkList2) - if(result!=null) - { + if(result!=null) { taggedIndexes.setTag(tagForRight); taggedIndexes.setIndexes(result.getIndexes().getRowIndex(), result.getIndexes().getColumnIndex()); - if( !((MatrixBlock)result.getValue()).isEmptyBlock() ) out.collect(taggedIndexes, result.getValue()); - //System.out.println("In Mapper: output "+taggedIndexes+" "+ result.getValue().getNumRows()+"x"+result.getValue().getNumColumns()); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java index c0455b3..4994576 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRMapper.java @@ -68,34 +68,29 @@ implements Mapper<Writable, Writable, Writable, Writable> //output the left matrix if(aggBinInput1s.contains(output)) { - for(long j=0; j<numRepeats.get(output); j++) - { + for(long j=0; j<numRepeats.get(output); j++) { triplebuffer.setIndexes(result.getIndexes().getRowIndex(), j+1, result.getIndexes().getColumnIndex()); taggedValue.setBaseObject(result.getValue()); taggedValue.setTag(output); out.collect(triplebuffer, taggedValue); - //System.out.println("output to reducer: "+triplebuffer+"\n"+taggedValue); } - }else if(aggBinInput2s.contains(output))//output the right matrix + } + else if(aggBinInput2s.contains(output))//output the right matrix { - for(long i=0; i<numRepeats.get(output); i++) - { + for(long i=0; i<numRepeats.get(output); i++) { triplebuffer.setIndexes(i+1, result.getIndexes().getColumnIndex(), result.getIndexes().getRowIndex()); taggedValue.setBaseObject(result.getValue()); taggedValue.setTag(output); out.collect(triplebuffer, taggedValue); - //System.out.println("output to reducer: "+triplebuffer+"\n"+taggedValue); } }else //output other matrix that are not involved in aggregate binary { triplebuffer.setIndexes(result.getIndexes().getRowIndex(), result.getIndexes().getColumnIndex(), -1); //////////////////////////////////////// - // taggedValueBuffer.getBaseObject().copy(result.getValue()); taggedValue.setBaseObject(result.getValue()); //////////////////////////////////////// taggedValue.setTag(output); out.collect(triplebuffer, taggedValue); - //System.out.println("output to reducer: "+triplebuffer+"\n"+taggedValue); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java index 1572719..2c25489 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MMRJMRReducer.java @@ -56,45 +56,33 @@ implements Reducer<TripleIndexes, TaggedMatrixValue, MatrixIndexes, MatrixValue> OutputCollector<MatrixIndexes, MatrixValue> out, Reporter report) throws IOException { long start=System.currentTimeMillis(); - // System.out.println("~~~~~ group: "+triple); commonSetup(report); //output previous results if needed if(prevIndexes.getFirstIndex()!=triple.getFirstIndex() || prevIndexes.getSecondIndex()!=triple.getSecondIndex()) { - // System.out.println("cacheValues before processReducerInstructions: \n"+cachedValues); //perform mixed operations processReducerInstructions(); - - // System.out.println("cacheValues before output: \n"+cachedValues); //output results outputResultsFromCachedValues(report); cachedValues.reset(); }else { //clear the buffer - for(AggregateBinaryInstruction aggBinInstruction: aggBinInstructions) - { -// System.out.println("cacheValues before remore: \n"+cachedValues); + for(AggregateBinaryInstruction aggBinInstruction: aggBinInstructions) { cachedValues.remove(aggBinInstruction.input1); - // System.out.println("cacheValues after remore: "+aggBinInstruction.input1+"\n"+cachedValues); cachedValues.remove(aggBinInstruction.input2); - // System.out.println("cacheValues after remore: "+aggBinInstruction.input2+"\n"+cachedValues); } } //perform aggregation first aggIndexes.setIndexes(triple.getFirstIndex(), triple.getSecondIndex()); processAggregateInstructions(aggIndexes, values); - - // System.out.println("cacheValues after aggregation: \n"+cachedValues); - + //perform aggbinary for this group for(AggregateBinaryInstruction aggBinInstruction: aggBinInstructions) processAggBinaryPerGroup(aggIndexes, aggBinInstruction); - - // System.out.println("cacheValues after aggbinary: \n"+cachedValues); prevIndexes.setIndexes(triple); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java index 11ff6e7..f128ea7 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/MapperBase.java @@ -95,8 +95,6 @@ public abstract class MapperBase extends MRBaseForCommonInstructions { long start=System.currentTimeMillis(); - //System.out.println("read in Mapper: "+rawKey+": "+rawValue); - //for each representative matrix, read the record and apply instructions for(int i=0; i<representativeMatrixes.size(); i++) { @@ -216,22 +214,17 @@ public abstract class MapperBase extends MRBaseForCommonInstructions //get the dimension of all the representative matrices rlens=new long[representativeMatrixes.size()]; clens=new long[representativeMatrixes.size()]; - for(int i=0; i<representativeMatrixes.size(); i++) - { + for(int i=0; i<representativeMatrixes.size(); i++) { rlens[i]=MRJobConfiguration.getNumRows(job, representativeMatrixes.get(i)); clens[i]=MRJobConfiguration.getNumColumns(job, representativeMatrixes.get(i)); - // System.out.println("get dimension for "+representativeMatrixes.get(i)+": "+rlens[i]+", "+clens[i]); } //get the block sizes of the representative matrices brlens=new int[representativeMatrixes.size()]; bclens=new int[representativeMatrixes.size()]; - - for(int i=0; i<representativeMatrixes.size(); i++) - { + for(int i=0; i<representativeMatrixes.size(); i++) { brlens[i]=MRJobConfiguration.getNumRowsPerBlock(job, representativeMatrixes.get(i)); bclens[i]=MRJobConfiguration.getNumColumnsPerBlock(job, representativeMatrixes.get(i)); - // System.out.println("get blocksize for "+representativeMatrixes.get(i)+": "+brlens[i]+", "+bclens[i]); } rbounds=new long[representativeMatrixes.size()]; @@ -242,10 +235,9 @@ public abstract class MapperBase extends MRBaseForCommonInstructions //calculate upper boundaries for key value pairs if(valueClass.equals(MatrixBlock.class)) { - for(int i=0; i<representativeMatrixes.size(); i++) - { - rbounds[i]=(long)Math.ceil((double)rlens[i]/(double)brlens[i]); - cbounds[i]=(long)Math.ceil((double)clens[i]/(double)bclens[i]); + for(int i=0; i<representativeMatrixes.size(); i++) { + rbounds[i]=(long)Math.max(Math.ceil((double)rlens[i]/brlens[i]),1); + cbounds[i]=(long)Math.max(Math.ceil((double)clens[i]/bclens[i]),1); lastblockrlens[i]=(int) (rlens[i]%brlens[i]); lastblockclens[i]=(int) (clens[i]%bclens[i]); @@ -253,24 +245,17 @@ public abstract class MapperBase extends MRBaseForCommonInstructions lastblockrlens[i]=brlens[i]; if(lastblockclens[i]==0) lastblockclens[i]=bclens[i]; - - /* - * what is this for???? - // DRB: the row indexes need to be fixed - rbounds[i] = rlens[i];*/ } - }else - { - for(int i=0; i<representativeMatrixes.size(); i++) - { + } + else { + for(int i=0; i<representativeMatrixes.size(); i++) { rbounds[i]=rlens[i]; cbounds[i]=clens[i]; lastblockrlens[i]=1; lastblockclens[i]=1; - // System.out.println("get bound for "+representativeMatrixes.get(i)+": "+rbounds[i]+", "+cbounds[i]); } } - + //load data from distributed cache (if required, reuse if jvm_reuse) try { setupDistCacheFiles(job); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java index 2e66437..52ad388 100644 --- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java +++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java @@ -87,15 +87,10 @@ public class MapReduceTool } public static int getUniqueTaskId(JobConf job) { - //TODO: investigate ID pattern, required for parallel jobs - /*String nodePrefix = job.get(MRConfigurationNames.MR_TASK_ATTEMPT_ID); - return IDHandler.extractIntID(nodePrefix);*/ - String nodePrefix = job.get(MRConfigurationNames.MR_TASK_ATTEMPT_ID); int j = nodePrefix.lastIndexOf("_"); int i=nodePrefix.lastIndexOf("_", j-1); nodePrefix = nodePrefix.substring(i+1, j); - // System.out.println("nodePrefix = " + nodePrefix) ; return Integer.valueOf(nodePrefix); } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/main/java/org/apache/sysml/runtime/util/SortUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java index 28db1cd..c3dcbaf 100644 --- a/src/main/java/org/apache/sysml/runtime/util/SortUtils.java +++ b/src/main/java/org/apache/sysml/runtime/util/SortUtils.java @@ -20,9 +20,6 @@ package org.apache.sysml.runtime.util; import java.util.Arrays; -import java.util.Random; - -import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysml.runtime.matrix.data.MatrixBlock; /** @@ -443,29 +440,4 @@ public class SortUtils return x < y ? (y < z ? b : (x < z ? c : a)) : (y > z ? b : (x > z ? c : a)); } - - - public static void main(String[] args) - { - int n = 10000000; - int[] indexes = new int[n]; - double[] values = new double[n]; - Random rand = new Random(); - for( int i=0; i<n; i++ ) - { - indexes[i] = rand.nextInt(); - values[i] = rand.nextDouble(); - } - - System.out.println("Running quicksort test ..."); - Timing time = new Timing(); - - time.start(); - SortUtils.sortByIndex(0, indexes.length, indexes, values); - System.out.println("quicksort n="+n+" in "+time.stop()+"ms."); - - time.start(); - boolean flag = SortUtils.isSorted(0, indexes.length, indexes); - System.out.println("check sorted n="+n+" in "+time.stop()+"ms, "+flag+"."); - } } http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/java/org/apache/sysml/test/integration/functions/misc/ZeroRowsColsMatrixTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ZeroRowsColsMatrixTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ZeroRowsColsMatrixTest.java new file mode 100644 index 0000000..349828a --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ZeroRowsColsMatrixTest.java @@ -0,0 +1,223 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.misc; + +import java.util.HashMap; + +import org.junit.Test; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class ZeroRowsColsMatrixTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "ZeroMatrix_RemoveEmpty"; + private final static String TEST_NAME2 = "ZeroMatrix_Cbind"; + private final static String TEST_NAME3 = "ZeroMatrix_Rbind"; + private final static String TEST_NAME4 = "ZeroMatrix_Aggregates"; + + private final static String TEST_DIR = "functions/misc/"; + private static final String TEST_CLASS_DIR = TEST_DIR + ZeroRowsColsMatrixTest.class.getSimpleName() + "/"; + private final static int dim = 1372; + private final static double eps = 1e-8; + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "R" })); + addTestConfiguration(TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" })); + addTestConfiguration(TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" })); + addTestConfiguration(TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" })); + } + +// @Test +// public void testEmptyMatrixRemoveEmptyNoRewritesCP() { +// runEmptyMatrixTest(TEST_NAME1, false, ExecType.CP); +// } +// +// @Test +// public void testEmptyMatrixRemoveEmptyRewritesCP() { +// runEmptyMatrixTest(TEST_NAME1, true, ExecType.CP); +// } +// +// @Test +// public void testEmptyMatrixRemoveEmptyNoRewritesMR() { +// runEmptyMatrixTest(TEST_NAME1, false, ExecType.MR); +// } +// +// @Test +// public void testEmptyMatrixRemoveEmptyRewritesMR() { +// runEmptyMatrixTest(TEST_NAME1, true, ExecType.MR); +// } +// +// @Test +// public void testEmptyMatrixRemoveEmptyNoRewritesSP() { +// runEmptyMatrixTest(TEST_NAME1, false, ExecType.SPARK); +// } +// +// @Test +// public void testEmptyMatrixRemoveEmptyRewritesSP() { +// runEmptyMatrixTest(TEST_NAME1, true, ExecType.SPARK); +// } + + @Test + public void testEmptyMatrixCbindNoRewritesCP() { + runEmptyMatrixTest(TEST_NAME2, false, ExecType.CP); + } + + @Test + public void testEmptyMatrixCbindRewritesCP() { + runEmptyMatrixTest(TEST_NAME2, true, ExecType.CP); + } + + @Test + public void testEmptyMatrixCbindNoRewritesMR() { + runEmptyMatrixTest(TEST_NAME2, false, ExecType.MR); + } + + @Test + public void testEmptyMatrixCbindRewritesMR() { + runEmptyMatrixTest(TEST_NAME2, true, ExecType.MR); + } + + @Test + public void testEmptyMatrixCbindNoRewritesSP() { + runEmptyMatrixTest(TEST_NAME2, false, ExecType.SPARK); + } + + @Test + public void testEmptyMatrixCbindRewritesSP() { + runEmptyMatrixTest(TEST_NAME2, true, ExecType.SPARK); + } + + @Test + public void testEmptyMatrixRbindNoRewritesCP() { + runEmptyMatrixTest(TEST_NAME3, false, ExecType.CP); + } + + @Test + public void testEmptyMatrixRbindRewritesCP() { + runEmptyMatrixTest(TEST_NAME3, true, ExecType.CP); + } + + @Test + public void testEmptyMatrixRbindNoRewritesMR() { + runEmptyMatrixTest(TEST_NAME3, false, ExecType.MR); + } + + @Test + public void testEmptyMatrixRbindRewritesMR() { + runEmptyMatrixTest(TEST_NAME3, true, ExecType.MR); + } + + @Test + public void testEmptyMatrixRbindNoRewritesSP() { + runEmptyMatrixTest(TEST_NAME3, false, ExecType.SPARK); + } + + @Test + public void testEmptyMatrixRbindRewritesSP() { + runEmptyMatrixTest(TEST_NAME3, true, ExecType.SPARK); + } + +// @Test +// public void testEmptyMatrixAggregatesNoRewritesCP() { +// runEmptyMatrixTest(TEST_NAME4, false, ExecType.CP); +// } +// +// @Test +// public void testEmptyMatrixAggregatesRewritesCP() { +// runEmptyMatrixTest(TEST_NAME4, true, ExecType.CP); +// } +// +// @Test +// public void testEmptyMatrixAggregatesNoRewritesMR() { +// runEmptyMatrixTest(TEST_NAME4, false, ExecType.MR); +// } +// +// @Test +// public void testEmptyMatrixAggregatesRewritesMR() { +// runEmptyMatrixTest(TEST_NAME4, true, ExecType.MR); +// } +// +// @Test +// public void testEmptyMatrixAggregatesNoRewritesSP() { +// runEmptyMatrixTest(TEST_NAME4, false, ExecType.SPARK); +// } +// +// @Test +// public void testEmptyMatrixAggregatesRewritesSP() { +// runEmptyMatrixTest(TEST_NAME4, true, ExecType.SPARK); +// } + + private void runEmptyMatrixTest( String testname, boolean rewrites, ExecType et ) + { + RUNTIME_PLATFORM platformOld = rtplatform; + switch( et ){ + case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; + case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; + default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break; + } + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + boolean rewritesOld = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites; + + try { + String TEST_NAME = testname; + TestConfiguration config = getTestConfiguration(TEST_NAME); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{"-args", String.valueOf(dim), output("R")}; + + fullRScriptName = HOME + TEST_NAME +".R"; + rCmd = getRCmd(String.valueOf(dim), expectedDir()); + + //run Tests + runTest(true, false, null, -1); + runRScript(true); + + //compare matrices + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R"); + HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R"); + TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); + + //check meta data + if( !testname.equals(TEST_NAME4) ) + checkDMLMetaDataFile("R", new MatrixCharacteristics(dim, 3, 1000, 1000)); + } + finally { + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewritesOld; + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.R b/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.R new file mode 100644 index 0000000..a7f00ba --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.R @@ -0,0 +1,34 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +n = as.integer(args[1]); +X = matrix(0, n, 0); +R = rbind(rbind( + as.matrix(sum(X)==0), + as.matrix(min(X)==(1.0/0.0))), + as.matrix(max(X)==(-1.0/0.0))); + +writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.dml b/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.dml new file mode 100644 index 0000000..690c7b6 --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_Aggregates.dml @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +X = matrix(0, $1, 0); +print(min(X)) +R = rbind( + as.matrix(sum(X)==0), + as.matrix(min(X)==(1.0/0.0)), + as.matrix(max(X)==(-1.0/0.0)) +); + +write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_Cbind.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_Cbind.R b/src/test/scripts/functions/misc/ZeroMatrix_Cbind.R new file mode 100644 index 0000000..fb520fd --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_Cbind.R @@ -0,0 +1,32 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +n = as.integer(args[1]); +R = matrix(0, n, 0); +for( i in 1:3 ) + R = cbind(R, matrix(7, n, 1)); + +writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_Cbind.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_Cbind.dml b/src/test/scripts/functions/misc/ZeroMatrix_Cbind.dml new file mode 100644 index 0000000..7544ac2 --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_Cbind.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +R = matrix(0, $1, 0); +print(nrow(R) + " " + ncol(R) + " " + length(R)); +for( i in 1:3 ) { + R = cbind(R, matrix(7, $1, 1)); + print(nrow(R) + " " + ncol(R) + " " + length(R)); +} +write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_Rbind.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_Rbind.R b/src/test/scripts/functions/misc/ZeroMatrix_Rbind.R new file mode 100644 index 0000000..656c654 --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_Rbind.R @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +n = as.integer(args[1]); +R = matrix(0, 0, n); +for( i in 1:3 ) + R = rbind(R, matrix(7, 1, n)); +R = t(R); + +writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_Rbind.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_Rbind.dml b/src/test/scripts/functions/misc/ZeroMatrix_Rbind.dml new file mode 100644 index 0000000..7400c9a --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_Rbind.dml @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +R = matrix(0, 0, $1); +print(nrow(R) + " " + ncol(R) + " " + length(R)); +for( i in 1:3 ) { + R = rbind(R, matrix(7, 1, $1)); + print(nrow(R) + " " + ncol(R) + " " + length(R)); +} +R = t(R); + +write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.R b/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.R new file mode 100644 index 0000000..900e15e --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.R @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +R = matrix(7, as.integer(args[1]), 7); + +writeMM(as(R, "CsparseMatrix"), paste(args[2], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.dml b/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.dml new file mode 100644 index 0000000..7ca8634 --- /dev/null +++ b/src/test/scripts/functions/misc/ZeroMatrix_RemoveEmpty.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +A = matrix(0, $1, $1); +B = removeEmpty(target=A, margin="rows", outEmpty=FALSE) + 7; +C = removeEmpty(target=A, margin="cols", outEmpty=FALSE) + 3; +R = matrix(7+sum(B)+sum(C), $1, 7); + +write(R, $2); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/92ee2cbf/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java index 8805500..9fbb2a3 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java @@ -77,6 +77,7 @@ import org.junit.runners.Suite; ValueTypeAutoCastingTest.class, ValueTypeCastingTest.class, ValueTypeMatrixScalarBuiltinTest.class, + ZeroRowsColsMatrixTest.class, })
