http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java index 4b0502d..58d008a 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java +++ b/src/main/java/org/apache/sysml/runtime/compress/CompressedMatrixBlock.java @@ -645,8 +645,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable Timing time = LOG.isDebugEnabled() ? new Timing(true) : null; - MatrixBlock ret = new MatrixBlock(rlen, clen, sparse, nonZeros); - ret.allocateDenseOrSparseBlock(); + MatrixBlock ret = new MatrixBlock(rlen, clen, sparse, nonZeros).allocateBlock(); //multi-threaded decompression try { @@ -2322,7 +2321,7 @@ public class CompressedMatrixBlock extends MatrixBlock implements Externalizable } } - private HashSet<Integer> seq(int from, int to, int incr) { + private static HashSet<Integer> seq(int from, int to, int incr) { HashSet<Integer> ret = new HashSet<>(); for (int i = from; i <= to; i+=incr) ret.add(i);
http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/compress/cocode/ColumnGroupPartitionerBinPacking.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/cocode/ColumnGroupPartitionerBinPacking.java b/src/main/java/org/apache/sysml/runtime/compress/cocode/ColumnGroupPartitionerBinPacking.java index 2bd6ce4..908da20 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/cocode/ColumnGroupPartitionerBinPacking.java +++ b/src/main/java/org/apache/sysml/runtime/compress/cocode/ColumnGroupPartitionerBinPacking.java @@ -71,7 +71,7 @@ public class ColumnGroupPartitionerBinPacking extends ColumnGroupPartitioner * @param itemWeights the weights of the items * @return */ - private List<List<Integer>> packFirstFit(int[] items, double[] itemWeights) + private static List<List<Integer>> packFirstFit(int[] items, double[] itemWeights) { List<List<Integer>> bins = new ArrayList<>(); List<Double> binWeights = new ArrayList<>(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java index 7d6c1e7..e345472 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java @@ -1234,7 +1234,7 @@ public class ParForProgramBlock extends ForProgramBlock * @param in array of input matrix objects * @throws DMLRuntimeException if DMLRuntimeException occurs */ - private void cleanWorkerResultVariables(ExecutionContext ec, MatrixObject out, MatrixObject[] in) + private static void cleanWorkerResultVariables(ExecutionContext ec, MatrixObject out, MatrixObject[] in) throws DMLRuntimeException { for( MatrixObject tmp : in ) { @@ -1255,7 +1255,7 @@ public class ParForProgramBlock extends ForProgramBlock * @param sb statement block * @throws DMLRuntimeException if DMLRuntimeException occurs */ - private void createEmptyUnscopedVariables( LocalVariableMap out, StatementBlock sb ) + private static void createEmptyUnscopedVariables( LocalVariableMap out, StatementBlock sb ) throws DMLRuntimeException { VariableSet updated = sb.variablesUpdated(); @@ -1580,7 +1580,7 @@ public class ParForProgramBlock extends ForProgramBlock _childBlocks, tid, new HashSet<String>(), null); } - private String writeTasksToFile(String fname, List<Task> tasks, int maxDigits) + private static String writeTasksToFile(String fname, List<Task> tasks, int maxDigits) throws DMLRuntimeException, IOException { BufferedWriter br = null; @@ -1607,7 +1607,7 @@ public class ParForProgramBlock extends ForProgramBlock return fname; } - private String writeTasksToFile(String fname, LocalTaskQueue<Task> queue, int maxDigits) + private static String writeTasksToFile(String fname, LocalTaskQueue<Task> queue, int maxDigits) throws DMLRuntimeException, IOException { BufferedWriter br = null; @@ -1635,7 +1635,7 @@ public class ParForProgramBlock extends ForProgramBlock return fname; } - private String createTaskFileLine( Task t, int maxDigits, boolean flagFirst ) { + private static String createTaskFileLine( Task t, int maxDigits, boolean flagFirst ) { //always pad to max digits in order to preserve task order return t.toCompactString(maxDigits) + (flagFirst?" ":"") + "\n"; } @@ -1776,7 +1776,7 @@ public class ParForProgramBlock extends ForProgramBlock } } - private long computeNumIterations( IntObject from, IntObject to, IntObject incr ) { + private static long computeNumIterations( IntObject from, IntObject to, IntObject incr ) { return (long)Math.ceil(((double)(to.getLongValue() - from.getLongValue() + 1)) / incr.getLongValue()); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java index c721ab5..d1fc759 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java @@ -348,7 +348,7 @@ public class ProgramBlock implements ParseInfo } } - private void checkSparsity( Instruction lastInst, LocalVariableMap vars ) + private static void checkSparsity( Instruction lastInst, LocalVariableMap vars ) throws DMLRuntimeException { for( String varname : vars.keySet() ) http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java index b8adb5b..f4c024d 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java @@ -299,7 +299,7 @@ public class ExecutionContext { * @return valid d1 or d2 * @throws DMLRuntimeException if error occurs */ - private long validateDimensions(long d1, long d2) throws DMLRuntimeException { + private static long validateDimensions(long d1, long d2) throws DMLRuntimeException { if(d1 >= 0 && d2 >= 0 && d1 != d2) { throw new DMLRuntimeException("Incorrect dimensions:" + d1 + " != " + d2); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index 110c203..472b735 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -1534,7 +1534,7 @@ public class SparkExecutionContext extends ExecutionContext * * @return spark version string */ - private String getSparkVersionString() { + private static String getSparkVersionString() { //check for existing spark context if( isSparkContextCreated() ) return getSparkContextStatic().version(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java index fe9f75a..c54e5e9 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerLocal.java @@ -702,15 +702,13 @@ public class DataPartitionerLocal extends DataPartitioner // read/write // ///////////////////////////////// - private long getKeyFromFilePath( String dir ) - { + private static long getKeyFromFilePath( String dir ) { String[] dirparts = dir.split("/"); long key = Long.parseLong( dirparts[dirparts.length-1] ); return key; } - private long getKey2FromFileName( String fname ) - { + private static long getKey2FromFileName( String fname ) { return Long.parseLong( fname.split("_")[1] ); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java index 6d0cd3a..4001d74 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java @@ -1887,7 +1887,7 @@ public class ProgramConverter return token; } - private int determineNextSameLevelIndexOf( String data, String pattern ) + private static int determineNextSameLevelIndexOf( String data, String pattern ) { String tmpdata = data; int index = 0; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java index ade08e6..02eb309 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java @@ -132,7 +132,7 @@ public class RemoteParForColocatedFileSplit extends FileSplit return getTopHosts(hosts); } - private void countHosts( HashMap<String,Integer> hosts, String[] names ) + private static void countHosts( HashMap<String,Integer> hosts, String[] names ) { for( String name : names ) { @@ -144,7 +144,7 @@ public class RemoteParForColocatedFileSplit extends FileSplit } } - private String[] getTopHosts( HashMap<String,Integer> hosts ) + private static String[] getTopHosts( HashMap<String,Integer> hosts ) { int max = Integer.MIN_VALUE; HashSet<String> maxName = new HashSet<String>(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java index 50eb960..cd0b3b7 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalFile.java @@ -205,7 +205,7 @@ public class ResultMergeLocalFile extends ResultMerge } } - private void mergeTextCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO ) + private static void mergeTextCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO ) throws DMLRuntimeException { try @@ -307,7 +307,7 @@ public class ResultMergeLocalFile extends ResultMerge } @SuppressWarnings("deprecation") - private void mergeBinaryCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO ) + private static void mergeBinaryCellWithoutComp( String fnameNew, MatrixObject outMo, ArrayList<MatrixObject> inMO ) throws DMLRuntimeException { try @@ -498,7 +498,7 @@ public class ResultMergeLocalFile extends ResultMerge } } - private void createTextCellStagingFile( String fnameStaging, MatrixObject mo, long ID ) + private static void createTextCellStagingFile( String fnameStaging, MatrixObject mo, long ID ) throws IOException, DMLRuntimeException { JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); @@ -560,7 +560,7 @@ public class ResultMergeLocalFile extends ResultMerge } @SuppressWarnings("deprecation") - private void createBinaryCellStagingFile( String fnameStaging, MatrixObject mo, long ID ) + private static void createBinaryCellStagingFile( String fnameStaging, MatrixObject mo, long ID ) throws IOException, DMLRuntimeException { JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); @@ -605,7 +605,7 @@ public class ResultMergeLocalFile extends ResultMerge } } - private void appendCellBufferToStagingArea( String fnameStaging, long ID, LinkedList<Cell> buffer, int brlen, int bclen ) + private static void appendCellBufferToStagingArea( String fnameStaging, long ID, LinkedList<Cell> buffer, int brlen, int bclen ) throws DMLRuntimeException, IOException { HashMap<Long,HashMap<Long,LinkedList<Cell>>> sortedBuffer = new HashMap<>(); @@ -1004,7 +1004,7 @@ public class ResultMergeLocalFile extends ResultMerge } } - private void copyAllFiles( String fnameNew, ArrayList<MatrixObject> inMO ) + private static void copyAllFiles( String fnameNew, ArrayList<MatrixObject> inMO ) throws CacheException, IOException { JobConf job = new JobConf(ConfigurationManager.getCachedJobConf()); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java index fdd3d97..1e7efb6 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java @@ -67,10 +67,9 @@ public class ResultMergeLocalMemory extends ResultMerge //create output matrices in correct format according to //the estimated number of non-zeros long estnnz = getOutputNnzEstimate(); - MatrixBlock outMBNew = new MatrixBlock( - outMB.getNumRows(), outMB.getNumColumns(), estnnz); + MatrixBlock outMBNew = new MatrixBlock(outMB.getNumRows(), + outMB.getNumColumns(), estnnz).allocateBlock(); boolean appendOnly = outMBNew.isInSparseFormat(); - outMBNew.allocateDenseOrSparseBlock(); //create compare matrix if required (existing data in result) _compare = createCompareMatrix(outMB); @@ -218,16 +217,11 @@ public class ResultMergeLocalMemory extends ResultMerge return moNew; } - private double[][] createCompareMatrix( MatrixBlock output ) - { - double[][] ret = null; - + private static double[][] createCompareMatrix( MatrixBlock output ) { //create compare matrix only if required - if( output.getNonZeros() > 0 ) { - ret = DataConverter.convertToDoubleMatrix( output ); - } - - return ret; + if( output.getNonZeros() > 0 ) + return DataConverter.convertToDoubleMatrix( output ); + return null; } private MatrixObject createNewMatrixObject( MatrixBlock data ) http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java index 44b7dc2..6ff8d65 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java @@ -205,7 +205,7 @@ public class ResultMergeRemoteSpark extends ResultMerge return ret; } - private int determineNumReducers(long rlen, long clen, int brlen, int bclen, long numRed) { + private static int determineNumReducers(long rlen, long clen, int brlen, int bclen, long numRed) { //set the number of mappers and reducers long reducerGroups = Math.max(rlen/brlen,1) * Math.max(clen/bclen, 1); int ret = (int)Math.min( numRed, reducerGroups ); @@ -214,7 +214,7 @@ public class ResultMergeRemoteSpark extends ResultMerge } @SuppressWarnings("unchecked") - private void setRDDHandleForMerge(MatrixObject mo, SparkExecutionContext sec) { + private static void setRDDHandleForMerge(MatrixObject mo, SparkExecutionContext sec) { InputInfo iinfo = InputInfo.BinaryBlockInputInfo; JavaSparkContext sc = sec.getSparkContext(); JavaPairRDD<MatrixIndexes,MatrixBlock> rdd = (JavaPairRDD<MatrixIndexes,MatrixBlock>) http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java index d571a71..b6be028 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerConstrained.java @@ -414,8 +414,7 @@ public class OptimizerConstrained extends OptimizerRuleBased super.rewriteSetFusedDataPartitioningExecution(pn, M, flagLIX, partitionedMatrices, vars); } - private PExecMode getPExecMode( OptNode pn ) - { + private static PExecMode getPExecMode( OptNode pn ) { ParForProgramBlock pfpb = (ParForProgramBlock) OptTreeConverter .getAbstractPlanMapping().getMappedProg(pn.getID())[1]; return pfpb.getExecMode(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java index e035f1b..16a2e7e 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java @@ -729,24 +729,24 @@ public class OptimizerRuleBased extends Optimizer return ret; } - private double estimateSizeSparseRowBlock( long rows ) { + private static double estimateSizeSparseRowBlock( long rows ) { //see MatrixBlock.estimateSizeSparseInMemory return 44 + rows * 8; } - private double estimateSizeSparseRow( long cols, long nnz ) { + private static double estimateSizeSparseRow( long cols, long nnz ) { //see MatrixBlock.estimateSizeSparseInMemory long cnnz = Math.max(SparseRowVector.initialCapacity, Math.max(cols, nnz)); return ( 116 + 12 * cnnz ); //sparse row } - private double estimateSizeSparseRowMin( long cols ) { + private static double estimateSizeSparseRowMin( long cols ) { //see MatrixBlock.estimateSizeSparseInMemory long cnnz = Math.min(SparseRowVector.initialCapacity, cols); return ( 116 + 12 * cnnz ); //sparse row } - private int estimateNumTasksSparseCol( double budget, long rows ) { + private static int estimateNumTasksSparseCol( double budget, long rows ) { //see MatrixBlock.estimateSizeSparseInMemory double lbudget = budget - rows * 116; return (int) Math.floor( lbudget / 12 ); @@ -1740,7 +1740,7 @@ public class OptimizerRuleBased extends Optimizer return ret; } - private double computeTotalSizeResultVariables(ArrayList<String> retVars, LocalVariableMap vars, int k) { + private static double computeTotalSizeResultVariables(ArrayList<String> retVars, LocalVariableMap vars, int k) { double sum = 1; for( String var : retVars ) { Data dat = vars.get(var); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java b/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java index e283667..5599dd5 100644 --- a/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java +++ b/src/main/java/org/apache/sysml/runtime/functionobjects/IntegerDivide.java @@ -59,7 +59,7 @@ public class IntegerDivide extends ValueFunction * @param in2 double input 2 * @return result */ - private double executeIntDiv( double in1, double in2 ) + private static double executeIntDiv( double in1, double in2 ) { //compute normal double devision double ret = in1 / in2; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java b/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java index f36b484..9a0c6c2 100644 --- a/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java +++ b/src/main/java/org/apache/sysml/runtime/functionobjects/ParameterizedBuiltin.java @@ -209,7 +209,7 @@ public class ParameterizedBuiltin extends ValueFunction * @throws MathArithmeticException if MathArithmeticException occurs * @throws DMLRuntimeException if DMLRuntimeException occurs */ - private double computeFromDistribution (ProbabilityDistributionCode dcode, HashMap<String,String> params, boolean inverse ) throws MathArithmeticException, DMLRuntimeException { + private static double computeFromDistribution (ProbabilityDistributionCode dcode, HashMap<String,String> params, boolean inverse ) throws MathArithmeticException, DMLRuntimeException { // given value is "quantile" when inverse=false, and it is "probability" when inverse=true double val = Double.parseDouble(params.get("target")); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java index 77ef635..d27c9ff 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/MRJobInstruction.java @@ -690,7 +690,7 @@ public class MRJobInstruction extends Instruction LOG.debug("\nMRInstructions: \n" + this.toString()); } - private String getOps(String inst) { + private static String getOps(String inst) { StringBuilder sb = new StringBuilder(); for ( String i : inst.split(Lop.INSTRUCTION_DELIMITOR)) { sb.append(","); @@ -1216,7 +1216,7 @@ public class MRJobInstruction extends Instruction * @param transMap transition map * @return result string */ - private String replaceInstructionStringWithTransMap( String inst, HashMap<Byte,Byte> transMap ) + private static String replaceInstructionStringWithTransMap( String inst, HashMap<Byte,Byte> transMap ) { //prevent unnecessary parsing and reconstruction if( inst == null || inst.isEmpty() || transMap.isEmpty() ) @@ -1253,7 +1253,7 @@ public class MRJobInstruction extends Instruction return instOut.toString(); } - private String concatenateInstructions(String inst1, String inst2) + private static String concatenateInstructions(String inst1, String inst2) { boolean emptyInst1 = (inst1 == null || inst1.length()==0); boolean emptyInst2 = (inst2 == null || inst2.length()==0); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java index 5545ea2..f6ff998 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/ConvolutionCPInstruction.java @@ -218,11 +218,10 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { } } - private int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, int index) + private static int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, int index) throws DMLRuntimeException { return (int) ec.getScalarInput(aL.get(index).getName(), - aL.get(index).getValueType(), aL.get(index).isLiteral()) - .getLongValue(); + aL.get(index).getValueType(), aL.get(index).isLiteral()).getLongValue(); } public void processReluBackwardInstruction(ExecutionContext ec) throws DMLRuntimeException { @@ -288,8 +287,8 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { } else { // As we always fill the output first with bias - outputBlock = new MatrixBlock(input.getNumRows(), input.getNumColumns(), input.isInSparseFormat()); - outputBlock.allocateDenseOrSparseBlock(); + outputBlock = new MatrixBlock(input.getNumRows(), input.getNumColumns(), + input.isInSparseFormat()).allocateBlock(); LibMatrixDNN.biasMultiply(input, bias, outputBlock, _numThreads); } @@ -301,7 +300,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { // Assumption: enableNative && NativeHelper.isNativeLibraryLoaded() is true // This increases the number of native calls. For example:the cases where filter is sparse but input is dense - private boolean isFilterSparse(MatrixBlock filter) throws DMLRuntimeException { + private static boolean isFilterSparse(MatrixBlock filter) throws DMLRuntimeException { long numElems = filter.getNumRows()*filter.getNumColumns(); // if filter is less than 10 MB in dense format (which handles almost all the cases). // In fact, using threshold of 1 MB is still sufficient for common CNNs. @@ -354,7 +353,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { outputBlock = new MatrixBlock(N, C*P*Q, true); } else { - outputBlock = getDenseOutputBlock(N, C*P*Q); + outputBlock = new MatrixBlock(N, C*P*Q, false).allocateBlock(); if(instOpcode.equalsIgnoreCase("maxpooling")) Arrays.fill(outputBlock.getDenseBlock(), -Double.MAX_VALUE); LibMatrixDNN.maxpooling(matBlock, outputBlock, params); @@ -366,7 +365,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { outputBlock = new MatrixBlock(N, C*H*W, true); } else { - outputBlock = getDenseOutputBlock(N, C*H*W); + outputBlock = new MatrixBlock(N, C*H*W, false).allocateBlock(); if(instOpcode.equalsIgnoreCase("maxpooling_backward")) LibMatrixDNN.maxpoolingBackward(matBlock, dout, outputBlock, params, false); else @@ -381,7 +380,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { outputBlock = new MatrixBlock(N, K*P*Q, true); } else { - outputBlock = getDenseOutputBlock(N, K*P*Q); + outputBlock = new MatrixBlock(N, K*P*Q, false).allocateBlock(); if(params.enableNative && !isFilterSparse(filter) && !matBlock.isInSparseFormat()) LibMatrixNative.conv2d(matBlock, filter, outputBlock, params); else @@ -405,12 +404,12 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { else if(isOutputConvEmpty && !bias.isEmpty()) { // Add bias to empty output block // bias_add(empty mb, bias) - outputBlock = getDenseOutputBlock(N, K*P*Q); + outputBlock = new MatrixBlock(N, K*P*Q, false).allocateBlock(); for(int n = 0; n < params.N; n++) ConvolutionUtils.fillBias(bias, outputBlock.getDenseBlock(), n, n+1, params.N, params.K, params.P*params.Q); } else { - outputBlock = getDenseOutputBlock(N, K*P*Q); + outputBlock = new MatrixBlock(N, K*P*Q, false).allocateBlock(); if(!bias.isEmpty()) { // Handle situation where both input and filter are non empty, but bias is empty params.bias = bias; @@ -429,7 +428,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { outputBlock = new MatrixBlock(K, C*R*S, true); } else { - outputBlock = getDenseOutputBlock(K, C*R*S); + outputBlock = new MatrixBlock(K, C*R*S, false).allocateBlock(); if(params.enableNative && !matBlock.isInSparseFormat() && !dout.isInSparseFormat()) LibMatrixNative.conv2dBackwardFilter(matBlock, dout, outputBlock, params); else @@ -443,7 +442,7 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { outputBlock = new MatrixBlock(N, C * H * W, true); } else { - outputBlock = getDenseOutputBlock(N, C * H * W); + outputBlock = new MatrixBlock(N, C * H * W, false).allocateBlock(); if(params.enableNative && !isFilterSparse(matBlock) && !dout.isInSparseFormat()) LibMatrixNative.conv2dBackwardData(matBlock, dout, outputBlock, params); else @@ -480,10 +479,4 @@ public class ConvolutionCPInstruction extends UnaryCPInstruction { } } } - - private MatrixBlock getDenseOutputBlock(int numRows, int numCols) throws DMLRuntimeException { - MatrixBlock outputBlock = new MatrixBlock(numRows, numCols, false); - outputBlock.allocateDenseBlock(); - return outputBlock; - } } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java index 799fdb3..f52b8f4 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/DataPartitionCPInstruction.java @@ -28,7 +28,6 @@ import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PDataPartition import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.instructions.InstructionUtils; -import org.apache.sysml.runtime.io.MatrixWriterFactory; import org.apache.sysml.runtime.io.WriterBinaryBlock; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; @@ -80,10 +79,9 @@ public class DataPartitionCPInstruction extends UnaryCPInstruction { try { //write matrix partitions to hdfs - WriterBinaryBlock writer = (WriterBinaryBlock) MatrixWriterFactory.createMatrixWriter(OutputInfo.BinaryBlockOutputInfo); - writer.writePartitionedBinaryBlockMatrixToHDFS( - new Path(fname), new JobConf(ConfigurationManager.getCachedJobConf()), mb, moIn.getNumRows(), moIn.getNumColumns(), - (int)moIn.getNumRowsPerBlock(), (int)moIn.getNumColumnsPerBlock(), _pformat); + WriterBinaryBlock.writePartitionedBinaryBlockMatrixToHDFS(new Path(fname), + new JobConf(ConfigurationManager.getCachedJobConf()), mb, moIn.getNumRows(), moIn.getNumColumns(), + (int)moIn.getNumRowsPerBlock(), (int)moIn.getNumColumnsPerBlock(), _pformat); //ensure correctness of output characteristics (required if input unknown during compile and no recompile) MatrixCharacteristics mc = new MatrixCharacteristics(moIn.getNumRows(), moIn.getNumColumns(), (int)moIn.getNumRowsPerBlock(), (int)moIn.getNumColumnsPerBlock(), moIn.getNnz()); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index 9525d2a..1c0f9e6 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -905,8 +905,8 @@ public class VariableCPInstruction extends CPInstruction { MatrixCharacteristics mc = mo.getMatrixCharacteristics(); if(oi == OutputInfo.TextCellOutputInfo) { try { - WriterMatrixMarket writer = new WriterMatrixMarket(); - writer.mergeTextcellToMatrixMarket(mo.getFileName(), fname, mc.getRows(), mc.getCols(), mc.getNonZeros()); + WriterMatrixMarket.mergeTextcellToMatrixMarket(mo.getFileName(), + fname, mc.getRows(), mc.getCols(), mc.getNonZeros()); } catch (IOException e) { throw new DMLRuntimeException(e); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java index 64a2b5f..a85b7ad 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cpfile/ParameterizedBuiltinCPFileInstruction.java @@ -222,7 +222,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC return createNewOutputObject(_src, _out, mc.getRows(), ret ); } - private MatrixObject createNewOutputObject( MatrixObject src, MatrixObject out, long rows, long cols ) + private static MatrixObject createNewOutputObject( MatrixObject src, MatrixObject out, long rows, long cols ) throws DMLRuntimeException { String varName = out.getVarName(); @@ -409,7 +409,7 @@ public class ParameterizedBuiltinCPFileInstruction extends ParameterizedBuiltinC return diagBlocks; } - private void appendCellBufferToStagingArea( String dir, LinkedList<Cell> buffer, int brlen, int bclen ) + private static void appendCellBufferToStagingArea( String dir, LinkedList<Cell> buffer, int brlen, int bclen ) throws DMLRuntimeException, IOException { HashMap<String,LinkedList<Cell>> sortedBuffer = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/gpu/AggregateBinaryGPUInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/AggregateBinaryGPUInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/AggregateBinaryGPUInstruction.java index 49915f2..dfc000f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/AggregateBinaryGPUInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/AggregateBinaryGPUInstruction.java @@ -95,7 +95,7 @@ public class AggregateBinaryGPUInstruction extends GPUInstruction { ec.setMetaData(_output.getName(), rlen, clen); LibMatrixCUDA.matmult(ec, ec.getGPUContext(0), getExtendedOpcode(), m1, m2, _output.getName(), _isLeftTransposed, _isRightTransposed); - + //release inputs/outputs ec.releaseMatrixInputForGPUInstruction(_input1.getName()); ec.releaseMatrixInputForGPUInstruction(_input2.getName()); @@ -103,13 +103,13 @@ public class AggregateBinaryGPUInstruction extends GPUInstruction { } @SuppressWarnings("unused") - private MatrixBlock transpose(MatrixBlock m1) throws DMLRuntimeException { + private static MatrixBlock transpose(MatrixBlock m1) throws DMLRuntimeException { ReorgOperator r_op = new ReorgOperator(SwapIndex.getSwapIndexFnObject(), 1); return (MatrixBlock) (m1.reorgOperations(r_op, new MatrixBlock(), 0, 0, 0)); } @SuppressWarnings("unused") - private boolean isSparse(ExecutionContext ec, String var) throws DMLRuntimeException { + private static boolean isSparse(ExecutionContext ec, String var) throws DMLRuntimeException { MatrixObject mo = ec.getMatrixObject(var); return LibMatrixCUDA.isInSparseFormat(ec.getGPUContext(0), mo); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/gpu/ConvolutionGPUInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/ConvolutionGPUInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/ConvolutionGPUInstruction.java index d841e69..354ea63 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/ConvolutionGPUInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/ConvolutionGPUInstruction.java @@ -353,11 +353,10 @@ public class ConvolutionGPUInstruction extends GPUInstruction { } - private int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, int index) + private static int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, int index) throws DMLRuntimeException { return (int) ec.getScalarInput(aL.get(index).getName(), - aL.get(index).getValueType(), aL.get(index).isLiteral()) - .getLongValue(); + aL.get(index).getValueType(), aL.get(index).isLiteral()).getLongValue(); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixMatrixAxpyGPUInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixMatrixAxpyGPUInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixMatrixAxpyGPUInstruction.java index fa01ab5..2595795 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixMatrixAxpyGPUInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/MatrixMatrixAxpyGPUInstruction.java @@ -103,11 +103,11 @@ public class MatrixMatrixAxpyGPUInstruction extends ArithmeticBinaryGPUInstructi ec.releaseMatrixOutputForGPUInstruction(_output.getName()); } - private boolean isValidMMOperation(long rlen1, long rlen2, long clen1, long clen2) { + private static boolean isValidMMOperation(long rlen1, long rlen2, long clen1, long clen2) { return rlen1 == rlen2 && clen1 == clen2; } - private boolean isValidMVOperation(long rlen1, long rlen2, long clen1, long clen2) { + private static boolean isValidMVOperation(long rlen1, long rlen2, long clen1, long clen2) { return (rlen1 == rlen2 && clen2 == 1) || (rlen2 == 1 && clen1 == clen2); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java index 9cfab2b..3ebed52 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java @@ -119,7 +119,7 @@ public class JCudaKernels { * @return * @throws DMLRuntimeException */ - private Pointer initKernels(String ptxFileName) throws DMLRuntimeException { + private static Pointer initKernels(String ptxFileName) throws DMLRuntimeException { InputStream in = null; ByteArrayOutputStream out = null; try { http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java index 7bb15cb..4ea4f0b 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/ConvolutionSPInstruction.java @@ -213,7 +213,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { } } - private JavaPairRDD<MatrixIndexes,MatrixBlock> reblockAsRectangularMatrices(SparkExecutionContext sec, String name, int numRowsPerBlock) throws DMLRuntimeException { + private static JavaPairRDD<MatrixIndexes,MatrixBlock> reblockAsRectangularMatrices(SparkExecutionContext sec, String name, int numRowsPerBlock) throws DMLRuntimeException { JavaPairRDD<MatrixIndexes,MatrixBlock> in1 = sec.getBinaryBlockRDDHandleForVariable( name ); MatrixCharacteristics mcRdd = sec.getMatrixCharacteristics(name); if(mcRdd.getColsPerBlock() < mcRdd.getCols() || mcRdd.getRowsPerBlock() != 1) { @@ -298,11 +298,10 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { } } - private int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, - int index) throws DMLRuntimeException { + private static int getScalarInput(ExecutionContext ec, ArrayList<CPOperand> aL, int index) + throws DMLRuntimeException { return (int) ec.getScalarInput(aL.get(index).getName(), - aL.get(index).getValueType(), aL.get(index).isLiteral()) - .getLongValue(); + aL.get(index).getValueType(), aL.get(index).isLiteral()).getLongValue(); } private static class RDDConv2dMapMMFunction implements PairFlatMapFunction<Iterator<Tuple2<MatrixIndexes, MatrixBlock>>, MatrixIndexes, MatrixBlock> { @@ -331,7 +330,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { outputBlock = new MatrixBlock(params.N, params.K*params.P*params.Q, true); } else { - outputBlock = getDenseOutputBlock(params.N, params.K*params.P*params.Q); + outputBlock = new MatrixBlock(params.N, params.K*params.P*params.Q, false).allocateDenseBlock(); if(enableNative) LibMatrixNative.conv2d(matBlock, filter, outputBlock, params); else @@ -345,7 +344,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { outputBlock = new MatrixBlock(params.N, params.K*params.P*params.Q, true); } else { - outputBlock = getDenseOutputBlock(params.N, params.K*params.P*params.Q); + outputBlock = new MatrixBlock(params.N, params.K*params.P*params.Q, false).allocateDenseBlock(); if(!bias.isEmptyBlock()) params.bias = bias; if(enableNative) @@ -359,7 +358,7 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { outputBlock = new MatrixBlock(params.N, params.C*params.P*params.Q, true); } else { - outputBlock = getDenseOutputBlock(params.N, params.C*params.P*params.Q); + outputBlock = new MatrixBlock(params.N, params.C*params.P*params.Q, false).allocateBlock(); if(instOpcode.equalsIgnoreCase("maxpooling")) Arrays.fill(outputBlock.getDenseBlock(), -Double.MAX_VALUE); LibMatrixDNN.maxpooling(matBlock, outputBlock, params); @@ -370,12 +369,6 @@ public class ConvolutionSPInstruction extends UnarySPInstruction { } return outputBlock; } - - private MatrixBlock getDenseOutputBlock(int numRows, int numCols) throws DMLRuntimeException { - MatrixBlock outputBlock = new MatrixBlock(numRows, numCols, false); - outputBlock.allocateDenseBlock(); - return outputBlock; - } @Override public Iterator<Tuple2<MatrixIndexes, MatrixBlock>> call( http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java index 33f3e1b..be5d12f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameAppendMSPInstruction.java @@ -75,8 +75,7 @@ public class FrameAppendMSPInstruction extends AppendMSPInstruction { sec.getFrameObject(input2.getName()))); } - private boolean preservesPartitioning( boolean cbind ) - { + private static boolean preservesPartitioning( boolean cbind ) { //Partitions for input1 will be preserved in case of cbind, // where as in case of rbind partitions will not be preserved. return cbind; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java index 71f17a0..ad020bb 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/FrameIndexingSPInstruction.java @@ -164,10 +164,9 @@ public class FrameIndexingSPInstruction extends IndexingSPInstruction { throw new DMLRuntimeException("Invalid opcode (" + opcode +") encountered in FrameIndexingSPInstruction."); } - private boolean isPartitioningPreservingRightIndexing(MatrixCharacteristics mcIn, IndexRange ixrange) - { + private static boolean isPartitioningPreservingRightIndexing(MatrixCharacteristics mcIn, IndexRange ixrange) { return ( mcIn.dimsKnown() && - (ixrange.rowStart==1 && ixrange.rowEnd==mcIn.getRows() )); //Entire Column/s + (ixrange.rowStart==1 && ixrange.rowEnd==mcIn.getRows() )); //Entire Column/s } private static void checkValidOutputDimensions(MatrixCharacteristics mcOut) http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java index e047457..4864d9e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MatrixAppendMSPInstruction.java @@ -82,7 +82,7 @@ public class MatrixAppendMSPInstruction extends AppendMSPInstruction { sec.addLineageBroadcast(output.getName(), input2.getName()); } - private boolean preservesPartitioning( MatrixCharacteristics mcIn1, MatrixCharacteristics mcIn2, boolean cbind ) + private static boolean preservesPartitioning( MatrixCharacteristics mcIn1, MatrixCharacteristics mcIn2, boolean cbind ) { long ncblksIn1 = cbind ? (long)Math.ceil((double)mcIn1.getCols()/mcIn1.getColsPerBlock()) : http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java index 19b0732..6d6478e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/MultiReturnParameterizedBuiltinSPInstruction.java @@ -172,7 +172,7 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI } } - private boolean containsMVImputeEncoder(Encoder encoder) { + private static boolean containsMVImputeEncoder(Encoder encoder) { if( encoder instanceof EncoderComposite ) for( Encoder cencoder : ((EncoderComposite)encoder).getEncoders() ) if( cencoder instanceof EncoderMVImpute ) @@ -180,7 +180,7 @@ public class MultiReturnParameterizedBuiltinSPInstruction extends ComputationSPI return false; } - private EncoderMVImpute getMVImputeEncoder(Encoder encoder) { + private static EncoderMVImpute getMVImputeEncoder(Encoder encoder) { if( encoder instanceof EncoderComposite ) for( Encoder cencoder : ((EncoderComposite)encoder).getEncoders() ) if( cencoder instanceof EncoderMVImpute ) http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java index d7fb6e4..e7f515a 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/QuantilePickSPInstruction.java @@ -157,7 +157,7 @@ public class QuantilePickSPInstruction extends BinarySPInstruction { * @return a summary of weighted quantiles * @throws DMLRuntimeException */ - private double[] getWeightedQuantileSummary(JavaPairRDD<MatrixIndexes,MatrixBlock> w, MatrixCharacteristics mc, Double... quantiles) + private static double[] getWeightedQuantileSummary(JavaPairRDD<MatrixIndexes,MatrixBlock> w, MatrixCharacteristics mc, Double... quantiles) throws DMLRuntimeException { double[] ret = new double[3*quantiles.length + 1]; @@ -217,7 +217,7 @@ public class QuantilePickSPInstruction extends BinarySPInstruction { return ret; } - private double lookupKey(JavaPairRDD<MatrixIndexes,MatrixBlock> in, long key, int brlen) + private static double lookupKey(JavaPairRDD<MatrixIndexes,MatrixBlock> in, long key, int brlen) throws DMLRuntimeException { long rix = UtilFunctions.computeBlockIndex(key, brlen); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java index 416e4a6..b34afad 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java @@ -651,14 +651,14 @@ public class SpoofSPInstruction extends SPInstruction { return ret.iterator(); } - private MatrixIndexes createOutputIndexes(MatrixIndexes in, SpoofOperator spoofOp) { + private static MatrixIndexes createOutputIndexes(MatrixIndexes in, SpoofOperator spoofOp) { if( ((SpoofOuterProduct)spoofOp).getOuterProdType() == OutProdType.LEFT_OUTER_PRODUCT ) return new MatrixIndexes(in.getColumnIndex(), 1); else if ( ((SpoofOuterProduct)spoofOp).getOuterProdType() == OutProdType.RIGHT_OUTER_PRODUCT) return new MatrixIndexes(in.getRowIndex(), 1); else return in; - } + } } public static class ReplicateRightFactorFunction implements PairFlatMapFunction<Tuple2<MatrixIndexes, MatrixBlock>, MatrixIndexes, MatrixBlock> http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java index e0c8d18..b4e48df 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/TernarySPInstruction.java @@ -285,7 +285,7 @@ public class TernarySPInstruction extends ComputationSPInstruction { private static final long serialVersionUID = 271459913267735850L; - private MatrixBlock extractBlock(Iterable<MatrixBlock> blks, MatrixBlock retVal) throws Exception { + private static MatrixBlock extractBlock(Iterable<MatrixBlock> blks, MatrixBlock retVal) throws Exception { for(MatrixBlock blk1 : blks) { if(retVal != null) { throw new Exception("ERROR: More than 1 matrixblock found for one of the inputs at a given index"); @@ -317,7 +317,7 @@ public class TernarySPInstruction extends ComputationSPInstruction { private static final long serialVersionUID = -4873754507037646974L; - private MatrixBlock extractBlock(Iterable<MatrixBlock> blks, MatrixBlock retVal) throws Exception { + private static MatrixBlock extractBlock(Iterable<MatrixBlock> blks, MatrixBlock retVal) throws Exception { for(MatrixBlock blk1 : blks) { if(retVal != null) { throw new Exception("ERROR: More than 1 matrixblock found for one of the inputs at a given index"); @@ -369,7 +369,7 @@ public class TernarySPInstruction extends ComputationSPInstruction { this.ignoreZeros = ignoreZeros; } - private void expectedALSize(int length, ArrayList<MatrixBlock> al) throws Exception { + private static void expectedALSize(int length, ArrayList<MatrixBlock> al) throws Exception { if(al.size() != length) { throw new Exception("Expected arraylist of size:" + length + ", but found " + al.size()); } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java index dd9d939..5147b8f 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/Tsmm2SPInstruction.java @@ -210,8 +210,7 @@ public class Tsmm2SPInstruction extends UnarySPInstruction { MatrixBlock mbin = arg0._2(); boolean fullBlock = _type.isLeft() ? ixin.getColumnIndex() == 1 : ixin.getRowIndex() == 1; - MatrixBlock out = new MatrixBlock(_outputDim, _outputDim, !fullBlock); - out.allocateDenseOrSparseBlock(); + MatrixBlock out = new MatrixBlock(_outputDim, _outputDim, !fullBlock).allocateBlock(); //execute block tsmm operation MatrixBlock out1 = mbin.transposeSelfMatrixMultOperations(new MatrixBlock(), _type); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java index 6578b86..046abf5 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/WriteSPInstruction.java @@ -264,7 +264,7 @@ public class WriteSPInstruction extends SPInstruction { MapReduceTool.writeMetaDataFile(fname + ".mtd", input1.getValueType(), schema, DataType.FRAME, mc, oi, formatProperties); } - private void customSaveTextFile(JavaRDD<String> rdd, String fname, boolean inSingleFile) + private static void customSaveTextFile(JavaRDD<String> rdd, String fname, boolean inSingleFile) throws DMLRuntimeException { if(inSingleFile) { http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java index b736931..4e836eb 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/FrameRDDConverterUtils.java @@ -647,7 +647,7 @@ public class FrameRDDConverterUtils return fb; } - private void flushBlocksToList( Long ix, FrameBlock fb, ArrayList<Tuple2<Long,FrameBlock>> ret ) + private static void flushBlocksToList( Long ix, FrameBlock fb, ArrayList<Tuple2<Long,FrameBlock>> ret ) throws DMLRuntimeException { if( fb != null && fb.getNumRows()>0 ) http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java index c5058a6..8ec02cc 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtils.java @@ -748,12 +748,12 @@ public class RDDConverterUtils int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); ix[cix-1] = new MatrixIndexes(rix, cix); mb[cix-1] = new MatrixBlock(lrlen, lclen, _sparse, (int)(lrlen*lclen*_sparsity)); - mb[cix-1].allocateDenseOrSparseBlock(); + mb[cix-1].allocateBlock(); } } // Flushes current state of filled column blocks to output list. - private void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) + private static void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) throws DMLRuntimeException { int len = ix.length; @@ -865,12 +865,12 @@ public class RDDConverterUtils int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); ix[cix-1] = new MatrixIndexes(rix, cix); mb[cix-1] = new MatrixBlock(lrlen, lclen, lsparse); - mb[cix-1].allocateDenseOrSparseBlock(); + mb[cix-1].allocateBlock(); } } // Flushes current state of filled column blocks to output list. - private void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) + private static void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) throws DMLRuntimeException { int len = ix.length; @@ -1107,12 +1107,12 @@ public class RDDConverterUtils int lclen = (int)UtilFunctions.computeBlockSize(_clen, cix, _bclen); ix[cix-1] = new MatrixIndexes(rix, cix); mb[cix-1] = new MatrixBlock(lrlen, lclen, _sparse,(int)(lrlen*lclen*_sparsity)); - mb[cix-1].allocateDenseOrSparseBlock(); + mb[cix-1].allocateBlock(); } } // Flushes current state of filled column blocks to output list. - private void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) + private static void flushBlocksToList( MatrixIndexes[] ix, MatrixBlock[] mb, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) throws DMLRuntimeException { int len = ix.length; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java index 775d11a..951877e 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/utils/RDDConverterUtilsExt.java @@ -166,7 +166,7 @@ public class RDDConverterUtilsExt public static MatrixBlock allocateDenseOrSparse(int rlen, int clen, boolean isSparse) { MatrixBlock ret = new MatrixBlock(rlen, clen, isSparse); - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); return ret; } public static MatrixBlock allocateDenseOrSparse(long rlen, long clen, boolean isSparse) throws DMLRuntimeException { @@ -392,7 +392,7 @@ public class RDDConverterUtilsExt return ret; } - private void flushBufferToList( ReblockBuffer rbuff, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) + private static void flushBufferToList( ReblockBuffer rbuff, ArrayList<Tuple2<MatrixIndexes,MatrixBlock>> ret ) throws IOException, DMLRuntimeException { //temporary list of indexed matrix values to prevent library dependencies http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java index 7bd2c00..e56b10f 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderBinaryBlock.java @@ -78,7 +78,7 @@ public class FrameReaderBinaryBlock extends FrameReader } @SuppressWarnings({ "deprecation" }) - protected final void readBinaryBlockFrameFromSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock dest ) + protected static void readBinaryBlockFrameFromSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock dest ) throws IOException, DMLRuntimeException { int rlen = dest.getNumRows(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java index 55128f8..f50fba2 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCell.java @@ -102,7 +102,7 @@ public class FrameReaderTextCell extends FrameReader } } - protected final void readTextCellFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest) + protected static void readTextCellFrameFromInputSplit( InputSplit split, TextInputFormat informat, JobConf job, FrameBlock dest) throws IOException { ValueType[] schema = dest.getSchema(); @@ -148,7 +148,7 @@ public class FrameReaderTextCell extends FrameReader } } - protected final void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, + protected static void readRawTextCellFrameFromHDFS( Path path, JobConf job, FileSystem fs, FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen) throws IOException { @@ -159,7 +159,7 @@ public class FrameReaderTextCell extends FrameReader readRawTextCellFrameFromInputStream(inputStream, dest, schema, names, rlen, clen); } - protected final void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen) + protected static void readRawTextCellFrameFromInputStream( InputStream is, FrameBlock dest, ValueType[] schema, String[] names, long rlen, long clen) throws IOException { //create buffered reader http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java index e208bbe..38e1afb 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterBinaryBlock.java @@ -86,7 +86,7 @@ public class FrameWriterBinaryBlock extends FrameWriter * @throws IOException if IOException occurs */ @SuppressWarnings("deprecation") - protected final void writeBinaryBlockFrameToSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int blen, int rl, int ru ) + protected static void writeBinaryBlockFrameToSequenceFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int blen, int rl, int ru ) throws DMLRuntimeException, IOException { //1) create sequence file writer http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java index 7d15d9b..97dab12 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCSV.java @@ -81,7 +81,7 @@ public class FrameWriterTextCSV extends FrameWriter IOUtilFunctions.deleteCrcFilesFromLocalFileSystem(fs, path); } - protected final void writeCSVFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru, CSVFileFormatProperties props ) + protected static void writeCSVFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru, CSVFileFormatProperties props ) throws IOException { //create buffered writer http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java index 134d961..2bbd206 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameWriterTextCell.java @@ -82,7 +82,7 @@ public class FrameWriterTextCell extends FrameWriter * @param ru upper row * @throws IOException if IOException occurs */ - protected final void writeTextCellFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru ) + protected static void writeTextCellFrameToFile( Path path, JobConf job, FileSystem fs, FrameBlock src, int rl, int ru ) throws IOException { boolean entriesWritten = false; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java index 5b5b8ca..5e7df99 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryBlock.java @@ -208,7 +208,7 @@ public class ReaderBinaryBlock extends MatrixReader } } - private void readBinaryBlockMatrixBlocksFromHDFS( Path path, JobConf job, FileSystem fs, Collection<IndexedMatrixValue> dest, long rlen, long clen, int brlen, int bclen ) + private static void readBinaryBlockMatrixBlocksFromHDFS( Path path, JobConf job, FileSystem fs, Collection<IndexedMatrixValue> dest, long rlen, long clen, int brlen, int bclen ) throws IOException { MatrixIndexes key = new MatrixIndexes(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java index 00a1301..3860ca2 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderBinaryCell.java @@ -68,7 +68,7 @@ public class ReaderBinaryCell extends MatrixReader } @SuppressWarnings("deprecation") - private void readBinaryCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen ) + private static void readBinaryCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen ) throws IOException { boolean sparse = dest.isInSparseFormat(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java index 42b1184..8d2ef72 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCSV.java @@ -96,7 +96,7 @@ public class ReaderTextCSV extends MatrixReader } @SuppressWarnings("unchecked") - private MatrixBlock readCSVMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, + private static MatrixBlock readCSVMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean hasHeader, String delim, boolean fill, double fillValue ) throws IOException, DMLRuntimeException { @@ -130,7 +130,7 @@ public class ReaderTextCSV extends MatrixReader return dest; } - private long readCSVMatrixFromInputStream( InputStream is, String srcInfo, MatrixBlock dest, MutableInt rowPos, + private static long readCSVMatrixFromInputStream( InputStream is, String srcInfo, MatrixBlock dest, MutableInt rowPos, long rlen, long clen, int brlen, int bclen, boolean hasHeader, String delim, boolean fill, double fillValue, boolean first ) throws IOException { @@ -221,7 +221,7 @@ public class ReaderTextCSV extends MatrixReader return lnnz; } - private MatrixBlock computeCSVSize( List<Path> files, JobConf job, FileSystem fs, boolean hasHeader, String delim, boolean fill, double fillValue) + private static MatrixBlock computeCSVSize( List<Path> files, JobConf job, FileSystem fs, boolean hasHeader, String delim, boolean fill, double fillValue) throws IOException, DMLRuntimeException { int nrow = -1; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java index fd3af44..7c51637 100644 --- a/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java +++ b/src/main/java/org/apache/sysml/runtime/io/ReaderTextCell.java @@ -97,7 +97,7 @@ public class ReaderTextCell extends MatrixReader return ret; } - private void readTextCellMatrixFromHDFS( Path path, JobConf job, MatrixBlock dest, long rlen, long clen, int brlen, int bclen ) + private static void readTextCellMatrixFromHDFS( Path path, JobConf job, MatrixBlock dest, long rlen, long clen, int brlen, int bclen ) throws IOException { boolean sparse = dest.isInSparseFormat(); @@ -159,7 +159,7 @@ public class ReaderTextCell extends MatrixReader } } - private void readRawTextCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket ) + private static void readRawTextCellMatrixFromHDFS( Path path, JobConf job, FileSystem fs, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket ) throws IOException { //create input stream for path @@ -169,7 +169,7 @@ public class ReaderTextCell extends MatrixReader readRawTextCellMatrixFromInputStream(inputStream, dest, rlen, clen, brlen, bclen, matrixMarket); } - private void readRawTextCellMatrixFromInputStream( InputStream is, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket ) + private static void readRawTextCellMatrixFromInputStream( InputStream is, MatrixBlock dest, long rlen, long clen, int brlen, int bclen, boolean matrixMarket ) throws IOException { BufferedReader br = new BufferedReader(new InputStreamReader( is )); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java index 78ffc0b..ae867bf 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterBinaryBlock.java @@ -240,7 +240,7 @@ public class WriterBinaryBlock extends MatrixWriter //copy submatrix to block src.sliceOperations( row_offset, row_offset+maxRow-1, - col_offset, col_offset+maxCol-1, block ); + col_offset, col_offset+maxCol-1, block ); } else //empty block (not on diagonal) { @@ -264,7 +264,7 @@ public class WriterBinaryBlock extends MatrixWriter } @SuppressWarnings("deprecation") - public final void writePartitionedBinaryBlockMatrixToHDFS( Path path, JobConf job, MatrixBlock src, long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pformat ) + public static final void writePartitionedBinaryBlockMatrixToHDFS( Path path, JobConf job, MatrixBlock src, long rlen, long clen, int brlen, int bclen, PDataPartitionFormat pformat ) throws IOException, DMLRuntimeException { boolean sparse = src.isInSparseFormat(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java index a2e2013..f862f3d 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterMatrixMarket.java @@ -89,7 +89,7 @@ public class WriterMatrixMarket extends MatrixWriter writeMatrixMarketMatrixToFile(path, job, fs, src, 0, src.getNumRows()); } - protected final void writeMatrixMarketMatrixToFile( Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru ) + protected static void writeMatrixMarketMatrixToFile( Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru ) throws IOException { boolean sparse = src.isInSparseFormat(); @@ -165,7 +165,7 @@ public class WriterMatrixMarket extends MatrixWriter } } - public final void mergeTextcellToMatrixMarket( String srcFileName, String fileName, long rlen, long clen, long nnz ) + public static void mergeTextcellToMatrixMarket( String srcFileName, String fileName, long rlen, long clen, long nnz ) throws IOException { Configuration conf = new Configuration(ConfigurationManager.getCachedJobConf()); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java index 6c20baa..f5c62ac 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCSV.java @@ -97,7 +97,7 @@ public class WriterTextCSV extends MatrixWriter writeCSVMatrixToFile(path, job, fs, src, 0, (int)src.getNumRows(), csvprops); } - protected final void writeCSVMatrixToFile( Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru, CSVFileFormatProperties props ) + protected static void writeCSVMatrixToFile( Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru, CSVFileFormatProperties props ) throws IOException { boolean sparse = src.isInSparseFormat(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java index 432d363..04dfd5a 100644 --- a/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java +++ b/src/main/java/org/apache/sysml/runtime/io/WriterTextCell.java @@ -79,7 +79,7 @@ public class WriterTextCell extends MatrixWriter writeTextCellMatrixToFile(path, job, fs, src, 0, (int)rlen); } - protected final void writeTextCellMatrixToFile( Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru ) + protected static void writeTextCellMatrixToFile( Path path, JobConf job, FileSystem fs, MatrixBlock src, int rl, int ru ) throws IOException { boolean sparse = src.isInSparseFormat(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java index 137b25e..7a3b474 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/SortMR.java @@ -114,7 +114,7 @@ public class SortMR * @return list of writable comparables * @throws IOException if IOException occurs */ - private ArrayList<WritableComparable> readPartitions(FileSystem fs, Path p, JobConf job) + private static ArrayList<WritableComparable> readPartitions(FileSystem fs, Path p, JobConf job) throws IOException { ArrayList<WritableComparable> parts = new ArrayList<WritableComparable>(); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java b/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java index 6d06ee5..2f180b0 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/ConvolutionParameters.java @@ -43,10 +43,9 @@ public class ConvolutionParameters implements Serializable { public MatrixBlock bias; public int [] start_indexes_h, end_indexes_h, start_indexes_w, end_indexes_w; - private int convertToInt(long val) throws DMLRuntimeException { - if( val > Integer.MAX_VALUE ) { + private static int convertToInt(long val) throws DMLRuntimeException { + if( val > Integer.MAX_VALUE ) throw new DMLRuntimeException("The value for ConvolutionParameters is too large:" + val); - } return (int) val; } http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java index b3dc311..6079956 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/FrameBlock.java @@ -759,7 +759,7 @@ public class FrameBlock implements Writable, CacheBlock, Externalizable * @param value string value * @return in-memory size of string value */ - private long getInMemoryStringSize(String value) { + private static long getInMemoryStringSize(String value) { if( value == null ) return 0; return 16 + 4 + 8 //object, hash, array ref http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java index 1e2649d..7622137 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixBincell.java @@ -411,7 +411,7 @@ public class LibMatrixBincell //prepare second input and allocate output MatrixBlock b = m1.sparse ? m2 : m1; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); for( int i=0; i<a.numRows(); i++ ) { if( a.isEmpty(i) ) continue; http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNConv2dHelper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNConv2dHelper.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNConv2dHelper.java index 66b2ed1..2685f52 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNConv2dHelper.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixDNNConv2dHelper.java @@ -83,7 +83,7 @@ public class LibMatrixDNNConv2dHelper { } // Copy the matrix src of shape [K X PQ] to params.output.denseBlock + destPos - private void add(MatrixBlock src, double [] dest, int destPos, int K, int PQ) { + private static void add(MatrixBlock src, double [] dest, int destPos, int K, int PQ) { // Copying is required as LibMatrixMult.matrixMult (and/or Java) is not pointer aware. // This is not required in Native implementation if(!src.isEmptyBlock()) { @@ -162,7 +162,7 @@ public class LibMatrixDNNConv2dHelper { } // Copy the matrix src of shape [K X PQ] to params.output.denseBlock + destPos - private void partialCopy1(MatrixBlock src, double [] dest, int destPos, int K, int PQ) { + private static void partialCopy1(MatrixBlock src, double [] dest, int destPos, int K, int PQ) { // Copying is required as LibMatrixMult.matrixMult (and/or Java) is not pointer aware. // This is not required in Native implementation if(!src.isEmptyBlock()) { http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java index 116c8a8..0885508 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixMult.java @@ -595,7 +595,7 @@ public class LibMatrixMult //pre-processing ret.sparse = mW.sparse; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); //core weighted square sum mm computation if( !mW.sparse && !mU.sparse && !mV.sparse && !mU.isEmptyBlock() && !mV.isEmptyBlock() ) @@ -632,7 +632,7 @@ public class LibMatrixMult //pre-processing ret.sparse = mW.sparse; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); try { @@ -691,7 +691,7 @@ public class LibMatrixMult //pre-processing ret.sparse = wt.isBasic()?mW.sparse:false; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); //core weighted div mm computation boolean scalarX = wt.hasScalar(); @@ -742,7 +742,7 @@ public class LibMatrixMult //pre-processing ret.sparse = wt.isBasic()?mW.sparse:false; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); if (!ret.isThreadSafe()){ matrixMultWDivMM(mW, mU, mV, mX, ret, wt); @@ -859,7 +859,7 @@ public class LibMatrixMult //pre-processing ret.sparse = mW.sparse; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); //core weighted square sum mm computation if( !mW.sparse && !mU.sparse && !mV.sparse && !mU.isEmptyBlock() && !mV.isEmptyBlock() ) @@ -896,10 +896,10 @@ public class LibMatrixMult //pre-processing ret.sparse = mW.sparse; - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); try - { + { ExecutorService pool = Executors.newFixedThreadPool(k); ArrayList<MatrixMultWuTask> tasks = new ArrayList<MatrixMultWuTask>(); int blklen = (int)(Math.ceil((double)mW.rlen/k)); http://git-wip-us.apache.org/repos/asf/systemml/blob/13a01758/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java index cb98aee..c64bbc6 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixReorg.java @@ -1897,7 +1897,7 @@ public class LibMatrixReorg final long nnz = in.nonZeros; boolean sp = MatrixBlock.evalSparseFormatInMemory(rlen, clen, nnz); ret.reset(rlen, clen, sp); - ret.allocateDenseOrSparseBlock(); + ret.allocateBlock(); //execute rexpand columns long rnnz = 0; //real nnz (due to cutoff max)
