http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/parser/WhileStatement.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/WhileStatement.java b/src/main/java/org/apache/sysml/parser/WhileStatement.java index e210818..d70b1ab 100644 --- a/src/main/java/org/apache/sysml/parser/WhileStatement.java +++ b/src/main/java/org/apache/sysml/parser/WhileStatement.java @@ -28,6 +28,7 @@ public class WhileStatement extends Statement private ConditionalPredicate _predicate; private ArrayList<StatementBlock> _body; + @Override public Statement rewriteStatement(String prefix) throws LanguageException{ LOG.error(this.printErrorLocation() + "should not call rewriteStatement for WhileStatement"); throw new LanguageException(this.printErrorLocation() + "should not call rewriteStatement for WhileStatement"); @@ -67,6 +68,7 @@ public class WhileStatement extends Statement _body = StatementBlock.mergeStatementBlocks(_body); } + @Override public String toString(){ StringBuilder sb = new StringBuilder(); sb.append("while ( "); @@ -79,15 +81,16 @@ public class WhileStatement extends Statement return sb.toString(); } + @Override public void initializeforwardLV(VariableSet activeIn) throws LanguageException{ LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement"); throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement"); } + @Override public VariableSet initializebackwardLV(VariableSet lo) throws LanguageException{ LOG.error(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement"); throw new LanguageException(this.printErrorLocation() + "should never call initializeforwardLV for WhileStatement"); - } @Override
http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java index 866bf1c..f3f613e 100644 --- a/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java +++ b/src/main/java/org/apache/sysml/parser/WhileStatementBlock.java @@ -164,7 +164,7 @@ public class WhileStatementBlock extends StatementBlock return ids; } - + @Override public VariableSet initializeforwardLV(VariableSet activeInPassed) throws LanguageException { WhileStatement wstmt = (WhileStatement)_statements.get(0); @@ -223,6 +223,7 @@ public class WhileStatementBlock extends StatementBlock return _liveOut; } + @Override public VariableSet initializebackwardLV(VariableSet loPassed) throws LanguageException{ WhileStatement wstmt = (WhileStatement)_statements.get(0); @@ -246,6 +247,7 @@ public class WhileStatementBlock extends StatementBlock _predicateHops = hops; } + @Override public ArrayList<Hop> get_hops() throws HopsException { if (_hops != null && !_hops.isEmpty()){ @@ -268,8 +270,8 @@ public class WhileStatementBlock extends StatementBlock _predicateLops = predicateLops; } + @Override public VariableSet analyze(VariableSet loPassed) throws LanguageException{ - VariableSet predVars = new VariableSet(); predVars.addVariables(((WhileStatement)_statements.get(0)).getConditionalPredicate().variablesRead()); predVars.addVariables(((WhileStatement)_statements.get(0)).getConditionalPredicate().variablesUpdated()); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java index f42f09f..b7a1c89 100644 --- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java @@ -53,6 +53,7 @@ import org.apache.sysml.parser.LanguageException; import org.apache.sysml.parser.ParForStatement; import org.apache.sysml.parser.ParameterExpression; import org.apache.sysml.parser.ParseException; +import org.apache.sysml.parser.ParserWrapper; import org.apache.sysml.parser.PathStatement; import org.apache.sysml.parser.Statement; import org.apache.sysml.parser.StatementBlock; @@ -599,7 +600,7 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D // ----------------------------------------------------------------- private static StatementBlock getStatementBlock(Statement current) { - return DMLParserWrapper.getStatementBlock(current); + return ParserWrapper.getStatementBlock(current); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java index d5c5e67..9c1510b 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java @@ -57,6 +57,7 @@ import org.apache.sysml.parser.LanguageException; import org.apache.sysml.parser.ParForStatement; import org.apache.sysml.parser.ParameterExpression; import org.apache.sysml.parser.ParseException; +import org.apache.sysml.parser.ParserWrapper; import org.apache.sysml.parser.PathStatement; import org.apache.sysml.parser.Statement; import org.apache.sysml.parser.StatementBlock; @@ -1217,7 +1218,7 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements // ----------------------------------------------------------------- private static StatementBlock getStatementBlock(Statement current) { - return PyDMLParserWrapper.getStatementBlock(current); + return ParserWrapper.getStatementBlock(current); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java index 4f55c79..ac8dc57 100644 --- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java +++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java @@ -149,6 +149,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator return new DoubleObject(sum); } + @Override public MatrixBlock execute(ArrayList<MatrixBlock> inputs, ArrayList<ScalarObject> scalarObjects, MatrixBlock out) throws DMLRuntimeException { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java index 91ae7d6..d84abfa 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlock.java @@ -912,6 +912,7 @@ public class ExternalFunctionProgramBlock extends FunctionProgramBlock return _otherParams; } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in external function program block generated from external function statement block between lines " + _beginLine + " and " + _endLine + " -- "; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java index d5a9125..29806c2 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ExternalFunctionProgramBlockCP.java @@ -30,7 +30,6 @@ import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.controlprogram.parfor.util.IDSequence; -import org.apache.sysml.runtime.instructions.Instruction; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; import org.apache.sysml.runtime.matrix.data.InputInfo; @@ -77,7 +76,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock super(prog, inputParams, outputParams, baseDir); //w/o instruction generation // copy other params - _otherParams = new HashMap<String, String>(); + _otherParams = new HashMap<>(); _otherParams.putAll(otherParams); // generate instructions (overwritten) @@ -131,7 +130,7 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock @Override protected void createInstructions() { - _inst = new ArrayList<Instruction>(); + _inst = new ArrayList<>(); // assemble information provided through keyvalue pairs String className = _otherParams.get(ExternalFunctionStatement.CLASS_NAME); @@ -180,16 +179,14 @@ public class ExternalFunctionProgramBlockCP extends ExternalFunctionProgramBlock } return ret; - } - + } - public String createDefaultOutputFilePathAndName( ) - { + public String createDefaultOutputFilePathAndName( ) { return _baseDir + DEFAULT_FILENAME + _defaultSeq.getNextID(); - } + } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in external function program block (for CP) generated from external function statement block between lines " + _beginLine + " and " + _endLine + " -- "; } - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java index 05b622d..29dfdfd 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java @@ -36,7 +36,7 @@ import org.apache.sysml.runtime.instructions.cp.ScalarObject; import org.apache.sysml.yarn.DMLAppMasterUtils; public class ForProgramBlock extends ProgramBlock -{ +{ protected ArrayList<Instruction> _fromInstructions; protected ArrayList<Instruction> _toInstructions; protected ArrayList<Instruction> _incrementInstructions; @@ -46,9 +46,8 @@ public class ForProgramBlock extends ProgramBlock public ForProgramBlock(Program prog, String iterPredVar) { super(prog); - - _exitInstructions = new ArrayList<Instruction>(); - _childBlocks = new ArrayList<ProgramBlock>(); + _exitInstructions = new ArrayList<>(); + _childBlocks = new ArrayList<>(); _iterPredVar = iterPredVar; } @@ -211,6 +210,7 @@ public class ForProgramBlock extends ProgramBlock return ret; } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in for program block generated from for statement block between lines " + _beginLine + " and " + _endLine + " -- "; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java index aa51c59..fa4838a 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java @@ -152,8 +152,8 @@ public class FunctionProgramBlock extends ProgramBlock return _recompileOnce; } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in function program block generated from function statement block between lines " + _beginLine + " and " + _endLine + " -- "; } - } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java index cecc8f7..b09a1a3 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java @@ -41,11 +41,10 @@ public class IfProgramBlock extends ProgramBlock public IfProgramBlock(Program prog, ArrayList<Instruction> predicate) { super(prog); - - _childBlocksIfBody = new ArrayList<ProgramBlock>(); - _childBlocksElseBody = new ArrayList<ProgramBlock>(); + _childBlocksIfBody = new ArrayList<>(); + _childBlocksElseBody = new ArrayList<>(); _predicate = predicate; - _exitInstructions = new ArrayList<Instruction>(); + _exitInstructions = new ArrayList<>(); } public ArrayList<ProgramBlock> getChildBlocksIfBody() { @@ -177,6 +176,7 @@ public class IfProgramBlock extends ProgramBlock return result; } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in if program block generated from if statement block between lines " + _beginLine + " and " + _endLine + " -- "; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java index 7e59951..0743d39 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java @@ -36,7 +36,7 @@ import org.apache.sysml.runtime.instructions.cp.Data; * */ public class LocalVariableMap implements Cloneable -{ +{ private static String eol = System.getProperty ("line.separator"); private static String ELEMENT_DELIM = org.apache.sysml.runtime.controlprogram.parfor.ProgramConverter.ELEMENT_DELIM; private static IDSequence _seq = new IDSequence(); @@ -45,12 +45,12 @@ public class LocalVariableMap implements Cloneable private final long localID; public LocalVariableMap() { - localMap = new HashMap <String, Data>(); + localMap = new HashMap<>(); localID = _seq.getNextID(); } public LocalVariableMap(LocalVariableMap vars) { - localMap = new HashMap <String, Data>(vars.localMap); + localMap = new HashMap<>(vars.localMap); localID = _seq.getNextID(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java index e345472..e2568cb 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java @@ -426,7 +426,7 @@ public class ParForProgramBlock extends ForProgramBlock //initialize program block cache if necessary if( USE_PB_CACHE ) - _pbcache = new HashMap<Long, ArrayList<ProgramBlock>>(); + _pbcache = new HashMap<>(); //created profiling report after parfor exec _monitorReport = _monitor; @@ -751,11 +751,10 @@ public class ParForProgramBlock extends ForProgramBlock try { // Step 1) init parallel workers, task queue and threads - LocalTaskQueue<Task> queue = new LocalTaskQueue<Task>(); + LocalTaskQueue<Task> queue = new LocalTaskQueue<>(); Thread[] threads = new Thread[_numThreads]; LocalParWorker[] workers = new LocalParWorker[_numThreads]; - for( int i=0; i<_numThreads; i++ ) - { + for( int i=0; i<_numThreads; i++ ) { //create parallel workers as (lazy) deep copies //including preparation of update-in-place variables workers[i] = createParallelWorker( _pwIDs[i], queue, ec, i); @@ -781,7 +780,7 @@ public class ParForProgramBlock extends ForProgramBlock if( USE_STREAMING_TASK_CREATION ) { //put tasks into queue (parworker start work on first tasks while creating tasks) - numCreatedTasks = partitioner.createTasks(queue); + numCreatedTasks = partitioner.createTasks(queue); } else { @@ -793,7 +792,7 @@ public class ParForProgramBlock extends ForProgramBlock queue.enqueueTask( t ); // mark end of task input stream - queue.closeInput(); + queue.closeInput(); } if( _monitor ) StatisticMonitor.putPFStat(_ID, Stat.PARFOR_INIT_TASKS_T, time.stop()); @@ -810,9 +809,9 @@ public class ParForProgramBlock extends ForProgramBlock LocalVariableMap [] localVariables = new LocalVariableMap [_numThreads]; for( int i=0; i<_numThreads; i++ ) { localVariables[i] = workers[i].getVariables(); - localVariables[i].removeAllNotIn(new HashSet<String>(_resultVars)); + localVariables[i].removeAllNotIn(new HashSet<>(_resultVars)); numExecutedTasks += workers[i].getExecutedTasks(); - numExecutedIterations += workers[i].getExecutedIterations(); + numExecutedIterations += workers[i].getExecutedIterations(); } //consolidate results into global symbol table consolidateAndCheckResults( ec, numIterations, numCreatedTasks, @@ -891,7 +890,7 @@ public class ParForProgramBlock extends ForProgramBlock int maxDigits = (int)Math.log10(to.getLongValue()) + 1; long numCreatedTasks = -1; if( USE_STREAMING_TASK_CREATION ) { - LocalTaskQueue<Task> queue = new LocalTaskQueue<Task>(); + LocalTaskQueue<Task> queue = new LocalTaskQueue<>(); //put tasks into queue and start writing to taskFile numCreatedTasks = partitioner.createTasks(queue); @@ -1025,7 +1024,7 @@ public class ParForProgramBlock extends ForProgramBlock // NOTES: each mapper changes filenames with regard to his ID as we submit a single // job, cannot reuse serialized string, since variables are serialized as well. ParForBody body = new ParForBody(_childBlocks, _resultVars, ec); - HashMap<String, byte[]> clsMap = new HashMap<String, byte[]>(); + HashMap<String, byte[]> clsMap = new HashMap<>(); String program = ProgramConverter.serializeParForBody(body, clsMap); if( _monitor ) @@ -1087,7 +1086,7 @@ public class ParForProgramBlock extends ForProgramBlock // NOTES: each mapper changes filenames with regard to his ID as we submit a single // job, cannot reuse serialized string, since variables are serialized as well. ParForBody body = new ParForBody( _childBlocks, _resultVars, ec ); - HashMap<String, byte[]> clsMap = new HashMap<String, byte[]>(); + HashMap<String, byte[]> clsMap = new HashMap<>(); String program = ProgramConverter.serializeParForBody( body, clsMap ); if( _monitor ) @@ -1302,7 +1301,7 @@ public class ParForProgramBlock extends ForProgramBlock throws CacheException { ParForStatementBlock sb = (ParForStatementBlock)getStatementBlock(); - HashSet<String> blacklist = new HashSet<String>(Arrays.asList(blacklistNames)); + HashSet<String> blacklist = new HashSet<>(Arrays.asList(blacklistNames)); if( LIVEVAR_AWARE_EXPORT && sb != null) { @@ -1358,7 +1357,7 @@ public class ParForProgramBlock extends ForProgramBlock { //create deep copies of required elements child blocks ArrayList<ProgramBlock> cpChildBlocks = null; - HashSet<String> fnNames = new HashSet<String>(); + HashSet<String> fnNames = new HashSet<>(); if( USE_PB_CACHE ) { if( _pbcache.containsKey(pwID) ) { @@ -1568,7 +1567,7 @@ public class ParForProgramBlock extends ForProgramBlock } //try recompile MR instructions to CP - HashSet<String> fnStack = new HashSet<String>(); + HashSet<String> fnStack = new HashSet<>(); Recompiler.recompileProgramBlockHierarchy2Forced(_childBlocks, tid, fnStack, ExecType.CP); return true; } @@ -1660,7 +1659,7 @@ public class ParForProgramBlock extends ForProgramBlock try { //enqueue all result vars as tasks - LocalTaskQueue<String> q = new LocalTaskQueue<String>(); + LocalTaskQueue<String> q = new LocalTaskQueue<>(); for( String var : _resultVars ) //foreach non-local write if( ec.getVariable(var) instanceof MatrixObject ) //robustness scalars q.enqueueTask(var); @@ -1971,6 +1970,7 @@ public class ParForProgramBlock extends ForProgramBlock } } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in parfor program block generated from parfor statement block between lines " + _beginLine + " and " + _endLine + " -- "; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java b/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java index 95b2e01..3867659 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/Program.java @@ -39,8 +39,8 @@ public class Program private HashMap<String, HashMap<String,FunctionProgramBlock>> _namespaceFunctions; public Program() throws DMLRuntimeException { - _namespaceFunctions = new HashMap<String, HashMap<String,FunctionProgramBlock>>(); - _programBlocks = new ArrayList<ProgramBlock>(); + _namespaceFunctions = new HashMap<>(); + _programBlocks = new ArrayList<>(); } public synchronized void addFunctionProgramBlock(String namespace, String fname, FunctionProgramBlock fpb) @@ -51,18 +51,16 @@ public class Program HashMap<String,FunctionProgramBlock> namespaceBlocks = null; namespaceBlocks = _namespaceFunctions.get(namespace); if (namespaceBlocks == null){ - namespaceBlocks = new HashMap<String,FunctionProgramBlock>(); + namespaceBlocks = new HashMap<>(); _namespaceFunctions.put(namespace,namespaceBlocks); } namespaceBlocks.put(fname,fpb); } - public synchronized void removeFunctionProgramBlock(String namespace, String fname) - { + public synchronized void removeFunctionProgramBlock(String namespace, String fname) { if (namespace == null) namespace = DMLProgram.DEFAULT_NAMESPACE; - HashMap<String,FunctionProgramBlock> namespaceBlocks = null; if( _namespaceFunctions.containsKey(namespace) ){ namespaceBlocks = _namespaceFunctions.get(namespace); @@ -72,8 +70,7 @@ public class Program } public synchronized HashMap<String,FunctionProgramBlock> getFunctionProgramBlocks(){ - - HashMap<String,FunctionProgramBlock> retVal = new HashMap<String,FunctionProgramBlock>(); + HashMap<String,FunctionProgramBlock> retVal = new HashMap<>(); //create copy of function program blocks for (String namespace : _namespaceFunctions.keySet()){ @@ -90,7 +87,6 @@ public class Program } public synchronized FunctionProgramBlock getFunctionProgramBlock(String namespace, String fname) throws DMLRuntimeException{ - if (namespace == null) namespace = DMLProgram.DEFAULT_NAMESPACE; HashMap<String,FunctionProgramBlock> namespaceFunctBlocks = _namespaceFunctions.get(namespace); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java index d1fc759..4ac9af0 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java @@ -64,13 +64,11 @@ public class ProgramBlock implements ParseInfo protected StatementBlock _sb = null; protected long _tid = 0; //by default _t0 - public ProgramBlock(Program prog) { _prog = prog; - _inst = new ArrayList<Instruction>(); + _inst = new ArrayList<>(); } - //////////////////////////////////////////////// // getters, setters and similar functionality //////////////////////////////////////////////// http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java index 6c8ed80..aac1f52 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java @@ -43,8 +43,8 @@ public class WhileProgramBlock extends ProgramBlock public WhileProgramBlock(Program prog, ArrayList<Instruction> predicate) { super(prog); _predicate = predicate; - _exitInstructions = new ArrayList<Instruction>(); - _childBlocks = new ArrayList<ProgramBlock>(); + _exitInstructions = new ArrayList<>(); + _childBlocks = new ArrayList<>(); } public void addProgramBlock(ProgramBlock childBlock) { @@ -98,6 +98,7 @@ public class WhileProgramBlock extends ProgramBlock return result; } + @Override public void execute(ExecutionContext ec) throws DMLRuntimeException { //execute while loop @@ -143,6 +144,7 @@ public class WhileProgramBlock extends ProgramBlock _childBlocks = childs; } + @Override public String printBlockErrorLocation(){ return "ERROR: Runtime error in while program block generated from while statement block between lines " + _beginLine + " and " + _endLine + " -- "; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java index 720b37d..5297e61 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/CacheableData.java @@ -210,7 +210,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data _uniqueID = (int)_seq.getNextID(); _cacheStatus = CacheStatus.EMPTY; _numReadThreads = 0; - _gpuObjects = new HashMap<GPUContext, GPUObject>(); + _gpuObjects = new HashMap<>(); } /** @@ -1254,7 +1254,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data * referenced cache block. */ protected void createCache( ) { - _cache = new SoftReference<T>( _data ); + _cache = new SoftReference<>( _data ); } /** @@ -1420,6 +1420,7 @@ public abstract class CacheableData<T extends CacheBlock> extends Data return ret; } + @Override public String toString() { StringBuilder str = new StringBuilder(); str.append(getClass().getSimpleName()); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java index 4105351..e9017e5 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/caching/MatrixObject.java @@ -329,7 +329,7 @@ public class MatrixObject extends CacheableData<MatrixBlock> { //put block into cache _partitionCacheName = fname; - _cache = new SoftReference<MatrixBlock>(mb); + _cache = new SoftReference<>(mb); if( _partitionFormat == PDataPartitionFormat.ROW_BLOCK_WISE ) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java index f4c024d..4cd371b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/ExecutionContext.java @@ -320,28 +320,28 @@ public class ExecutionContext { MatrixObject mo = allocateGPUMatrixObject(varName, numRows, numCols); boolean allocated = mo.getGPUObject(getGPUContext(0)).acquireDeviceModifyDense(); mo.getMatrixCharacteristics().setNonZeros(-1); - return new Pair<MatrixObject, Boolean>(mo, allocated); + return new Pair<>(mo, allocated); } /** - * Allocates a sparse matrix in CSR format on the GPU. - * Assumes that mat.getNumRows() returns a valid number - * - * @param varName variable name - * @param numRows number of rows of matrix object + * Allocates a sparse matrix in CSR format on the GPU. + * Assumes that mat.getNumRows() returns a valid number + * + * @param varName variable name + * @param numRows number of rows of matrix object * @param numCols number of columns of matrix object - * @param nnz number of non zeroes - * @return matrix object - * @throws DMLRuntimeException if DMLRuntimeException occurs - */ - public Pair<MatrixObject, Boolean> getSparseMatrixOutputForGPUInstruction(String varName, long numRows, long numCols, long nnz) - throws DMLRuntimeException - { - MatrixObject mo = allocateGPUMatrixObject(varName, numRows, numCols); - mo.getMatrixCharacteristics().setNonZeros(nnz); + * @param nnz number of non zeroes + * @return matrix object + * @throws DMLRuntimeException if DMLRuntimeException occurs + */ + public Pair<MatrixObject, Boolean> getSparseMatrixOutputForGPUInstruction(String varName, long numRows, long numCols, long nnz) + throws DMLRuntimeException + { + MatrixObject mo = allocateGPUMatrixObject(varName, numRows, numCols); + mo.getMatrixCharacteristics().setNonZeros(nnz); boolean allocated = mo.getGPUObject(getGPUContext(0)).acquireDeviceModifySparse(); - return new Pair<MatrixObject, Boolean>(mo, allocated); - } + return new Pair<>(mo, allocated); + } /** * Allocates the {@link GPUObject} for a given LOPS Variable (eg. _mVar3) @@ -535,7 +535,7 @@ public class ExecutionContext { public HashMap<String,Boolean> pinVariables(ArrayList<String> varList) { //2-pass approach since multiple vars might refer to same matrix object - HashMap<String, Boolean> varsState = new HashMap<String,Boolean>(); + HashMap<String, Boolean> varsState = new HashMap<>(); //step 1) get current information for( String var : varList ) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index 472b735..be95164 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -534,7 +534,7 @@ public class SparkExecutionContext extends ExecutionContext //create partitioned matrix block and release memory consumed by input MatrixBlock mb = mo.acquireRead(); - PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<MatrixBlock>(mb, brlen, bclen); + PartitionedBlock<MatrixBlock> pmb = new PartitionedBlock<>(mb, brlen, bclen); mo.release(); //determine coarse-grained partitioning @@ -559,8 +559,8 @@ public class SparkExecutionContext extends ExecutionContext pmb.clearBlocks(); } - bret = new PartitionedBroadcast<MatrixBlock>(ret); - BroadcastObject<MatrixBlock> bchandle = new BroadcastObject<MatrixBlock>(bret, varname, + bret = new PartitionedBroadcast<>(ret); + BroadcastObject<MatrixBlock> bchandle = new BroadcastObject<>(bret, varname, OptimizerUtils.estimatePartitionedSizeExactSparsity(mo.getMatrixCharacteristics())); mo.setBroadcastHandle(bchandle); CacheableData.addBroadcastSize(bchandle.getSize()); @@ -604,7 +604,7 @@ public class SparkExecutionContext extends ExecutionContext //create partitioned frame block and release memory consumed by input FrameBlock mb = fo.acquireRead(); - PartitionedBlock<FrameBlock> pmb = new PartitionedBlock<FrameBlock>(mb, brlen, bclen); + PartitionedBlock<FrameBlock> pmb = new PartitionedBlock<>(mb, brlen, bclen); fo.release(); //determine coarse-grained partitioning @@ -629,8 +629,8 @@ public class SparkExecutionContext extends ExecutionContext pmb.clearBlocks(); } - bret = new PartitionedBroadcast<FrameBlock>(ret); - BroadcastObject<FrameBlock> bchandle = new BroadcastObject<FrameBlock>(bret, varname, + bret = new PartitionedBroadcast<>(ret); + BroadcastObject<FrameBlock> bchandle = new BroadcastObject<>(bret, varname, OptimizerUtils.estimatePartitionedSizeExactSparsity(fo.getMatrixCharacteristics())); fo.setBroadcastHandle(bchandle); CacheableData.addBroadcastSize(bchandle.getSize()); @@ -674,12 +674,12 @@ public class SparkExecutionContext extends ExecutionContext throws DMLRuntimeException { long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0; - LinkedList<Tuple2<MatrixIndexes,MatrixBlock>> list = new LinkedList<Tuple2<MatrixIndexes,MatrixBlock>>(); + LinkedList<Tuple2<MatrixIndexes,MatrixBlock>> list = new LinkedList<>(); if( src.getNumRows() <= brlen && src.getNumColumns() <= bclen ) { - list.addLast(new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(1,1), src)); + list.addLast(new Tuple2<>(new MatrixIndexes(1,1), src)); } else { @@ -703,7 +703,7 @@ public class SparkExecutionContext extends ExecutionContext //append block to sequence file MatrixIndexes indexes = new MatrixIndexes(blockRow+1, blockCol+1); - list.addLast(new Tuple2<MatrixIndexes,MatrixBlock>(indexes, block)); + list.addLast(new Tuple2<>(indexes, block)); } } @@ -720,7 +720,7 @@ public class SparkExecutionContext extends ExecutionContext throws DMLRuntimeException { long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0; - LinkedList<Tuple2<Long,FrameBlock>> list = new LinkedList<Tuple2<Long,FrameBlock>>(); + LinkedList<Tuple2<Long,FrameBlock>> list = new LinkedList<>(); //create and write subblocks of matrix int blksize = ConfigurationManager.getBlocksize(); @@ -737,7 +737,7 @@ public class SparkExecutionContext extends ExecutionContext block.setColumnMetadata(src.getColumnMetadata()); //append block to sequence file - list.addLast(new Tuple2<Long,FrameBlock>((long)roffset+1, block)); + list.addLast(new Tuple2<>((long)roffset+1, block)); } JavaPairRDD<Long,FrameBlock> result = sc.parallelizePairs(list); @@ -930,7 +930,7 @@ public class SparkExecutionContext extends ExecutionContext long t0 = DMLScript.STATISTICS ? System.nanoTime() : 0; - PartitionedBlock<MatrixBlock> out = new PartitionedBlock<MatrixBlock>(rlen, clen, brlen, bclen); + PartitionedBlock<MatrixBlock> out = new PartitionedBlock<>(rlen, clen, brlen, bclen); List<Tuple2<MatrixIndexes,MatrixBlock>> list = rdd.collect(); //copy blocks one-at-a-time into output matrix block @@ -1567,7 +1567,7 @@ public class SparkExecutionContext extends ExecutionContext public MemoryManagerParRDDs(double fractionMem) { _limit = (long)(fractionMem * InfrastructureAnalyzer.getLocalMaxMemory()); _size = 0; - _rdds = new HashMap<Integer, Long>(); + _rdds = new HashMap<>(); } public synchronized boolean reserve(long rddSize) { http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java index 0f76156..89456dc 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/DataPartitionerRemoteSparkMapper.java @@ -72,7 +72,7 @@ public class DataPartitionerRemoteSparkMapper extends ParWorker implements PairF public Iterator<Tuple2<Long, Writable>> call(Tuple2<MatrixIndexes, MatrixBlock> arg0) throws Exception { - List<Tuple2<Long, Writable>> ret = new LinkedList<Tuple2<Long, Writable>>(); + List<Tuple2<Long, Writable>> ret = new LinkedList<>(); MatrixIndexes key2 = arg0._1(); MatrixBlock value2 = arg0._2(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java index dc4ca64..5d9880c 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/LocalTaskQueue.java @@ -48,7 +48,7 @@ public class LocalTaskQueue<T> public LocalTaskQueue() { - _data = new LinkedList<T>(); + _data = new LinkedList<>(); _closedInput = false; } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java index 4001d74..0effc7f 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ProgramConverter.java @@ -214,7 +214,7 @@ public class ProgramConverter public static ArrayList<ProgramBlock> rcreateDeepCopyProgramBlocks(ArrayList<ProgramBlock> childBlocks, long pid, int IDPrefix, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean forceDeepCopy) throws DMLRuntimeException { - ArrayList<ProgramBlock> tmp = new ArrayList<ProgramBlock>(); + ArrayList<ProgramBlock> tmp = new ArrayList<>(); for( ProgramBlock pb : childBlocks ) { @@ -380,8 +380,8 @@ public class ProgramConverter //create deep copy FunctionProgramBlock copy = null; - ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>(); - ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>(); + ArrayList<DataIdentifier> tmp1 = new ArrayList<>(); + ArrayList<DataIdentifier> tmp2 = new ArrayList<>(); if( fpb.getInputParams()!= null ) tmp1.addAll(fpb.getInputParams()); if( fpb.getOutputParams()!= null ) @@ -435,8 +435,8 @@ public class ProgramConverter //create deep copy FunctionProgramBlock copy = null; - ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>(); - ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>(); + ArrayList<DataIdentifier> tmp1 = new ArrayList<>(); + ArrayList<DataIdentifier> tmp2 = new ArrayList<>(); if( fpb.getInputParams()!= null ) tmp1.addAll(fpb.getInputParams()); if( fpb.getOutputParams()!= null ) @@ -472,7 +472,7 @@ public class ProgramConverter public static ArrayList<Instruction> createDeepCopyInstructionSet(ArrayList<Instruction> instSet, long pid, int IDPrefix, Program prog, HashSet<String> fnStack, HashSet<String> fnCreated, boolean plain, boolean cpFunctions) throws DMLRuntimeException { - ArrayList<Instruction> tmp = new ArrayList<Instruction>(); + ArrayList<Instruction> tmp = new ArrayList<>(); for( Instruction inst : instSet ) { if( inst instanceof FunctionCallCPInstruction && cpFunctions ) @@ -795,7 +795,7 @@ public class ProgramConverter //but in order to avoid redundancy, we only serialize function program blocks HashMap<String, FunctionProgramBlock> fpb = prog.getFunctionProgramBlocks(); - HashSet<String> cand = new HashSet<String>(); + HashSet<String> cand = new HashSet<>(); rFindSerializationCandidates(pbs, cand); return rSerializeFunctionProgramBlocks( fpb, cand, clsMap ); @@ -1381,7 +1381,7 @@ public class ProgramConverter private static HashMap<String,FunctionProgramBlock> parseFunctionProgramBlocks( String in, Program prog, int id ) throws DMLRuntimeException { - HashMap<String,FunctionProgramBlock> ret = new HashMap<String, FunctionProgramBlock>(); + HashMap<String,FunctionProgramBlock> ret = new HashMap<>(); HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer( in, ELEMENT_DELIM ); while( st.hasMoreTokens() ) @@ -1401,7 +1401,7 @@ public class ProgramConverter private static ArrayList<ProgramBlock> rParseProgramBlocks(String in, Program prog, int id) throws DMLRuntimeException { - ArrayList<ProgramBlock> pbs = new ArrayList<ProgramBlock>(); + ArrayList<ProgramBlock> pbs = new ArrayList<>(); String tmpdata = in.substring(PARFOR_PBS_BEGIN.length(),in.length()-PARFOR_PBS_END.length()); HierarchyAwareStringTokenizer st = new HierarchyAwareStringTokenizer(tmpdata, ELEMENT_DELIM); @@ -1564,8 +1564,8 @@ public class ProgramConverter //program blocks ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id); - ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>(dat1); - ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>(dat2); + ArrayList<DataIdentifier> tmp1 = new ArrayList<>(dat1); + ArrayList<DataIdentifier> tmp2 = new ArrayList<>(dat2); FunctionProgramBlock fpb = new FunctionProgramBlock(prog, tmp1, tmp2); fpb.setInstructions(inst); fpb.setChildBlocks(pbs); @@ -1595,8 +1595,8 @@ public class ProgramConverter //program blocks ArrayList<ProgramBlock> pbs = rParseProgramBlocks(st.nextToken(), prog, id); - ArrayList<DataIdentifier> tmp1 = new ArrayList<DataIdentifier>(dat1); - ArrayList<DataIdentifier> tmp2 = new ArrayList<DataIdentifier>(dat2); + ArrayList<DataIdentifier> tmp1 = new ArrayList<>(dat1); + ArrayList<DataIdentifier> tmp2 = new ArrayList<>(dat2); //only CP external functions, because no nested MR jobs for reblocks ExternalFunctionProgramBlockCP efpb = new ExternalFunctionProgramBlockCP(prog, tmp1, tmp2, dat3, basedir); @@ -1622,49 +1622,40 @@ public class ProgramConverter private static ArrayList<Instruction> parseInstructions( String in, int id ) throws DMLRuntimeException { - ArrayList<Instruction> insts = new ArrayList<Instruction>(); - + ArrayList<Instruction> insts = new ArrayList<>(); String lin = in.substring( PARFOR_INST_BEGIN.length(),in.length()-PARFOR_INST_END.length()); StringTokenizer st = new StringTokenizer(lin, ELEMENT_DELIM); - while(st.hasMoreTokens()) - { + while(st.hasMoreTokens()) { //Note that at this point only CP instructions and External function instruction can occur String instStr = st.nextToken(); - - try - { + try { Instruction tmpinst = CPInstructionParser.parseSingleInstruction(instStr); tmpinst = saveReplaceThreadID(tmpinst, CP_ROOT_THREAD_ID, CP_CHILD_THREAD+id ); insts.add( tmpinst ); } - catch(Exception ex) - { + catch(Exception ex) { throw new DMLRuntimeException("Failed to parse instruction: " + instStr, ex); } } - return insts; } - private static HashMap<String,String> parseStringHashMap( String in ) - { - HashMap<String,String> vars = new HashMap<String, String>(); + private static HashMap<String,String> parseStringHashMap( String in ) { + HashMap<String,String> vars = new HashMap<>(); StringTokenizer st = new StringTokenizer(in,ELEMENT_DELIM); - while( st.hasMoreTokens() ) - { + while( st.hasMoreTokens() ) { String lin = st.nextToken(); int index = lin.indexOf( KEY_VALUE_DELIM ); String tmp1 = lin.substring(0, index); - String tmp2 = lin.substring(index + 1); + String tmp2 = lin.substring(index + 1); vars.put(tmp1, tmp2); } - return vars; } private static ArrayList<String> parseStringArrayList( String in ) { - ArrayList<String> vars = new ArrayList<String>(); + ArrayList<String> vars = new ArrayList<>(); StringTokenizer st = new StringTokenizer(in,ELEMENT_DELIM); while( st.hasMoreTokens() ) { String tmp = st.nextToken(); @@ -1676,7 +1667,7 @@ public class ProgramConverter private static ArrayList<DataIdentifier> parseDataIdentifiers( String in ) { - ArrayList<DataIdentifier> vars = new ArrayList<DataIdentifier>(); + ArrayList<DataIdentifier> vars = new ArrayList<>(); StringTokenizer st = new StringTokenizer(in, ELEMENT_DELIM); while( st.hasMoreTokens() ) { String tmp = st.nextToken(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java index 2bccdba..800cdb4 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForMR.java @@ -244,7 +244,7 @@ public class RemoteDPParForMR public static LocalVariableMap [] readResultFile( JobConf job, String fname ) throws DMLRuntimeException, IOException { - HashMap<Long,LocalVariableMap> tmp = new HashMap<Long,LocalVariableMap>(); + HashMap<Long,LocalVariableMap> tmp = new HashMap<>(); Path path = new Path(fname); FileSystem fs = IOUtilFunctions.getFileSystem(path, job); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java index 3246cb7..4562b06 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSpark.java @@ -197,7 +197,7 @@ public class RemoteDPParForSpark @Override public Tuple2<Long, Iterable<Writable>> call(Tuple2<Long, Writable> arg0) throws Exception { - return new Tuple2<Long, Iterable<Writable>>(arg0._1(), Collections.singletonList(arg0._2())); + return new Tuple2<>(arg0._1(), Collections.singletonList(arg0._2())); } } @@ -247,9 +247,7 @@ public class RemoteDPParForSpark mb.appendValue(0, j-off, UtilFunctions.getDouble(row.get(j))); } mb.examSparsity(); - - return new Tuple2<Long, Writable>(rowix, - new PairWritableBlock(new MatrixIndexes(1,1),mb)); + return new Tuple2<>(rowix, new PairWritableBlock(new MatrixIndexes(1,1),mb)); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java index dbc3fbf..367cc8b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteDPParForSparkWorker.java @@ -99,7 +99,7 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF public Iterator<Tuple2<Long, String>> call(Iterator<Tuple2<Long, Iterable<Writable>>> arg0) throws Exception { - ArrayList<Tuple2<Long,String>> ret = new ArrayList<Tuple2<Long,String>>(); + ArrayList<Tuple2<Long,String>> ret = new ArrayList<>(); //lazy parworker initialization configureWorker( TaskContext.get().taskAttemptId() ); @@ -119,15 +119,15 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF //update in-memory matrix partition MatrixObject mo = _ec.getMatrixObject( _inputVar ); mo.setInMemoryPartition( partition ); - + //create tasks for input data Task lTask = new Task(TaskType.SET); lTask.addIteration( new IntObject(_iterVar, larg._1()) ); - + //execute program long numIter = getExecutedIterations(); super.executeTask( lTask ); - + //maintain accumulators _aTasks.add( 1 ); _aIters.add( (int)(getExecutedIterations()-numIter) ); @@ -136,7 +136,7 @@ public class RemoteDPParForSparkWorker extends ParWorker implements PairFlatMapF //write output if required (matrix indexed write) ArrayList<String> tmp = RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars ); for( String val : tmp ) - ret.add(new Tuple2<Long,String>(_workerID, val)); + ret.add(new Tuple2<>(_workerID, val)); return ret.iterator(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java index 02eb309..410fe86 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForColocatedFileSplit.java @@ -48,7 +48,6 @@ import org.apache.sysml.runtime.io.IOUtilFunctions; */ public class RemoteParForColocatedFileSplit extends FileSplit { - private String _fname = null; private int _blen = 1; @@ -57,9 +56,8 @@ public class RemoteParForColocatedFileSplit extends FileSplit * via reflection (since private not inherited from FileSplit). */ @SuppressWarnings("unused") - private RemoteParForColocatedFileSplit() - { - super( null, -1, -1, new String[]{} ); + private RemoteParForColocatedFileSplit() { + super( null, -1, -1, new String[]{} ); } public RemoteParForColocatedFileSplit( FileSplit split, String fname, int blen ) @@ -99,7 +97,7 @@ public class RemoteParForColocatedFileSplit extends FileSplit Task t = Task.parseCompactString( value.toString() ); //get all locations - HashMap<String, Integer> hosts = new HashMap<String,Integer>(); + HashMap<String, Integer> hosts = new HashMap<>(); if( t.getType() == TaskType.SET ) { @@ -132,10 +130,8 @@ public class RemoteParForColocatedFileSplit extends FileSplit return getTopHosts(hosts); } - private static void countHosts( HashMap<String,Integer> hosts, String[] names ) - { - for( String name : names ) - { + private static void countHosts( HashMap<String,Integer> hosts, String[] names ) { + for( String name : names ) { Integer tmp = hosts.get(name); if( tmp != null ) hosts.put(name, tmp+1); @@ -144,11 +140,9 @@ public class RemoteParForColocatedFileSplit extends FileSplit } } - private static String[] getTopHosts( HashMap<String,Integer> hosts ) - { + private static String[] getTopHosts( HashMap<String,Integer> hosts ) { int max = Integer.MIN_VALUE; - HashSet<String> maxName = new HashSet<String>(); - + HashSet<String> maxName = new HashSet<>(); for( Entry<String,Integer> e : hosts.entrySet() ) if( e.getValue() > max ) { maxName.clear(); @@ -157,8 +151,6 @@ public class RemoteParForColocatedFileSplit extends FileSplit } else if( e.getValue() == max ) maxName.add(e.getKey()); - - //System.out.println("HOSTS: "+ProgramConverter.serializeStringHashSet(maxName)); return maxName.toArray(new String[0]); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java index 1a3b8c3..1994628 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForMR.java @@ -255,7 +255,7 @@ public class RemoteParForMR public static LocalVariableMap [] readResultFile( JobConf job, String fname ) throws DMLRuntimeException, IOException { - HashMap<Long,LocalVariableMap> tmp = new HashMap<Long,LocalVariableMap>(); + HashMap<Long,LocalVariableMap> tmp = new HashMap<>(); Path path = new Path(fname); FileSystem fs = IOUtilFunctions.getFileSystem(path, job); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java index ec61cee..614f946 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForSparkWorker.java @@ -86,10 +86,10 @@ public class RemoteParForSparkWorker extends ParWorker implements PairFlatMapFun //write output if required (matrix indexed write) //note: this copy is necessary for environments without spark libraries - ArrayList<Tuple2<Long,String>> ret = new ArrayList<Tuple2<Long,String>>(); + ArrayList<Tuple2<Long,String>> ret = new ArrayList<>(); ArrayList<String> tmp = RemoteParForUtils.exportResultVariables( _workerID, _ec.getVariables(), _resultVars ); for( String val : tmp ) - ret.add(new Tuple2<Long,String>(_workerID, val)); + ret.add(new Tuple2<>(_workerID, val)); return ret.iterator(); } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java index fd99429..30c57b8 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParForUtils.java @@ -164,7 +164,7 @@ public class RemoteParForUtils public static ArrayList<String> exportResultVariables( long workerID, LocalVariableMap vars, ArrayList<String> resultVars) throws DMLRuntimeException, IOException { - ArrayList<String> ret = new ArrayList<String>(); + ArrayList<String> ret = new ArrayList<>(); //foreach result variables probe if export necessary for( String rvar : resultVars ) @@ -225,7 +225,7 @@ public class RemoteParForUtils public static LocalVariableMap[] getResults( List<Tuple2<Long,String>> out, Log LOG ) throws DMLRuntimeException { - HashMap<Long,LocalVariableMap> tmp = new HashMap<Long,LocalVariableMap>(); + HashMap<Long,LocalVariableMap> tmp = new HashMap<>(); int countAll = 0; for( Tuple2<Long,String> entry : out ) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java index 6ce5d18..730adbd 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/RemoteParWorkerMapper.java @@ -63,10 +63,9 @@ public class RemoteParWorkerMapper extends ParWorker //MapReduceBase not requir protected String _stringID = null; protected HashMap<String, String> _rvarFnames = null; - static - { + static { //init cache (once per JVM) - _sCache = new HashMap<String, RemoteParWorkerMapper>(); + _sCache = new HashMap<>(); } @@ -74,7 +73,7 @@ public class RemoteParWorkerMapper extends ParWorker //MapReduceBase not requir { //only used if JVM reuse is enabled in order to ensure consistent output //filenames across tasks of one reused worker (preaggregation) - _rvarFnames = new HashMap<String, String>(); + _rvarFnames = new HashMap<>(); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java index 2b10a86..86c60dd 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMerge.java @@ -19,7 +19,7 @@ package org.apache.sysml.runtime.controlprogram.parfor; -import java.util.ArrayList; +import java.util.List; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -149,27 +149,17 @@ public abstract class ResultMerge } } - protected long computeNonZeros( MatrixObject out, ArrayList<MatrixObject> in ) + protected long computeNonZeros( MatrixObject out, List<MatrixObject> in ) { MatrixCharacteristics mc = out.getMatrixCharacteristics(); long outNNZ = mc.getNonZeros(); long ret = outNNZ; - for( MatrixObject tmp : in ) - { + for( MatrixObject tmp : in ) { MatrixCharacteristics tmpmc = tmp.getMatrixCharacteristics(); long inNNZ = tmpmc.getNonZeros(); - ret += (inNNZ - outNNZ); + ret += (inNNZ - outNNZ); } return ret; } - - protected ArrayList<MatrixObject> convertToList(MatrixObject[] in) - { - ArrayList<MatrixObject> ret = new ArrayList<MatrixObject>(); - for( MatrixObject mo : in ) - ret.add( mo ); - - return ret; - } } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java index 1e7efb6..638a845 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeLocalMemory.java @@ -151,9 +151,8 @@ public class ResultMergeLocalMemory extends ResultMerge { //get matrix blocks through caching MatrixBlock outMB = _output.acquireRead(); - ArrayList<MatrixObject> inMO = new ArrayList<MatrixObject>(); - for( MatrixObject in : _inputs ) - { + ArrayList<MatrixObject> inMO = new ArrayList<>(); + for( MatrixObject in : _inputs ) { //check for empty inputs (no iterations executed) if( in !=null && in != _output ) inMO.add( in ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java index e2377c4..909d0d0 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteMR.java @@ -97,26 +97,20 @@ public class ResultMergeRemoteMR extends ResultMerge throws DMLRuntimeException { MatrixObject moNew = null; //always create new matrix object (required for nested parallelism) - - //Timing time = null; - LOG.trace("ResultMerge (remote, mr): Execute serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+")"); - // time = new Timing(); - // time.start(); - + if( LOG.isTraceEnabled() ) + LOG.trace("ResultMerge (remote, mr): Execute serial merge for output " + +_output.getVarName()+" (fname="+_output.getFileName()+")"); try { //collect all relevant inputs - Collection<String> srcFnames = new LinkedList<String>(); - ArrayList<MatrixObject> inMO = new ArrayList<MatrixObject>(); - for( MatrixObject in : _inputs ) - { + Collection<String> srcFnames = new LinkedList<>(); + ArrayList<MatrixObject> inMO = new ArrayList<>(); + for( MatrixObject in : _inputs ) { //check for empty inputs (no iterations executed) - if( in !=null && in != _output ) - { + if( in !=null && in != _output ) { //ensure that input file resides on disk in.exportData(); - //add to merge list srcFnames.add( in.getFileName() ); inMO.add(in); @@ -166,7 +160,7 @@ public class ResultMergeRemoteMR extends ResultMerge //LOG.trace("ResultMerge (local, file): Executed serial merge for output "+_output.getVarName()+" (fname="+_output.getFileName()+") in "+time.stop()+"ms"); - return moNew; + return moNew; } @SuppressWarnings({ "unused", "deprecation" }) @@ -316,12 +310,11 @@ public class ResultMergeRemoteMR extends ResultMerge catch(Exception ex) { throw new DMLRuntimeException(ex); - } + } if( DMLScript.STATISTICS ){ long t1 = System.nanoTime(); Statistics.maintainCPHeavyHitters("MR-Job_"+jobname, t1-t0); } } - } http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java index 2314845..7d59230 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteReducer.java @@ -132,7 +132,7 @@ public class ResultMergeRemoteReducer //scan for compare object (incl result merge if compare available) MatrixIndexes key2 = (MatrixIndexes) key; Double cellCompare = null; - Collection<Double> cellList = new LinkedList<Double>(); + Collection<Double> cellList = new LinkedList<>(); boolean found = false; while( valueList.hasNext() ) { TaggedMatrixCell tVal = (TaggedMatrixCell) valueList.next(); @@ -228,7 +228,7 @@ public class ResultMergeRemoteReducer //scan for compare object (incl result merge if compare available) Double cellCompare = null; - Collection<Double> cellList = new LinkedList<Double>(); + Collection<Double> cellList = new LinkedList<>(); boolean found = false; while( valueList.hasNext() ) { TaggedMatrixCell tVal = (TaggedMatrixCell) valueList.next(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java index 6ff8d65..95dbb8b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSpark.java @@ -20,6 +20,8 @@ package org.apache.sysml.runtime.controlprogram.parfor; +import java.util.Arrays; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileInputFormat; import org.apache.hadoop.mapred.JobConf; @@ -101,7 +103,7 @@ public class ResultMergeRemoteSpark extends ResultMerge InputInfo iiOld = metadata.getInputInfo(); MatrixCharacteristics mc = new MatrixCharacteristics(mcOld.getRows(),mcOld.getCols(), mcOld.getRowsPerBlock(),mcOld.getColsPerBlock()); - mc.setNonZeros( computeNonZeros(_output, convertToList(_inputs)) ); + mc.setNonZeros( computeNonZeros(_output, Arrays.asList(_inputs)) ); MatrixFormatMetaData meta = new MatrixFormatMetaData(mc,oiOld,iiOld); moNew.setMetaData( meta ); moNew.setRDDHandle( ro ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java index 3fe6a50..007f1ac 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/ResultMergeRemoteSparkWCompare.java @@ -53,7 +53,7 @@ public class ResultMergeRemoteSparkWCompare extends ResultMerge implements PairF mergeWithComp(out, din.next(), compare); //create output tuple - return new Tuple2<MatrixIndexes,MatrixBlock>(new MatrixIndexes(ixin), out); + return new Tuple2<>(new MatrixIndexes(ixin), out); } @Override http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java index de6a9d0..7edb3f2 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/Task.java @@ -54,42 +54,33 @@ public class Task implements Serializable //default constructor for serialize } - public Task( TaskType type ) - { + public Task( TaskType type ) { _type = type; - - _iterations = new LinkedList<IntObject>(); + _iterations = new LinkedList<>(); } - public void addIteration( IntObject indexVal ) - { + public void addIteration( IntObject indexVal ) { if( indexVal.getName().length() > MAX_VARNAME_SIZE ) throw new RuntimeException("Cannot add iteration, MAX_VARNAME_SIZE exceeded."); - if( size() >= MAX_TASK_SIZE ) throw new RuntimeException("Cannot add iteration, MAX_TASK_SIZE reached."); - _iterations.addLast( indexVal ); } - public List<IntObject> getIterations() - { + public List<IntObject> getIterations() { return _iterations; } - public TaskType getType() - { + public TaskType getType() { return _type; } - public int size() - { + public int size() { return _iterations.size(); } @Override - public String toString() - { + public String toString() { return toFormatedString(); } @@ -176,7 +167,7 @@ public class Task implements Serializable public static Task parseCompactString( String stask ) { - StringTokenizer st = new StringTokenizer( stask.trim(), "." ); + StringTokenizer st = new StringTokenizer( stask.trim(), "." ); Task newTask = new Task( TaskType.valueOf(st.nextToken()) ); String meta = st.nextToken(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java index 7dd25bf..4a00037 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFactoring.java @@ -55,12 +55,10 @@ public class TaskPartitionerFactoring extends TaskPartitioner public List<Task> createTasks() throws DMLRuntimeException { - LinkedList<Task> tasks = new LinkedList<Task>(); - + LinkedList<Task> tasks = new LinkedList<>(); long lFrom = _fromVal.getLongValue(); long lTo = _toVal.getLongValue(); long lIncr = _incrVal.getLongValue(); - int P = _numThreads; // number of parallel workers long N = _numIter; // total number of iterations long R = N; // remaining number of iterations http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java index f5455f5..d69837f 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/TaskPartitionerFixedsize.java @@ -35,7 +35,6 @@ import org.apache.sysml.runtime.instructions.cp.IntObject; */ public class TaskPartitionerFixedsize extends TaskPartitioner { - protected int _firstnPlus1 = 0; //add one to these firstn tasks public TaskPartitionerFixedsize( long taskSize, String iterVarName, IntObject fromVal, IntObject toVal, IntObject incrVal ) @@ -47,7 +46,7 @@ public class TaskPartitionerFixedsize extends TaskPartitioner public List<Task> createTasks() throws DMLRuntimeException { - LinkedList<Task> tasks = new LinkedList<Task>(); + LinkedList<Task> tasks = new LinkedList<>(); //range tasks (similar to run-length encoding) make only sense if taskSize>3 TaskType type = (ParForProgramBlock.USE_RANGE_TASKS_IF_USEFUL && _taskSize>3 ) ? http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java index 0021667..93e428d 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/MergedMRJobInstruction.java @@ -41,9 +41,9 @@ public class MergedMRJobInstruction public MergedMRJobInstruction() { - ids = new LinkedList<Long>(); - outIxOffs = new HashMap<Long,Integer>(); - outIxLens = new HashMap<Long,Integer>(); + ids = new LinkedList<>(); + outIxOffs = new HashMap<>(); + outIxLens = new HashMap<>(); } public void addInstructionMetaData(long instID, int outIxOffset, int outIxLen) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java index fd3a4fc..73ae779 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorker.java @@ -41,7 +41,7 @@ public abstract class PiggybackingWorker extends Thread protected PiggybackingWorker() { - _results = new HashMap<Long, JobReturn>(); + _results = new HashMap<>(); _stop = false; } @@ -80,7 +80,7 @@ public abstract class PiggybackingWorker extends Thread protected LinkedList<MergedMRJobInstruction> mergeMRJobInstructions( LinkedList<Pair<Long,MRJobInstruction>> workingSet ) throws IllegalAccessException { - LinkedList<MergedMRJobInstruction> ret = new LinkedList<MergedMRJobInstruction>(); + LinkedList<MergedMRJobInstruction> ret = new LinkedList<>(); Timing time = new Timing(true); //NOTE currently all merged into one (might be invalid due to memory constraints) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java index 98fae95..33ce803 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerTimeSequential.java @@ -85,7 +85,7 @@ public class PiggybackingWorkerTimeSequential extends PiggybackingWorker LOG.error("Failed to run merged mr-job instruction:\n"+minst.inst.toString()); // split job return - LinkedList<JobReturn> ret = new LinkedList<JobReturn>(); + LinkedList<JobReturn> ret = new LinkedList<>(); for( Long id : minst.ids ){ ret.add( minst.constructJobReturn(id, mret) ); Statistics.decrementNoOfExecutedMRJobs(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java index 2ae8062..26f17ab 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilDecayParallel.java @@ -149,7 +149,7 @@ public class PiggybackingWorkerUtilDecayParallel extends PiggybackingWorker LOG.error("Failed to run merged mr-job instruction:\n"+_minst.inst.toString()); // split job return - LinkedList<JobReturn> ret = new LinkedList<JobReturn>(); + LinkedList<JobReturn> ret = new LinkedList<>(); for( Long id : _minst.ids ){ ret.add( _minst.constructJobReturn(id, mret) ); Statistics.decrementNoOfExecutedMRJobs(); @@ -163,7 +163,7 @@ public class PiggybackingWorkerUtilDecayParallel extends PiggybackingWorker //handle unsuccessful job returns for failed job //(otherwise clients would literally wait forever for results) - LinkedList<JobReturn> ret = new LinkedList<JobReturn>(); + LinkedList<JobReturn> ret = new LinkedList<>(); for( Long id : _minst.ids ){ JobReturn fret = new JobReturn(new MatrixCharacteristics[_minst.outIxLens.get(id)], false); ret.add( _minst.constructJobReturn(id, fret) ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java index ddb33ba..25c45d8 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/PiggybackingWorkerUtilTimeParallel.java @@ -148,7 +148,7 @@ public class PiggybackingWorkerUtilTimeParallel extends PiggybackingWorker LOG.error("Failed to run merged mr-job instruction:\n"+_minst.inst.toString()); // split job return - LinkedList<JobReturn> ret = new LinkedList<JobReturn>(); + LinkedList<JobReturn> ret = new LinkedList<>(); for( Long id : _minst.ids ){ ret.add( _minst.constructJobReturn(id, mret) ); Statistics.decrementNoOfExecutedMRJobs(); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java index 66c882e..bab457b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/mqo/RuntimePiggybacking.java @@ -55,8 +55,8 @@ public class RuntimePiggybacking static { //initialize mr-job instruction pool - _pool = new HashMap<JobType, LinkedList<Long>>(); - _jobs = new HashMap<Long, MRJobInstruction>(); + _pool = new HashMap<>(); + _jobs = new HashMap<>(); //init id sequence _idSeq = new IDSequence(); @@ -200,10 +200,10 @@ public class RuntimePiggybacking return null; //create working set and remove from pool - ret = new LinkedList<Pair<Long,MRJobInstruction>>(); + ret = new LinkedList<>(); LinkedList<Long> tmp = _pool.remove(currType); for( Long id : tmp ) - ret.add( new Pair<Long, MRJobInstruction>(id,_jobs.get(id)) ); + ret.add( new Pair<>(id,_jobs.get(id)) ); } return ret; http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java index 2464bf6..1707d3e 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptNode.java @@ -22,7 +22,6 @@ package org.apache.sysml.runtime.controlprogram.parfor.opt; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.LinkedList; import java.util.Set; import org.apache.commons.lang.ArrayUtils; @@ -143,7 +142,7 @@ public class OptNode public void addParam(ParamType ptype, String val) { if( _params == null ) - _params = new HashMap<ParamType, String>(); + _params = new HashMap<>(); _params.put(ptype, val); } @@ -179,13 +178,13 @@ public class OptNode public void addChild( OptNode child ) { if( _childs==null ) - _childs = new ArrayList<OptNode>(); + _childs = new ArrayList<>(); _childs.add( child ); } public void addChilds( ArrayList<OptNode> childs ) { if( _childs==null ) - _childs = new ArrayList<OptNode>(); + _childs = new ArrayList<>(); _childs.addAll( childs ); } @@ -260,7 +259,7 @@ public class OptNode //recursive methods public Collection<OptNode> getNodeList() { - Collection<OptNode> nodes = new LinkedList<OptNode>(); + Collection<OptNode> nodes = new ArrayList<>(); if(!isLeaf()) for( OptNode n : _childs ) nodes.addAll( n.getNodeList() ); @@ -269,7 +268,7 @@ public class OptNode } public Collection<OptNode> getNodeList( ExecType et ) { - Collection<OptNode> nodes = new LinkedList<OptNode>(); + Collection<OptNode> nodes = new ArrayList<>(); if(!isLeaf()) for( OptNode n : _childs ) nodes.addAll( n.getNodeList( et ) ); @@ -279,7 +278,7 @@ public class OptNode } public Collection<OptNode> getRelevantNodeList() { - Collection<OptNode> nodes = new LinkedList<OptNode>(); + Collection<OptNode> nodes = new ArrayList<>(); if( !isLeaf() ) for( OptNode n : _childs ) nodes.addAll( n.getRelevantNodeList() ); http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java index 5676869..d5b990b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreeConverter.java @@ -96,7 +96,7 @@ public class OptTreeConverter { case ABSTRACT_PLAN: _hlMap.putRootProgram(pfsb.getDMLProg(), pfpb.getProgram()); - Set<String> memo = new HashSet<String>(); + Set<String> memo = new HashSet<>(); root = rCreateAbstractOptNode(pfsb, pfpb, ec.getVariables(), true, memo); root.checkAndCleanupRecursiveFunc(new HashSet<String>()); //create consistency between recursive info root.checkAndCleanupLeafNodes(); //prune unnecessary nodes @@ -249,7 +249,7 @@ public class OptTreeConverter public static ArrayList<OptNode> createOptNodes (ArrayList<Instruction> instset, LocalVariableMap vars, boolean storeObjs) throws DMLRuntimeException { - ArrayList<OptNode> tmp = new ArrayList<OptNode>(instset.size()); + ArrayList<OptNode> tmp = new ArrayList<>(instset.size()); for( Instruction inst : instset ) tmp.add( createOptNode(inst,vars,storeObjs) ); return tmp; @@ -481,7 +481,7 @@ public class OptTreeConverter public static ArrayList<OptNode> createAbstractOptNodes(ArrayList<Hop> hops, LocalVariableMap vars, Set<String> memo ) throws DMLRuntimeException, HopsException { - ArrayList<OptNode> ret = new ArrayList<OptNode>(); + ArrayList<OptNode> ret = new ArrayList<>(); //reset all hops Hop.resetVisitStatus(hops); @@ -497,7 +497,7 @@ public class OptTreeConverter public static ArrayList<OptNode> rCreateAbstractOptNodes(Hop hop, LocalVariableMap vars, Set<String> memo) throws DMLRuntimeException, HopsException { - ArrayList<OptNode> ret = new ArrayList<OptNode>(); + ArrayList<OptNode> ret = new ArrayList<>(); ArrayList<Hop> in = hop.getInput(); if( hop.isVisited() ) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java index 2342b76..597b7c0 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanChecker.java @@ -152,7 +152,7 @@ public class OptTreePlanChecker root.resetVisitStatus(); //get all function op in this dag - HashMap<String, FunctionOp> fops = new HashMap<String, FunctionOp>(); + HashMap<String, FunctionOp> fops = new HashMap<>(); getAllFunctionOps(root, fops); for( Instruction linst : inst ) http://git-wip-us.apache.org/repos/asf/systemml/blob/e106966a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java index c1d1f88..4f9bd69 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptTreePlanMapping.java @@ -38,7 +38,7 @@ public class OptTreePlanMapping public OptTreePlanMapping() { _idSeq = new IDSequence(); - _id_optnode = new HashMap<Long, OptNode>(); + _id_optnode = new HashMap<>(); } public OptNode getOptNode( long id )
