Repository: incubator-systemml Updated Branches: refs/heads/master 795a94e7b -> 39f75ca06
[SYSTEMML-1425] Robustness parfor parameter handling (cmd args, quotes) Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/128acb3a Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/128acb3a Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/128acb3a Branch: refs/heads/master Commit: 128acb3adccfa03fa5e91495349d2e95c5cf0f2e Parents: 795a94e Author: Matthias Boehm <[email protected]> Authored: Mon Mar 20 12:24:28 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Mar 20 12:24:28 2017 -0700 ---------------------------------------------------------------------- .../sysml/parser/dml/DmlSyntacticValidator.java | 5 +- .../parser/pydml/PydmlSyntacticValidator.java | 5 +- .../runtime/controlprogram/ForProgramBlock.java | 3 +- .../controlprogram/FunctionProgramBlock.java | 2 +- .../runtime/controlprogram/IfProgramBlock.java | 2 +- .../controlprogram/ParForProgramBlock.java | 105 ++++++++----------- .../runtime/controlprogram/ProgramBlock.java | 4 +- .../controlprogram/WhileProgramBlock.java | 2 +- 8 files changed, 55 insertions(+), 73 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java index 8ff918d..2ae6fd8 100644 --- a/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/dml/DmlSyntacticValidator.java @@ -685,7 +685,10 @@ public class DmlSyntacticValidator extends CommonSyntacticValidator implements D HashMap<String, String> parForParamValues = new HashMap<String, String>(); if(ctx.parForParams != null && ctx.parForParams.size() > 0) { for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) { - parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText()); + String paramVal = parForParamCtx.paramVal.getText(); + if( argVals.containsKey(paramVal) ) + paramVal = argVals.get(paramVal); + parForParamValues.put(parForParamCtx.paramName.getText(), paramVal); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java index cc886cd..4c9b416 100644 --- a/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java +++ b/src/main/java/org/apache/sysml/parser/pydml/PydmlSyntacticValidator.java @@ -1353,7 +1353,10 @@ public class PydmlSyntacticValidator extends CommonSyntacticValidator implements HashMap<String, String> parForParamValues = new HashMap<String, String>(); if(ctx.parForParams != null && ctx.parForParams.size() > 0) { for(StrictParameterizedExpressionContext parForParamCtx : ctx.parForParams) { - parForParamValues.put(parForParamCtx.paramName.getText(), parForParamCtx.paramVal.getText()); + String paramVal = parForParamCtx.paramVal.getText(); + if( argVals.containsKey(paramVal) ) + paramVal = argVals.get(paramVal); + parForParamValues.put(parForParamCtx.paramName.getText(), paramVal); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java index 103ce70..4a529a1 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ForProgramBlock.java @@ -49,8 +49,7 @@ public class ForProgramBlock extends ProgramBlock protected String[] _iterablePredicateVars; //from,to,where constants/internal vars not captured via instructions - public ForProgramBlock(Program prog, String[] iterPredVars) throws DMLRuntimeException - { + public ForProgramBlock(Program prog, String[] iterPredVars) { super(prog); _exitInstructions = new ArrayList<Instruction>(); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java index b2f6e53..830251e 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/FunctionProgramBlock.java @@ -42,7 +42,7 @@ public class FunctionProgramBlock extends ProgramBlock private boolean _recompileOnce = false; - public FunctionProgramBlock( Program prog, ArrayList<DataIdentifier> inputParams, ArrayList<DataIdentifier> outputParams) throws DMLRuntimeException + public FunctionProgramBlock( Program prog, ArrayList<DataIdentifier> inputParams, ArrayList<DataIdentifier> outputParams) { super(prog); _childBlocks = new ArrayList<ProgramBlock>(); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java index d065aa5..cbb5ced 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/IfProgramBlock.java @@ -50,7 +50,7 @@ public class IfProgramBlock extends ProgramBlock private ArrayList<ProgramBlock> _childBlocksIfBody; private ArrayList<ProgramBlock> _childBlocksElseBody; - public IfProgramBlock(Program prog, ArrayList<Instruction> predicate) throws DMLRuntimeException{ + public IfProgramBlock(Program prog, ArrayList<Instruction> predicate) { super(prog); _childBlocksIfBody = new ArrayList<ProgramBlock>(); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java index df50939..0e3574f 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java @@ -103,6 +103,7 @@ import org.apache.sysml.runtime.instructions.cp.StringObject; import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction; import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.matrix.data.OutputInfo; +import org.apache.sysml.runtime.util.UtilFunctions; import org.apache.sysml.utils.Statistics; import org.apache.sysml.yarn.ropt.YarnClusterAnalyzer; @@ -308,7 +309,6 @@ public class ParForProgramBlock extends ForProgramBlock * @throws DMLRuntimeException if DMLRuntimeException occurs */ public ParForProgramBlock(int ID, Program prog, String[] iterPredVars, HashMap<String,String> params) - throws DMLRuntimeException { super(prog, iterPredVars); @@ -322,21 +322,18 @@ public class ParForProgramBlock extends ForProgramBlock //parse and use internal parameters (already set to default if not specified) _params = params; - try - { - _numThreads = Integer.parseInt( _params.get(ParForStatementBlock.PAR) ); - _taskPartitioner = PTaskPartitioner.valueOf( _params.get(ParForStatementBlock.TASK_PARTITIONER).toUpperCase() ); - _taskSize = Integer.parseInt( _params.get(ParForStatementBlock.TASK_SIZE) ); - _dataPartitioner = PDataPartitioner.valueOf( _params.get(ParForStatementBlock.DATA_PARTITIONER).toUpperCase() ); - _resultMerge = PResultMerge.valueOf( _params.get(ParForStatementBlock.RESULT_MERGE).toUpperCase() ); - _execMode = PExecMode.valueOf( _params.get(ParForStatementBlock.EXEC_MODE).toUpperCase() ); - _optMode = POptMode.valueOf( _params.get(ParForStatementBlock.OPT_MODE).toUpperCase()); - _optLogLevel = Level.toLevel( _params.get(ParForStatementBlock.OPT_LOG)); - _monitor = (Integer.parseInt(_params.get(ParForStatementBlock.PROFILE) ) == 1); + try { + _numThreads = Integer.parseInt( getParForParam(ParForStatementBlock.PAR) ); + _taskPartitioner = PTaskPartitioner.valueOf( getParForParam(ParForStatementBlock.TASK_PARTITIONER) ); + _taskSize = Integer.parseInt( getParForParam(ParForStatementBlock.TASK_SIZE) ); + _dataPartitioner = PDataPartitioner.valueOf( getParForParam(ParForStatementBlock.DATA_PARTITIONER) ); + _resultMerge = PResultMerge.valueOf( getParForParam(ParForStatementBlock.RESULT_MERGE) ); + _execMode = PExecMode.valueOf( getParForParam(ParForStatementBlock.EXEC_MODE) ); + _optMode = POptMode.valueOf( getParForParam(ParForStatementBlock.OPT_MODE) ); + _optLogLevel = Level.toLevel( getParForParam(ParForStatementBlock.OPT_LOG) ); + _monitor = (Integer.parseInt(getParForParam(ParForStatementBlock.PROFILE) ) == 1); } - catch(Exception ex) - { - //runtime exception in order to keep signature of program block + catch(Exception ex) { throw new RuntimeException("Error parsing specified ParFOR parameters.",ex); } @@ -364,129 +361,113 @@ public class ParForProgramBlock extends ForProgramBlock LOG.trace("PARFOR: ParForProgramBlock created with mode = "+_execMode+", optmode = "+_optMode+", numThreads = "+_numThreads); } - public long getID() - { + public long getID() { return _ID; } - public PExecMode getExecMode() - { + public PExecMode getExecMode() { return _execMode; } - public HashMap<String,String> getParForParams() - { + public HashMap<String,String> getParForParams() { return _params; } + + public String getParForParam(String key) { + String tmp = getParForParams().get(key); + return (tmp == null) ? null : + UtilFunctions.unquote(tmp).toUpperCase(); + } public ArrayList<String> getResultVariables() { return _resultVars; } - public void setResultVariables(ArrayList<String> resultVars) - { + public void setResultVariables(ArrayList<String> resultVars) { _resultVars = resultVars; } - public void disableOptimization() - { + public void disableOptimization() { _optMode = POptMode.NONE; } - public POptMode getOptimizationMode() - { + public POptMode getOptimizationMode() { return _optMode; } - public int getDegreeOfParallelism() - { + public int getDegreeOfParallelism() { return _numThreads; } - public void setDegreeOfParallelism(int k) - { + public void setDegreeOfParallelism(int k) { _numThreads = k; _params.put(ParForStatementBlock.PAR, String.valueOf(_numThreads)); //kept up-to-date for copies setLocalParWorkerIDs(); } - public void setCPCaching(boolean flag) - { + public void setCPCaching(boolean flag) { _enableCPCaching = flag; } - public void setRuntimePiggybacking(boolean flag) - { + public void setRuntimePiggybacking(boolean flag) { _enableRuntimePiggybacking = flag; } - public void setExecMode( PExecMode mode ) - { + public void setExecMode( PExecMode mode ) { _execMode = mode; _params.put(ParForStatementBlock.EXEC_MODE, String.valueOf(_execMode)); //kept up-to-date for copies } - public void setTaskPartitioner( PTaskPartitioner partitioner ) - { + public void setTaskPartitioner( PTaskPartitioner partitioner ) { _taskPartitioner = partitioner; _params.put(ParForStatementBlock.TASK_PARTITIONER, String.valueOf(_taskPartitioner)); //kept up-to-date for copies } - public void setTaskSize( long tasksize ) - { + public void setTaskSize( long tasksize ) { _taskSize = tasksize; _params.put(ParForStatementBlock.TASK_SIZE, String.valueOf(_taskSize)); //kept up-to-date for copies } - public void setDataPartitioner(PDataPartitioner partitioner) - { + public void setDataPartitioner(PDataPartitioner partitioner) { _dataPartitioner = partitioner; _params.put(ParForStatementBlock.DATA_PARTITIONER, String.valueOf(_dataPartitioner)); //kept up-to-date for copies } - public void enableColocatedPartitionedMatrix( String varname ) - { + public void enableColocatedPartitionedMatrix( String varname ) { //only called from optimizer _colocatedDPMatrix = varname; } - public void setTransposeSparseColumnVector( boolean flag ) - { + public void setTransposeSparseColumnVector( boolean flag ) { _tSparseCol = flag; } - public void setPartitionReplicationFactor( int rep ) - { + public void setPartitionReplicationFactor( int rep ) { //only called from optimizer _replicationDP = rep; } - public void setExportReplicationFactor( int rep ) - { + public void setExportReplicationFactor( int rep ) { //only called from optimizer _replicationExport = rep; } - public void disableJVMReuse() - { + public void disableJVMReuse() { //only called from optimizer _jvmReuse = false; } - public void disableMonitorReport() - { + public void disableMonitorReport() { _monitorReport = false; } - public void setResultMerge(PResultMerge merge) - { + public void setResultMerge(PResultMerge merge) { _resultMerge = merge; _params.put(ParForStatementBlock.RESULT_MERGE, String.valueOf(_resultMerge)); //kept up-to-date for copies } - public void setRecompileMemoryBudget( double localMem ) - { + public void setRecompileMemoryBudget( double localMem ) { _recompileMemoryBudget = localMem; } @@ -502,8 +483,7 @@ public class ParForProgramBlock extends ForProgramBlock _variablesECache = vars; } - public long getNumIterations() - { + public long getNumIterations() { return _numIterations; } @@ -511,8 +491,7 @@ public class ParForProgramBlock extends ForProgramBlock return _hasFunctions; } - public static void initInternalConfigurations( DMLConfig conf ) - { + public static void initInternalConfigurations( DMLConfig conf ) { ALLOW_REUSE_MR_JVMS = conf.getBooleanValue(DMLConfig.JVM_REUSE); ALLOW_REUSE_MR_PAR_WORKER = ALLOW_REUSE_MR_JVMS; } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java index 739b1cf..1847847 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ProgramBlock.java @@ -64,9 +64,7 @@ public class ProgramBlock protected long _tid = 0; //by default _t0 - public ProgramBlock(Program prog) - throws DMLRuntimeException - { + public ProgramBlock(Program prog) { _prog = prog; _inst = new ArrayList<Instruction>(); } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/128acb3a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java index c435884..9a2b53c 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/WhileProgramBlock.java @@ -50,7 +50,7 @@ public class WhileProgramBlock extends ProgramBlock private ArrayList <Instruction> _exitInstructions ; private ArrayList<ProgramBlock> _childBlocks; - public WhileProgramBlock(Program prog, ArrayList<Instruction> predicate) throws DMLRuntimeException{ + public WhileProgramBlock(Program prog, ArrayList<Instruction> predicate) { super(prog); _predicate = predicate; _predicateResultVar = findPredicateResultVar ();
