http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java index 538103f..c0a50ec 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/GPUObject.java @@ -30,7 +30,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysml.api.DMLScript; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.controlprogram.caching.CacheException; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.runtime.instructions.cp.CPInstruction; import org.apache.sysml.runtime.instructions.gpu.GPUInstruction; @@ -619,24 +618,19 @@ public class GPUObject { * * @param instName name of the instruction * @return true if a copy to host happened, false otherwise - * @throws CacheException ? */ - public boolean acquireHostRead(String instName) throws CacheException { + public boolean acquireHostRead(String instName) throws DMLRuntimeException { boolean copied = false; - try { + if(LOG.isTraceEnabled()) { + LOG.trace("GPU : acquireDeviceModifySparse on " + this + ", GPUContext=" + getGPUContext()); + } + if (isAllocated() && dirty) { if(LOG.isTraceEnabled()) { - LOG.trace("GPU : acquireDeviceModifySparse on " + this + ", GPUContext=" + getGPUContext()); + LOG.trace("GPU : data is dirty on device, copying to host, on " + this + ", GPUContext=" + + getGPUContext()); } - if (isAllocated() && dirty) { - if(LOG.isTraceEnabled()) { - LOG.trace("GPU : data is dirty on device, copying to host, on " + this + ", GPUContext=" - + getGPUContext()); - } - copyFromDeviceToHost(instName, false); - copied = true; - } - } catch (DMLRuntimeException e) { - throw new CacheException(e); + copyFromDeviceToHost(instName, false); + copied = true; } return copied; } @@ -699,7 +693,7 @@ public class GPUObject { timestamp.set(-System.nanoTime()); break; default: - throw new CacheException("The eviction policy is not supported:" + evictionPolicy.name()); + throw new DMLRuntimeException("The eviction policy is not supported:" + evictionPolicy.name()); } } @@ -712,7 +706,7 @@ public class GPUObject { releaseReadLock(); updateReleaseLocks(); if (!isAllocated()) - throw new CacheException("Attempting to release an input before allocating it"); + throw new DMLRuntimeException("Attempting to release an input before allocating it"); } /** @@ -728,7 +722,7 @@ public class GPUObject { // Ideally, we would want to throw CacheException("Attempting to release an output that was not acquired via acquireDeviceModify") if !isDirty() dirty = true; if (!isAllocated()) - throw new CacheException("Attempting to release an output before allocating it"); + throw new DMLRuntimeException("Attempting to release an output before allocating it"); } void allocateDenseMatrixOnDevice() throws DMLRuntimeException { @@ -764,7 +758,7 @@ public class GPUObject { setSparseMatrixCudaPointer(tmp); } - void deallocateMemoryOnDevice(boolean eager) throws DMLRuntimeException { + void deallocateMemoryOnDevice(boolean eager) { if(LOG.isTraceEnabled()) { LOG.trace("GPU : deallocateMemoryOnDevice, on " + this + ", GPUContext=" + getGPUContext()); } @@ -779,7 +773,7 @@ public class GPUObject { resetReadWriteLock(); } - protected long getSizeOnDevice() throws DMLRuntimeException { + protected long getSizeOnDevice() { long GPUSize = 0; long rlen = mat.getNumRows(); long clen = mat.getNumColumns(); @@ -793,7 +787,7 @@ public class GPUObject { return GPUSize; } - void copyFromHostToDevice(String opcode) throws DMLRuntimeException { + void copyFromHostToDevice(String opcode) { if(LOG.isTraceEnabled()) { LOG.trace("GPU : copyFromHostToDevice, on " + this + ", GPUContext=" + getGPUContext()); } @@ -910,14 +904,14 @@ public class GPUObject { GPUStatistics.cudaToDevCount.add(1); } - public static int toIntExact(long l) throws DMLRuntimeException { + public static int toIntExact(long l) { if (l < Integer.MIN_VALUE || l > Integer.MAX_VALUE) { throw new DMLRuntimeException("Cannot be cast to int:" + l); } return (int) l; } - protected void copyFromDeviceToHost(String instName, boolean isEviction) throws DMLRuntimeException { + protected void copyFromDeviceToHost(String instName, boolean isEviction) { if(LOG.isTraceEnabled()) { LOG.trace("GPU : copyFromDeviceToHost, on " + this + ", GPUContext=" + getGPUContext()); } @@ -988,10 +982,8 @@ public class GPUObject { /** * lazily clears the data associated with this {@link GPUObject} instance - * - * @throws CacheException ? */ - public void clearData() throws DMLRuntimeException { + public void clearData() { clearData(DMLScript.EAGER_CUDA_FREE); } @@ -999,9 +991,8 @@ public class GPUObject { * Clears the data associated with this {@link GPUObject} instance * * @param eager whether to be done synchronously or asynchronously - * @throws CacheException ? */ - public void clearData(boolean eager) throws DMLRuntimeException { + public void clearData(boolean eager) { deallocateMemoryOnDevice(eager); getGPUContext().getMemoryManager().removeGPUObject(this); }
http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/udf/lib/CumSumProd.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/lib/CumSumProd.java b/src/main/java/org/apache/sysml/udf/lib/CumSumProd.java index e9edb0a..ba3795f 100644 --- a/src/main/java/org/apache/sysml/udf/lib/CumSumProd.java +++ b/src/main/java/org/apache/sysml/udf/lib/CumSumProd.java @@ -22,7 +22,6 @@ import java.io.IOException; import java.util.Iterator; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.controlprogram.caching.CacheException; import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -99,57 +98,53 @@ public class CumSumProd extends PackageFunction { @Override public void execute() { - try { - X = ((Matrix) getFunctionInput(0)).getMatrixObject().acquireRead(); - C = ((Matrix) getFunctionInput(1)).getMatrixObject().acquireRead(); - if(X.getNumRows() != C.getNumRows()) - throw new RuntimeException("Number of rows of X and C should match"); - if( X.getNumColumns() != C.getNumColumns() && C.getNumColumns() != 1 ) - throw new RuntimeException("Incorrect Number of columns of X and C (Expected C to be of same dimension or a vector)"); - start = Double.parseDouble(((Scalar)getFunctionInput(2)).getValue()); - isReverse = Boolean.parseBoolean(((Scalar)getFunctionInput(3)).getValue()); - - numRetRows = X.getNumRows(); - numRetCols = X.getNumColumns(); - allocateOutput(); - - // Copy X to Y - denseBlock = retMB.getDenseBlockValues(); - if(X.isInSparseFormat()) { - Iterator<IJV> iter = X.getSparseBlockIterator(); - while(iter.hasNext()) { - IJV ijv = iter.next(); - denseBlock[ijv.getI()*numRetCols + ijv.getJ()] = ijv.getV(); - } - } - else { - if(X.getDenseBlock() != null) - System.arraycopy(X.getDenseBlockValues(), 0, denseBlock, 0, denseBlock.length); + X = ((Matrix) getFunctionInput(0)).getMatrixObject().acquireRead(); + C = ((Matrix) getFunctionInput(1)).getMatrixObject().acquireRead(); + if(X.getNumRows() != C.getNumRows()) + throw new RuntimeException("Number of rows of X and C should match"); + if( X.getNumColumns() != C.getNumColumns() && C.getNumColumns() != 1 ) + throw new RuntimeException("Incorrect Number of columns of X and C (Expected C to be of same dimension or a vector)"); + start = Double.parseDouble(((Scalar)getFunctionInput(2)).getValue()); + isReverse = Boolean.parseBoolean(((Scalar)getFunctionInput(3)).getValue()); + + numRetRows = X.getNumRows(); + numRetCols = X.getNumColumns(); + allocateOutput(); + + // Copy X to Y + denseBlock = retMB.getDenseBlockValues(); + if(X.isInSparseFormat()) { + Iterator<IJV> iter = X.getSparseBlockIterator(); + while(iter.hasNext()) { + IJV ijv = iter.next(); + denseBlock[ijv.getI()*numRetCols + ijv.getJ()] = ijv.getV(); } - - if(!isReverse) { - // Y [1, ] = X [1, ] + C [1, ] * start; - // Y [i+1, ] = X [i+1, ] + C [i+1, ] * Y [i, ] - addCNConstant(0, start); - for(int i = 1; i < numRetRows; i++) { - addC(i, true); - } + } + else { + if(X.getDenseBlock() != null) + System.arraycopy(X.getDenseBlockValues(), 0, denseBlock, 0, denseBlock.length); + } + + if(!isReverse) { + // Y [1, ] = X [1, ] + C [1, ] * start; + // Y [i+1, ] = X [i+1, ] + C [i+1, ] * Y [i, ] + addCNConstant(0, start); + for(int i = 1; i < numRetRows; i++) { + addC(i, true); } - else { - // Y [m, ] = X [m, ] + C [m, ] * start; - // Y [i-1, ] = X [i-1, ] + C [i-1, ] * Y [i, ] - addCNConstant(numRetRows-1, start); - for(int i = numRetRows - 2; i >= 0; i--) { - addC(i, false); - } + } + else { + // Y [m, ] = X [m, ] + C [m, ] * start; + // Y [i-1, ] = X [i-1, ] + C [i-1, ] * Y [i, ] + addCNConstant(numRetRows-1, start); + for(int i = numRetRows - 2; i >= 0; i--) { + addC(i, false); } - - ((Matrix) getFunctionInput(1)).getMatrixObject().release(); - ((Matrix) getFunctionInput(0)).getMatrixObject().release(); - } catch (CacheException e) { - throw new RuntimeException("Error while executing CumSumProd", e); } + ((Matrix) getFunctionInput(1)).getMatrixObject().release(); + ((Matrix) getFunctionInput(0)).getMatrixObject().release(); + retMB.recomputeNonZeros(); try { retMB.examSparsity(); @@ -243,7 +238,4 @@ public class CumSumProd extends PackageFunction { retMB = new MatrixBlock((int) numRetRows, (int) numRetCols, false); retMB.allocateDenseBlock(); } - - - } http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/udf/lib/MultiInputCbind.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/lib/MultiInputCbind.java b/src/main/java/org/apache/sysml/udf/lib/MultiInputCbind.java index e56523c..e2b9f0e 100644 --- a/src/main/java/org/apache/sysml/udf/lib/MultiInputCbind.java +++ b/src/main/java/org/apache/sysml/udf/lib/MultiInputCbind.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.Iterator; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.controlprogram.caching.CacheException; import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -72,89 +71,81 @@ public class MultiInputCbind extends PackageFunction { spagetize = Boolean.parseBoolean(((Scalar)getFunctionInput(1)).getValue()); // Compute output dimensions - try { - numRetCols = 0; - if(spagetize) { - // Assumption the inputs are of same shape - MatrixBlock in = ((Matrix) getFunctionInput(2)).getMatrixObject().acquireRead(); - numRetRows = in.getNumRows()*in.getNumColumns(); - numRetCols = numInputs; - ((Matrix) getFunctionInput(2)).getMatrixObject().release(); - } - else { - for(int inputID = 2; inputID < numInputs + 2; inputID++) { - MatrixBlock in = ((Matrix) getFunctionInput(inputID)).getMatrixObject().acquireRead(); - numRetRows = in.getNumRows(); - numRetCols += in.getNumColumns(); - ((Matrix) getFunctionInput(inputID)).getMatrixObject().release(); - } + numRetCols = 0; + if(spagetize) { + // Assumption the inputs are of same shape + MatrixBlock in = ((Matrix) getFunctionInput(2)).getMatrixObject().acquireRead(); + numRetRows = in.getNumRows()*in.getNumColumns(); + numRetCols = numInputs; + ((Matrix) getFunctionInput(2)).getMatrixObject().release(); + } + else { + for(int inputID = 2; inputID < numInputs + 2; inputID++) { + MatrixBlock in = ((Matrix) getFunctionInput(inputID)).getMatrixObject().acquireRead(); + numRetRows = in.getNumRows(); + numRetCols += in.getNumColumns(); + ((Matrix) getFunctionInput(inputID)).getMatrixObject().release(); } - } catch (CacheException e) { - throw new RuntimeException("Error while executing MultiInputCbind", e); } allocateOutput(); // Performs cbind (cbind (cbind ( X1, X2 ), X3 ), X4) double [] retData = retMB.getDenseBlockValues(); - try { - int startColumn = 0; - for(int inputID = 2; inputID < numInputs + 2; inputID++) { - MatrixBlock in = ((Matrix) getFunctionInput(inputID)).getMatrixObject().acquireRead(); - if(spagetize && in.getNumRows()*in.getNumColumns() != numRetRows) { - throw new RuntimeException("Expected the inputs to be of same size when spagetization is turned on."); - } - int inputNumCols = in.getNumColumns(); - if(in.isInSparseFormat()) { - Iterator<IJV> iter = in.getSparseBlockIterator(); - while(iter.hasNext()) { - IJV ijv = iter.next(); - if(spagetize) { - // Perform matrix(X1, rows=length(X1), cols=1) operation before cbind - // Output Column ID = inputID-2 for all elements of inputs - int outputRowIndex = ijv.getI()*inputNumCols + ijv.getJ(); - int outputColIndex = inputID-2; - retData[(int) (outputRowIndex*retMB.getNumColumns() + outputColIndex)] = ijv.getV(); - } - else { - // Traditional cbind - // Row ID remains the same as that of input - int outputRowIndex = ijv.getI(); - int outputColIndex = ijv.getJ() + startColumn; - retData[(int) (outputRowIndex*retMB.getNumColumns() + outputColIndex)] = ijv.getV(); - } + int startColumn = 0; + for(int inputID = 2; inputID < numInputs + 2; inputID++) { + MatrixBlock in = ((Matrix) getFunctionInput(inputID)).getMatrixObject().acquireRead(); + if(spagetize && in.getNumRows()*in.getNumColumns() != numRetRows) { + throw new RuntimeException("Expected the inputs to be of same size when spagetization is turned on."); + } + int inputNumCols = in.getNumColumns(); + if(in.isInSparseFormat()) { + Iterator<IJV> iter = in.getSparseBlockIterator(); + while(iter.hasNext()) { + IJV ijv = iter.next(); + if(spagetize) { + // Perform matrix(X1, rows=length(X1), cols=1) operation before cbind + // Output Column ID = inputID-2 for all elements of inputs + int outputRowIndex = ijv.getI()*inputNumCols + ijv.getJ(); + int outputColIndex = inputID-2; + retData[(int) (outputRowIndex*retMB.getNumColumns() + outputColIndex)] = ijv.getV(); + } + else { + // Traditional cbind + // Row ID remains the same as that of input + int outputRowIndex = ijv.getI(); + int outputColIndex = ijv.getJ() + startColumn; + retData[(int) (outputRowIndex*retMB.getNumColumns() + outputColIndex)] = ijv.getV(); } } - else { - double [] denseBlock = in.getDenseBlockValues(); - if(denseBlock != null) { - if(spagetize) { - // Perform matrix(X1, rows=length(X1), cols=1) operation before cbind - // Output Column ID = inputID-2 for all elements of inputs - int j = inputID-2; - for(int i = 0; i < numRetRows; i++) { - retData[(int) (i*numRetCols + j)] = denseBlock[i]; - } + } + else { + double [] denseBlock = in.getDenseBlockValues(); + if(denseBlock != null) { + if(spagetize) { + // Perform matrix(X1, rows=length(X1), cols=1) operation before cbind + // Output Column ID = inputID-2 for all elements of inputs + int j = inputID-2; + for(int i = 0; i < numRetRows; i++) { + retData[(int) (i*numRetCols + j)] = denseBlock[i]; } - else { - // Traditional cbind - // Row ID remains the same as that of input - for(int i = 0; i < retMB.getNumRows(); i++) { - for(int j = 0; j < inputNumCols; j++) { - int outputColIndex = j + startColumn; - retData[(int) (i*numRetCols + outputColIndex)] = denseBlock[i*inputNumCols + j]; - } + } + else { + // Traditional cbind + // Row ID remains the same as that of input + for(int i = 0; i < retMB.getNumRows(); i++) { + for(int j = 0; j < inputNumCols; j++) { + int outputColIndex = j + startColumn; + retData[(int) (i*numRetCols + outputColIndex)] = denseBlock[i*inputNumCols + j]; } } } } - ((Matrix) getFunctionInput(inputID)).getMatrixObject().release(); - startColumn += inputNumCols; } - } catch (CacheException e) { - throw new RuntimeException("Error while executing MultiInputCbind", e); + ((Matrix) getFunctionInput(inputID)).getMatrixObject().release(); + startColumn += inputNumCols; } - + retMB.recomputeNonZeros(); try { retMB.examSparsity(); http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/udf/lib/SGDNesterovUpdate.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/lib/SGDNesterovUpdate.java b/src/main/java/org/apache/sysml/udf/lib/SGDNesterovUpdate.java index a644969..7a7f48e 100644 --- a/src/main/java/org/apache/sysml/udf/lib/SGDNesterovUpdate.java +++ b/src/main/java/org/apache/sysml/udf/lib/SGDNesterovUpdate.java @@ -23,7 +23,6 @@ import java.io.IOException; import java.util.Iterator; import java.util.Random; -import org.apache.sysml.runtime.controlprogram.caching.CacheException; import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -142,8 +141,6 @@ public class SGDNesterovUpdate extends PackageFunction { ((Matrix) getFunctionInput(0)).getMatrixObject().release(); ((Matrix) getFunctionInput(1)).getMatrixObject().release(); ((Matrix) getFunctionInput(4)).getMatrixObject().release(); - } catch (CacheException e) { - throw new RuntimeException("Exception while executing SGDNesterovUpdate", e); } catch (IOException e) { throw new RuntimeException("Exception while executing SGDNesterovUpdate", e); } http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/utils/Explain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java index e0a9c92..12f4d0f 100644 --- a/src/main/java/org/apache/sysml/utils/Explain.java +++ b/src/main/java/org/apache/sysml/utils/Explain.java @@ -31,7 +31,6 @@ import org.apache.sysml.hops.BinaryOp; import org.apache.sysml.hops.DataOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.Hop.DataOpTypes; -import org.apache.sysml.hops.HopsException; import org.apache.sysml.hops.LiteralOp; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.hops.ReorgOp; @@ -49,7 +48,6 @@ import org.apache.sysml.parser.FunctionStatement; import org.apache.sysml.parser.FunctionStatementBlock; import org.apache.sysml.parser.IfStatement; import org.apache.sysml.parser.IfStatementBlock; -import org.apache.sysml.parser.LanguageException; import org.apache.sysml.parser.ParForStatementBlock; import org.apache.sysml.parser.StatementBlock; import org.apache.sysml.parser.WhileStatement; @@ -108,8 +106,7 @@ public class Explain ////////////// // public explain interface - public static String display(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts) - throws HopsException, DMLRuntimeException, LanguageException { + public static String display(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts) { if( counts == null ) counts = countDistributedOperations(rtprog); @@ -198,16 +195,14 @@ public class Explain sb.append( rk2 ); } - return sb.toString(); + return sb.toString(); } - public static String explain(DMLProgram prog, Program rtprog, ExplainType type) - throws HopsException, DMLRuntimeException, LanguageException { + public static String explain(DMLProgram prog, Program rtprog, ExplainType type) { return explain(prog, rtprog, type, null); - } + } public static String explain(DMLProgram prog, Program rtprog, ExplainType type, ExplainCounts counts) - throws HopsException, DMLRuntimeException, LanguageException { //dispatch to individual explain utils switch( type ) { @@ -227,7 +222,6 @@ public class Explain } public static String explain(DMLProgram prog) - throws HopsException, DMLRuntimeException, LanguageException { StringBuilder sb = new StringBuilder(); @@ -270,8 +264,7 @@ public class Explain return sb.toString(); } - public static String getHopDAG(DMLProgram prog, ArrayList<Integer> lines, boolean withSubgraph) - throws HopsException, DMLRuntimeException, LanguageException { + public static String getHopDAG(DMLProgram prog, ArrayList<Integer> lines, boolean withSubgraph) { StringBuilder sb = new StringBuilder(); StringBuilder nodes = new StringBuilder(); @@ -314,12 +307,11 @@ public class Explain return sb.toString(); } - public static String explain( Program rtprog ) throws HopsException { + public static String explain( Program rtprog ) { return explain(rtprog, null); } public static String explain( Program rtprog, ExplainCounts counts ) - throws HopsException { //counts number of instructions boolean sparkExec = OptimizerUtils.isSparkExecutionMode(); @@ -328,7 +320,7 @@ public class Explain countCompiledInstructions(rtprog, counts, !sparkExec, true, sparkExec); } - StringBuilder sb = new StringBuilder(); + StringBuilder sb = new StringBuilder(); //create header sb.append("\nPROGRAM ( size CP/"+(sparkExec?"SP":"MR")+" = "); @@ -393,34 +385,25 @@ public class Explain return explainInstructions(inst, level); } - public static String explain( Instruction inst ) - { + public static String explain( Instruction inst ) { return explainGenericInstruction(inst, 0); } - public static String explain( StatementBlock sb ) - throws HopsException, DMLRuntimeException - { + public static String explain( StatementBlock sb ) { return explainStatementBlock(sb, 0); } - public static String explainHops( ArrayList<Hop> hops ) - throws DMLRuntimeException - { + public static String explainHops( ArrayList<Hop> hops ) { return explainHops(hops, 0); } - public static String explainHops( ArrayList<Hop> hops, int level ) - throws DMLRuntimeException - { + public static String explainHops( ArrayList<Hop> hops, int level ) { StringBuilder sb = new StringBuilder(); - Hop.resetVisitStatus(hops); for( Hop hop : hops ) sb.append(explainHop(hop, level)); Hop.resetVisitStatus(hops); - - return sb.toString(); + return sb.toString(); } public static String explain( Hop hop ) @@ -515,7 +498,7 @@ public class Explain } private static StringBuilder getHopDAG(StatementBlock sb, StringBuilder nodes, ArrayList<Integer> lines, - boolean withSubgraph) throws HopsException, DMLRuntimeException { + boolean withSubgraph) { StringBuilder builder = new StringBuilder(); if (sb instanceof WhileStatementBlock) { @@ -618,7 +601,6 @@ public class Explain } private static String explainStatementBlock(StatementBlock sb, int level) - throws HopsException, DMLRuntimeException { StringBuilder builder = new StringBuilder(); String offset = createOffset(level); @@ -1207,7 +1189,6 @@ public class Explain } private static String explainFunctionCallGraph(FunctionCallGraph fgraph, HashSet<String> fstack, String fkey, int level) - throws HopsException { StringBuilder builder = new StringBuilder(); String offset = createOffset(level); http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/utils/MLContextProxy.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/MLContextProxy.java b/src/main/java/org/apache/sysml/utils/MLContextProxy.java index d67abac..bf769bb 100644 --- a/src/main/java/org/apache/sysml/utils/MLContextProxy.java +++ b/src/main/java/org/apache/sysml/utils/MLContextProxy.java @@ -22,7 +22,6 @@ package org.apache.sysml.utils; import org.apache.sysml.api.mlcontext.MLContext; import org.apache.sysml.api.mlcontext.MLContextException; import org.apache.sysml.parser.Expression; -import org.apache.sysml.parser.LanguageException; /** * The purpose of this proxy is to shield systemml internals from direct access to MLContext @@ -44,9 +43,7 @@ public class MLContextProxy return _active; } - public static void setAppropriateVarsForRead(Expression source, String targetname) - throws LanguageException - { + public static void setAppropriateVarsForRead(Expression source, String targetname) { MLContext.getActiveMLContext().getInternalProxy().setAppropriateVarsForRead(source, targetname); } http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java index 837ad32..4f49cd9 100644 --- a/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java +++ b/src/main/java/org/apache/sysml/yarn/DMLYarnClient.java @@ -53,7 +53,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.Records; import org.apache.sysml.conf.DMLConfig; -import org.apache.sysml.parser.ParseException; import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.DMLScriptException; import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing; @@ -263,7 +262,7 @@ public class DMLYarnClient @SuppressWarnings("deprecation") private void copyResourcesToHdfsWorkingDir( YarnConfiguration yconf, String hdfsWD ) - throws ParseException, IOException, DMLRuntimeException, InterruptedException + throws IOException, InterruptedException { Path confPath = new Path(hdfsWD, DML_CONFIG_NAME); FileSystem fs = IOUtilFunctions.getFileSystem(confPath, yconf); http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java index 934967e..a295218 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/GridEnumerationMemory.java @@ -96,16 +96,12 @@ public class GridEnumerationMemory extends GridEnumeration return val; } - private void getMemoryEstimates( ArrayList<ProgramBlock> pbs, ArrayList<Long> mem ) - throws HopsException - { + private void getMemoryEstimates( ArrayList<ProgramBlock> pbs, ArrayList<Long> mem ) { for( ProgramBlock pb : pbs ) getMemoryEstimates(pb, mem); } - private void getMemoryEstimates( ProgramBlock pb, ArrayList<Long> mem ) - throws HopsException - { + private void getMemoryEstimates( ProgramBlock pb, ArrayList<Long> mem ) { if (pb instanceof FunctionProgramBlock) { FunctionProgramBlock fpb = (FunctionProgramBlock)pb; http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java index b2d20d1..610fe79 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceConfig.java @@ -22,7 +22,6 @@ package org.apache.sysml.yarn.ropt; import java.util.ArrayList; import java.util.Collections; -import org.apache.sysml.hops.HopsException; import org.apache.sysml.parser.ForStatementBlock; import org.apache.sysml.parser.IfStatementBlock; import org.apache.sysml.parser.WhileStatementBlock; @@ -45,9 +44,7 @@ public class ResourceConfig _mrres = mr; } - public ResourceConfig( ArrayList<ProgramBlock> prog, long init ) - throws HopsException - { + public ResourceConfig( ArrayList<ProgramBlock> prog, long init ) { //init cp memory _cpres = init; @@ -56,35 +53,28 @@ public class ResourceConfig addProgramBlocks(prog, init); } - public long getCPResource() - { + public long getCPResource() { return (long)_cpres; } - public void setCPResource( long res ) - { + public void setCPResource( long res ) { _cpres = res; } - public long getMRResources( int i ) - throws DMLRuntimeException - { + public long getMRResources( int i ) { if( _mrres.size() <= i ) throw new DMLRuntimeException("Memo table out-of-bounds: "+_mrres.size()+" vs "+i); - return _mrres.get(i); } - public double[][] getMRResourcesMemo() - { + public double[][] getMRResourcesMemo() { int len = _mrres.size(); double[][] ret = new double[len][2]; for( int i=0; i< len; i++ ){ ret[i][0] = _mrres.get(i); ret[i][1] = -1; } - return ret; } @@ -142,15 +132,12 @@ public class ResourceConfig return new ResourceConfig(cp, mr); } - private void addProgramBlocks( ArrayList<ProgramBlock> pbs, long init ) - throws HopsException - { + private void addProgramBlocks( ArrayList<ProgramBlock> pbs, long init ) { for( ProgramBlock pb : pbs ) addProgramBlock(pb, init); } private void addProgramBlock( ProgramBlock pb, long init ) - throws HopsException { if (pb instanceof FunctionProgramBlock) { http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java index 43030da..18b9156 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java @@ -448,7 +448,6 @@ public class ResourceOptimizer } private static ArrayList<ProgramBlock> pruneProgramBlocks( ArrayList<ProgramBlock> B ) - throws HopsException { //prune all program blocks w/o mr instructions (mr budget does not matter) if( PRUNING_SMALL ){ @@ -468,11 +467,10 @@ public class ResourceOptimizer B = Bp; } - return B; + return B; } private static boolean pruneHasOnlyUnknownMR( ProgramBlock pb ) - throws HopsException { if (pb instanceof WhileProgramBlock) { @@ -503,18 +501,14 @@ public class ResourceOptimizer } } - private static boolean pruneHasOnlyUnknownMR( ArrayList<Hop> hops ) - throws HopsException - { + private static boolean pruneHasOnlyUnknownMR( ArrayList<Hop> hops ) { boolean ret = false; - if( hops!=null ){ ret = true; Hop.resetVisitStatus(hops); for( Hop hop : hops ) ret &= pruneHasOnlyUnknownMR(hop); } - return ret; } http://git-wip-us.apache.org/repos/asf/systemml/blob/215a1cc6/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java index 2b7886f..b9d6b51 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java @@ -24,248 +24,6 @@ package org.apache.sysml.yarn.ropt; * Experimental feature not used in production. * */ -public class ResourceOptimizerCPMigration -{ - - /*// FIXME MB - private static final Log LOG = LogFactory.getLog(CPMigrationOptimizer.class.getName()); - - public static void initResumeInfoFromFile(String file, ExecutionContext ec) throws IOException, DMLRuntimeException { - DMLScript.resumeSbIdRStack.clear(); - DMLScript.resumeFuncVarRStack.clear(); - DMLScript.resumeLoopAndFuncEntryVarRStack.clear(); - if (file == null) - return; - - FileSystem fs = FileSystem.get(new YarnConfiguration()); - Path path = new Path(file); - if (!fs.exists(path)) - throw new IOException("File " + file + " does not exist"); - - //BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(path))); - FSDataInputStream reader = fs.open(path); - DMLScript.predecessorAppIdStr = reader.readUTF().trim(); - DMLScript.resumeSbIdRStack = DMLScript.deserializeToReverseStack(reader.readUTF().trim()); - - // Read and invert the function symbol table stack - Stack<LocalVariableMap> tmpStack = new Stack<LocalVariableMap>(); - int n = Integer.parseInt(reader.readUTF().trim()); - for (int i = 0; i < n; i++) - tmpStack.push(LocalVariableMap.deserialize(reader.readUTF().trim())); - for (int i = 0; i < n; i++) - DMLScript.resumeFuncVarRStack.push(tmpStack.pop()); - - // Read and invert the loop and func entry symbol table stack - tmpStack.clear(); - n = Integer.parseInt(reader.readUTF().trim()); - for (int i = 0; i < n; i++) - tmpStack.push(LocalVariableMap.deserialize(reader.readUTF().trim())); - for (int i = 0; i < n; i++) - DMLScript.resumeLoopAndFuncEntryVarRStack.push(tmpStack.pop()); - - reader.close(); - - // Append this CP's AppId to the predecessor's resume file - FSDataOutputStream fout = fs.append(path); - fout.writeUTF(MyApplicationMaster.appId + "\n"); - fout.close(); - - // Log the resume info - StringBuilder sb = new StringBuilder(); - sb.append("Predecessor is " + DMLScript.predecessorAppIdStr + "\n"); - sb.append("Inverse sbID stack: "); - for (Long l : DMLScript.resumeSbIdRStack) - sb.append(l + ", "); - sb.append("\n" + DMLScript.resumeFuncVarRStack.size() + " inverse function symbol table stack:\n"); - for (LocalVariableMap symbolTable : DMLScript.resumeFuncVarRStack) - sb.append("\t" + symbolTable.serialize() + "\n"); - sb.append(DMLScript.resumeLoopAndFuncEntryVarRStack.size() + " inverse loop and func entry symbol table stack:\n"); - for (LocalVariableMap symbolTable : DMLScript.resumeLoopAndFuncEntryVarRStack) - sb.append("\t" + symbolTable.serialize() + "\n"); - LOG.info(sb.toString()); - - // Recover the most outer layer of symbol table before execution starts - LocalVariableMap varMap = DMLScript.resumeFuncVarRStack.pop(); - DMLScript.lastPopedFuncVarStr = varMap.serialize(); // For later correctness check - ec.setVariables(varMap); - } - - // Try to migrate to a new CP, return false if failed - public static boolean migrateCP(LocalVariableMap currentSymbolTable) throws DMLRuntimeException { - DMLScript.execFuncVarStack.push(currentSymbolTable); - - long start = System.currentTimeMillis(); - for (LocalVariableMap symbolTable : DMLScript.execFuncVarStack) { - for (String var : symbolTable.keySet()) { - Data data = symbolTable.get(var); - if ( data.getDataType() == DataType.MATRIX ) { - long time = System.currentTimeMillis(); - MatrixObject matrix = (MatrixObject) data; - matrix.exportData(); - time = System.currentTimeMillis() - time; - LOG.info("Exporting " + var + " took " + time + "ms"); - } - } - } - start = System.currentTimeMillis() - start; - LOG.info("Exporting data to hdfs took " + start + "ms"); - - DMLConfig config = ConfigurationManager.getConfig(); - String hdfsWorkingDir = MyYarnClient.getHDFSWorkingDir(config, MyApplicationMaster.appId); - - try { - FileSystem fs = FileSystem.get(MyApplicationMaster.conf); - Path resume_file = new Path(hdfsWorkingDir, DMLScript.CP_RESUME_HDFS_FILE); - if (fs.exists(resume_file)) - throw new IOException("File " + resume_file + " already exists?"); - - FSDataOutputStream fout = fs.create(resume_file); - fout.writeUTF(MyApplicationMaster.appId + "\n"); - - String tmp = DMLScript.serializeExecSbIdStack(DMLScript.execSbIdStack); - fout.writeUTF(tmp + "\n"); - LOG.info("SbId Stack: " + tmp); - - fout.writeUTF(DMLScript.execFuncVarStack.size() + "\n"); - LOG.info(DMLScript.execFuncVarStack.size() + " func symbol tables"); - for (LocalVariableMap symbolTable : DMLScript.execFuncVarStack) { - tmp = symbolTable.serialize(); - fout.writeUTF(symbolTable.serialize() + "\n"); - LOG.info("\t" + tmp); - } - - fout.writeUTF(DMLScript.execLoopAndFuncEntryVarStack.size() + "\n"); - LOG.info(DMLScript.execLoopAndFuncEntryVarStack.size() + " loop and func entry symbol tables"); - for (LocalVariableMap symbolTable : DMLScript.execLoopAndFuncEntryVarStack) { - tmp = symbolTable.serialize(); - fout.writeUTF(symbolTable.serialize() + "\n"); - LOG.info("\t" + tmp); - } - fout.close(); - - // If specified, run the new CP within this old CP automatically - if (DMLScript.newCpBudgetByte != -1) { - YarnApplicationState state; - MyYarnClient client = new MyYarnClient(); - state = client.runResumeCP(MyApplicationMaster.fullArgs, DMLScript.newCpBudgetByte, hdfsWorkingDir); - if (state != YarnApplicationState.FINISHED) - throw new DMLRuntimeException("Resuming CP " + state); - } - } catch (Exception e) { - throw new DMLRuntimeException(e); - } - - // Make this CP stop gracefully - DMLScript.migratedAndStop = true; - - LOG.info("CP migration done, exiting gracefully"); - return true; - } - - - - - - // A complete full copy of the runtime plan for recompile and costing - public ArrayList<ProgramBlock> copiedProgramBlocks; - Program prog; - - public HashMap<Long, ProgramBlock> sbIdMap; // Pointer to all reachable program blocks - - public CPMigrationOptimizer(Program rtprog) throws DMLRuntimeException, HopsException { - prog = rtprog; - copiedProgramBlocks = ProgramConverter.rcreateDeepCopyProgramBlocks(prog._programBlocks, 1, -1, new HashSet<String>(), false); - - // Temporarily disable dynamic recompile to clear all flags - boolean flag = OptimizerUtils.ALLOW_DYN_RECOMPILATION; - OptimizerUtils.ALLOW_DYN_RECOMPILATION = false; - - sbIdMap = new HashMap<Long, ProgramBlock> (); - for (ProgramBlock pb : copiedProgramBlocks) - initTraverse(pb); - OptimizerUtils.ALLOW_DYN_RECOMPILATION = flag; - - - } - - public void recompile() throws HopsException { - resetAllFlags(); - } - - public boolean shouldMigrate(Stack<Long> execSbIdStack, Stack<LocalVariableMap> execLoopAndFuncEntryVarStack) - throws HopsException { - - int i = 0, j = 0; - int base = -1; // Index of base function, -1 for main - LocalVariableMap baseVar = new LocalVariableMap(); - int loop = -1; // Index of the outer most loop - HashMap<Long, LocalVariableMap> loopEntryVarMap = new HashMap<Long, LocalVariableMap>(); - - for (Long sbId : execSbIdStack) { - ProgramBlock pb = sbIdMap.get(sbId); - if (pb instanceof FunctionProgramBlock) { - base = i; - baseVar = execLoopAndFuncEntryVarStack.get(j); - loop = -1; - loopEntryVarMap.clear(); - j++; - } else if (pb instanceof WhileProgramBlock || pb instanceof ForProgramBlock) { - if (loop == -1) - loop = i; - loopEntryVarMap.put(sbId, execLoopAndFuncEntryVarStack.get(j)); - j++; - } - i++; - } - - // To be continued !!!!! - - return false; - } - - public void resetAllFlags() throws HopsException { - boolean flag = OptimizerUtils.ALLOW_DYN_RECOMPILATION; - OptimizerUtils.ALLOW_DYN_RECOMPILATION = false; - for (Map.Entry<Long, ProgramBlock> entry : sbIdMap.entrySet()) - entry.getValue().getStatementBlock().updateRecompilationFlag(); - OptimizerUtils.ALLOW_DYN_RECOMPILATION = flag; - } - - public void initTraverse(ProgramBlock pb) throws HopsException, DMLRuntimeException { - long sbId = pb.getStatementBlock().getID(); - if (sbIdMap.containsKey(sbId)) - return; - - sbIdMap.put(sbId, pb); - if (pb instanceof WhileProgramBlock) { - WhileProgramBlock tmp = (WhileProgramBlock)pb; - for (ProgramBlock pb2 : tmp.getChildBlocks()) - initTraverse(pb2); - } else if (pb instanceof IfProgramBlock) { - IfProgramBlock tmp = (IfProgramBlock)pb; - for (ProgramBlock pb2 : tmp.getChildBlocksIfBody()) - initTraverse(pb2); - for (ProgramBlock pb2 : tmp.getChildBlocksElseBody()) - initTraverse(pb2); - } else if (pb instanceof ForProgramBlock) { - ForProgramBlock tmp = (ForProgramBlock)pb; - for (ProgramBlock pb2 : tmp.getChildBlocks()) - initTraverse(pb2); - } else if (pb instanceof FunctionProgramBlock && !(pb instanceof ExternalFunctionProgramBlock)) { - FunctionProgramBlock tmp = (FunctionProgramBlock) pb; - for (ProgramBlock pb2 : tmp.getChildBlocks()) - initTraverse(pb2); - } else { - // Clear the flag on leaf blocks - pb.getStatementBlock().updateRecompilationFlag(); - for (Instruction inst : pb.getInstructions()) { - if (inst instanceof FunctionCallCPInstruction) { - FunctionCallCPInstruction finst = (FunctionCallCPInstruction)inst; - initTraverse(prog.getFunctionProgramBlock(finst.getNamespace(), finst.getFunctionName())); - } - } - } - } - - */ +public class ResourceOptimizerCPMigration { + // FIXME MB }
