Repository: incubator-systemml Updated Branches: refs/heads/master b280b7bc9 -> f21eab7db
[SYSML-282] UpdateInPlace parfor intermediate cleanup Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f21eab7d Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f21eab7d Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f21eab7d Branch: refs/heads/master Commit: f21eab7db4c6980b0ff56f2f16b59a050e2f70b1 Parents: b280b7b Author: Arvind Surve <[email protected]> Authored: Thu Feb 18 13:53:34 2016 -0800 Committer: Arvind Surve <[email protected]> Committed: Thu Feb 18 13:53:34 2016 -0800 ---------------------------------------------------------------------- .../parfor/opt/OptimizerRuleBased.java | 2 +- .../cp/MatrixIndexingCPInstruction.java | 10 ++++ .../instructions/cp/VariableCPInstruction.java | 4 +- .../java/org/apache/sysml/utils/Statistics.java | 19 +++++--- .../updateinplace/UpdateInPlaceTest.java | 49 +++++++++++--------- 5 files changed, 50 insertions(+), 34 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f21eab7d/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java index 9754756..6c4d254 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/parfor/opt/OptimizerRuleBased.java @@ -155,7 +155,7 @@ public class OptimizerRuleBased extends Optimizer public static final boolean APPLY_REWRITE_NESTED_PARALLELISM = false; public static final String FUNCTION_UNFOLD_NAMEPREFIX = "__unfold_"; - public static final boolean APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE = true; + public static final boolean APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE = false; public static final double PAR_K_FACTOR = OptimizationWrapper.PAR_FACTOR_INFRASTRUCTURE; public static final double PAR_K_MR_FACTOR = 1.0 * OptimizationWrapper.PAR_FACTOR_INFRASTRUCTURE; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f21eab7d/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java index c03e7ae..bfd2b0c 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixIndexingCPInstruction.java @@ -19,6 +19,7 @@ package org.apache.sysml.runtime.instructions.cp; +import org.apache.sysml.api.DMLScript; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; @@ -30,6 +31,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.matrix.operators.SimpleOperator; import org.apache.sysml.runtime.util.IndexRange; +import org.apache.sysml.utils.Statistics; public class MatrixIndexingCPInstruction extends UnaryCPInstruction { @@ -165,6 +167,14 @@ public class MatrixIndexingCPInstruction extends UnaryCPInstruction else if ( opcode.equalsIgnoreCase("leftIndex")) { boolean inplace = mo.isUpdateInPlaceEnabled(); + if(DMLScript.STATISTICS) + { + if(inplace) + Statistics.incrementTotalLixUIP(); + Statistics.incrementTotalLix(); + } + + MatrixBlock matBlock = ec.getMatrixInput(input1.getName()); MatrixBlock resultBlock = null; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f21eab7d/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index bf8d792..22bd2ab 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -447,9 +447,7 @@ public class VariableCPInstruction extends CPInstruction mobj.enableUpdateInPlace(updateInPlace); ec.setVariable(input1.getName(), mobj); if(updateInPlace) - Statistics.incrementTotUpdateInPlace(); - else - Statistics.incrementTotNonUpdateInPlace(); + Statistics.incrementTotalUIPVar(); } else if ( input1.getDataType() == DataType.SCALAR ){ ScalarObject sobj = null; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f21eab7d/src/main/java/org/apache/sysml/utils/Statistics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java index 499b19c..167f6db 100644 --- a/src/main/java/org/apache/sysml/utils/Statistics.java +++ b/src/main/java/org/apache/sysml/utils/Statistics.java @@ -88,8 +88,9 @@ public class Statistics private static HashMap<String,Long> _cpInstTime = new HashMap<String, Long>(); private static HashMap<String,Long> _cpInstCounts = new HashMap<String, Long>(); - private static AtomicLong lTotUpdateInPlace = new AtomicLong(0); - private static AtomicLong lTotNonUpdateInPlace = new AtomicLong(0); + private static AtomicLong lTotalUIPVar = new AtomicLong(0); + private static AtomicLong lTotalLix = new AtomicLong(0); + private static AtomicLong lTotalLixUIP = new AtomicLong(0); public static synchronized void setNoOfExecutedMRJobs(int iNoOfExecutedMRJobs) { Statistics.iNoOfExecutedMRJobs = iNoOfExecutedMRJobs; @@ -147,12 +148,16 @@ public class Statistics iNoOfCompiledSPInst ++; } - public static void incrementTotUpdateInPlace() { - lTotUpdateInPlace.incrementAndGet(); + public static void incrementTotalUIPVar() { + lTotalUIPVar.incrementAndGet(); } - public static void incrementTotNonUpdateInPlace() { - lTotNonUpdateInPlace.incrementAndGet(); + public static void incrementTotalLixUIP() { + lTotalLixUIP.incrementAndGet(); + } + + public static void incrementTotalLix() { + lTotalLix.incrementAndGet(); } /** @@ -550,7 +555,7 @@ public class Statistics sb.append("ParFor optimize time:\t\t" + String.format("%.3f", ((double)getParforOptTime())/1000) + " sec.\n"); sb.append("ParFor initialize time:\t\t" + String.format("%.3f", ((double)getParforInitTime())/1000) + " sec.\n"); sb.append("ParFor result merge time:\t" + String.format("%.3f", ((double)getParforMergeTime())/1000) + " sec.\n"); - sb.append("ParFor total update in-place:\t" + lTotUpdateInPlace + "/" + (lTotUpdateInPlace.get()+lTotNonUpdateInPlace.get()) + "\n"); + sb.append("ParFor total update in-place:\t" + lTotalUIPVar + "/" + lTotalLixUIP + "/" + lTotalLix + "\n"); } sb.append("Total JIT compile time:\t\t" + ((double)getJITCompileTime())/1000 + " sec.\n"); sb.append("Total JVM GC count:\t\t" + getJVMgcCount() + ".\n"); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f21eab7d/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java b/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java index 2732820..503a42f 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/updateinplace/UpdateInPlaceTest.java @@ -240,35 +240,38 @@ public class UpdateInPlaceTest extends AutomatedTestBase // This is for running the junit test the new way, i.e., construct the arguments directly String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + iTestNumber + ".dml"; - programArgs = new String[]{}; //new String[]{"-args", input("A"), output("B") }; + programArgs = new String[]{"-stats"}; //new String[]{"-args", input("A"), output("B") }; runTest(true, false, null, -1); - List<String> listUIPRes = OptimizerRuleBased.getUIPList(); - int iUIPResCount = 0; - - // If UpdateInPlace list specified in the argument, verify the list. - if (listUIPExp != null) + if(OptimizerRuleBased.APPLY_REWRITE_UPDATE_INPLACE_INTERMEDIATE) { - if(listUIPRes != null) + List<String> listUIPRes = OptimizerRuleBased.getUIPList(); + int iUIPResCount = 0; + + // If UpdateInPlace list specified in the argument, verify the list. + if (listUIPExp != null) { - for (String strUIPMatName: listUIPExp) - Assert.assertTrue("Expected UpdateInPlace matrix " + strUIPMatName - + " does not exist in the result UpdateInPlace matrix list.", - listUIPRes.contains(strUIPMatName)); - - iUIPResCount = listUIPRes.size(); + if(listUIPRes != null) + { + for (String strUIPMatName: listUIPExp) + Assert.assertTrue("Expected UpdateInPlace matrix " + strUIPMatName + + " does not exist in the result UpdateInPlace matrix list.", + listUIPRes.contains(strUIPMatName)); + + iUIPResCount = listUIPRes.size(); + } + + Assert.assertTrue("Expected # of UpdateInPlace matrix object/s " + listUIPExp.size() + + " does not match with the # of matrix objects " + iUIPResCount + " from optimization result.", + (iUIPResCount == listUIPExp.size())); + } + else + { + Assert.assertTrue("Expected # of UpdateInPlace matrix object/s " + "0" + + " does not match with the # of matrix objects " + "0" + " from optimization result.", + (listUIPRes == null || listUIPRes.size() == 0)); } - - Assert.assertTrue("Expected # of UpdateInPlace matrix object/s " + listUIPExp.size() + - " does not match with the # of matrix objects " + iUIPResCount + " from optimization result.", - (iUIPResCount == listUIPExp.size())); - } - else - { - Assert.assertTrue("Expected # of UpdateInPlace matrix object/s " + "0" + - " does not match with the # of matrix objects " + "0" + " from optimization result.", - (listUIPRes == null || listUIPRes.size() == 0)); } } finally{
