Repository: systemml Updated Branches: refs/heads/master a16905327 -> ae268a9ee
http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/main/java/org/apache/sysml/utils/Statistics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Statistics.java b/src/main/java/org/apache/sysml/utils/Statistics.java index 44667ba..ca55564 100644 --- a/src/main/java/org/apache/sysml/utils/Statistics.java +++ b/src/main/java/org/apache/sysml/utils/Statistics.java @@ -34,6 +34,7 @@ import java.util.concurrent.atomic.LongAdder; import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.ConfigurationManager; +import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.controlprogram.caching.CacheStatistics; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; @@ -241,7 +242,7 @@ public class Statistics numExecutedSPInst.reset(); numExecutedMRJobs.reset(); - if( DMLScript.USE_ACCELERATOR ) + if( ConfigurationManager.isGPU() ) GPUStatistics.setNoOfExecutedGPUInst(0); } @@ -400,12 +401,12 @@ public class Statistics } public static void startCompileTimer() { - if( DMLScript.STATISTICS ) + if( ConfigurationManager.isStatistics() ) compileStartTime = System.nanoTime(); } public static void stopCompileTimer() { - if( DMLScript.STATISTICS ) + if( ConfigurationManager.isStatistics() ) compileEndTime = System.nanoTime(); } @@ -722,11 +723,12 @@ public class Statistics maxCountLen = Math.max(maxCountLen, String.valueOf(hh.getValue().count.longValue()).length()); } - maxInstLen = Math.min(maxInstLen, DMLScript.STATISTICS_MAX_WRAP_LEN); + int statsMaxWrapLength = ConfigurationManager.getDMLConfig().getIntValue(DMLConfig.STATS_MAX_WRAP_LEN); + maxInstLen = Math.min(maxInstLen, statsMaxWrapLength); sb.append(String.format( " %" + maxNumLen + "s %-" + maxInstLen + "s %" + maxTimeSLen + "s %" + maxCountLen + "s", numCol, instCol, timeSCol, countCol)); - if (DMLScript.FINEGRAINED_STATISTICS) { + if (ConfigurationManager.isFinegrainedStatistics()) { sb.append(" "); sb.append(gpuCol); } @@ -743,15 +745,15 @@ public class Statistics int numLines = wrappedInstruction.length; String [] miscTimers = null; - if (DMLScript.FINEGRAINED_STATISTICS) { - miscTimers = wrap(GPUStatistics.getStringForCPMiscTimesPerInstruction(instruction), DMLScript.STATISTICS_MAX_WRAP_LEN); + if (ConfigurationManager.isFinegrainedStatistics()) { + miscTimers = wrap(GPUStatistics.getStringForCPMiscTimesPerInstruction(instruction), statsMaxWrapLength); numLines = Math.max(numLines, miscTimers.length); } - String miscFormatString = (DMLScript.FINEGRAINED_STATISTICS) ? " %" + DMLScript.STATISTICS_MAX_WRAP_LEN + "s" : "%s"; + String miscFormatString = (ConfigurationManager.isFinegrainedStatistics()) ? " %" + statsMaxWrapLength + "s" : "%s"; for(int wrapIter = 0; wrapIter < numLines; wrapIter++) { String instStr = (wrapIter < wrappedInstruction.length) ? wrappedInstruction[wrapIter] : ""; - String miscTimerStr = ( (DMLScript.FINEGRAINED_STATISTICS) && wrapIter < miscTimers.length) ? miscTimers[wrapIter] : ""; + String miscTimerStr = ( (ConfigurationManager.isFinegrainedStatistics()) && wrapIter < miscTimers.length) ? miscTimers[wrapIter] : ""; if(wrapIter == 0) { // Display instruction count sb.append(String.format( @@ -907,7 +909,7 @@ public class Statistics * @return statistics as a string */ public static String display() { - return display(DMLScript.STATISTICS_COUNT); + return display(ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters()); } @@ -931,27 +933,27 @@ public class Statistics StringBuilder sb = new StringBuilder(); sb.append("SystemML Statistics:\n"); - if( DMLScript.STATISTICS ) { + if( ConfigurationManager.isStatistics() ) { sb.append("Total elapsed time:\t\t" + String.format("%.3f", (getCompileTime()+getRunTime())*1e-9) + " sec.\n"); // nanoSec --> sec sb.append("Total compilation time:\t\t" + String.format("%.3f", getCompileTime()*1e-9) + " sec.\n"); // nanoSec --> sec } sb.append("Total execution time:\t\t" + String.format("%.3f", getRunTime()*1e-9) + " sec.\n"); // nanoSec --> sec if( OptimizerUtils.isSparkExecutionMode() ) { - if( DMLScript.STATISTICS ) //moved into stats on Shiv's request + if( ConfigurationManager.isStatistics() ) //moved into stats on Shiv's request sb.append("Number of compiled Spark inst:\t" + getNoOfCompiledSPInst() + ".\n"); sb.append("Number of executed Spark inst:\t" + getNoOfExecutedSPInst() + ".\n"); } else { - if( DMLScript.STATISTICS ) //moved into stats on Shiv's request + if( ConfigurationManager.isStatistics() ) //moved into stats on Shiv's request sb.append("Number of compiled MR Jobs:\t" + getNoOfCompiledMRJobs() + ".\n"); sb.append("Number of executed MR Jobs:\t" + getNoOfExecutedMRJobs() + ".\n"); } - if( DMLScript.USE_ACCELERATOR && DMLScript.STATISTICS) + if( ConfigurationManager.isGPU() && ConfigurationManager.isStatistics()) sb.append(GPUStatistics.getStringForCudaTimers()); //show extended caching/compilation statistics - if( DMLScript.STATISTICS ) + if( ConfigurationManager.isStatistics() ) { if(NativeHelper.CURRENT_NATIVE_BLAS_STATE == NativeHelper.NativeBlasState.SUCCESSFULLY_LOADED_NATIVE_BLAS_AND_IN_USE) { String blas = NativeHelper.getCurrentBLAS(); @@ -1027,7 +1029,7 @@ public class Statistics sb.append("Total JVM GC time:\t\t" + ((double)getJVMgcTime())/1000 + " sec.\n"); LibMatrixDNN.appendStatistics(sb); sb.append("Heavy hitter instructions:\n" + getHeavyHitters(maxHeavyHitters)); - if (DMLScript.JMLC_MEM_STATISTICS && DMLScript.FINEGRAINED_STATISTICS) + if (DMLScript.JMLC_MEM_STATISTICS && ConfigurationManager.isFinegrainedStatistics()) sb.append("Heavy hitter objects:\n" + getCPHeavyHittersMem(maxHeavyHitters)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java index 61997f4..a7243f5 100644 --- a/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java +++ b/src/test/java/org/apache/sysml/test/integration/applications/parfor/ParForCorrelationTest.java @@ -23,8 +23,8 @@ import java.util.HashMap; import org.junit.Test; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.lops.LopProperties.ExecType; import org.apache.sysml.runtime.controlprogram.ParForProgramBlock.PExecMode; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; @@ -161,8 +161,6 @@ public class ParForCorrelationTest extends AutomatedTestBase config.addVariable("cols", cols); loadTestConfiguration(config); - boolean oldStatistics = DMLScript.STATISTICS; - /* This is for running the junit test the new way, i.e., construct the arguments directly */ String HOME = SCRIPT_DIR + TEST_DIR; fullDMLScriptName = HOME + TEST_NAME + scriptNum + ".dml"; @@ -190,7 +188,7 @@ public class ParForCorrelationTest extends AutomatedTestBase } finally { - DMLScript.STATISTICS = oldStatistics; + ConfigurationManager.resetStatistics(); rtplatform = oldPlatform; } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java index de0a26f..fcd2326 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/APICodegenTest.java @@ -23,12 +23,12 @@ import static org.apache.sysml.api.mlcontext.ScriptFactory.dml; import org.apache.spark.SparkConf; import org.apache.spark.api.java.JavaSparkContext; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.jmlc.Connection; import org.apache.sysml.api.jmlc.PreparedScript; import org.apache.sysml.api.mlcontext.MLContext; import org.apache.sysml.api.mlcontext.Script; import org.apache.sysml.conf.CompilerConfig.ConfigType; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -77,15 +77,20 @@ public class APICodegenTest extends AutomatedTestBase //execute scripts if( jmlc ) { - DMLScript.STATISTICS = true; - Connection conn = new Connection(ConfigType.CODEGEN_ENABLED, - ConfigType.ALLOW_DYN_RECOMPILATION); - PreparedScript pscript = conn.prepareScript( - s, new String[]{"X"}, new String[]{"R"}, false); - pscript.setMatrix("X", mX, false); - pscript.executeScript(); - conn.close(); - System.out.println(Statistics.display()); + ConfigurationManager.setStatistics(true); + try { + Connection conn = new Connection(ConfigType.CODEGEN_ENABLED, + ConfigType.ALLOW_DYN_RECOMPILATION); + PreparedScript pscript = conn.prepareScript( + s, new String[]{"X"}, new String[]{"R"}, false); + pscript.setMatrix("X", mX, false); + pscript.executeScript(); + conn.close(); + System.out.println(Statistics.display()); + } + finally { + ConfigurationManager.resetStatistics(); + } } else { SparkConf conf = SparkExecutionContext.createSystemMLSparkConf() http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java index 6dcb673..c4c974d 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameConverterTest.java @@ -35,6 +35,7 @@ import org.apache.spark.sql.SparkSession; import org.apache.spark.sql.types.StructType; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; @@ -203,7 +204,7 @@ public class FrameConverterTest extends AutomatedTestBase private void runFrameConverterTest( ValueType[] schema, ConvType type) { RUNTIME_PLATFORM platformOld = rtplatform; - DMLScript.rtplatform = RUNTIME_PLATFORM.SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.SPARK); boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; DMLScript.USE_LOCAL_SPARK_CONFIG = true; @@ -262,7 +263,7 @@ public class FrameConverterTest extends AutomatedTestBase } finally { - DMLScript.rtplatform = platformOld; + ConfigurationManager.getDMLOptions().setExecutionMode(platformOld); DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCClonedPreparedScriptTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCClonedPreparedScriptTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCClonedPreparedScriptTest.java index ca80e47..80cab67 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCClonedPreparedScriptTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCClonedPreparedScriptTest.java @@ -30,9 +30,9 @@ import java.util.concurrent.Future; import org.junit.Assert; import org.junit.Test; import org.apache.sysml.api.DMLException; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.jmlc.Connection; import org.apache.sysml.api.jmlc.PreparedScript; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.utils.Statistics; @@ -100,8 +100,9 @@ public class JMLCClonedPreparedScriptTest extends AutomatedTestBase int k = InfrastructureAnalyzer.getLocalParallelism(); boolean failed = false; + try( Connection conn = new Connection() ) { - DMLScript.STATISTICS = true; + ConfigurationManager.setStatistics(true); Statistics.reset(); PreparedScript pscript = conn.prepareScript( script, new String[]{}, new String[]{"out"}, false); @@ -119,6 +120,9 @@ public class JMLCClonedPreparedScriptTest extends AutomatedTestBase catch(Exception ex) { failed = true; } + finally { + ConfigurationManager.resetStatistics(); + } //check expected failure Assert.assertTrue(failed==!clone || k==1); http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCParfor2ForCompileTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCParfor2ForCompileTest.java b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCParfor2ForCompileTest.java index e806f0c..2f2022e 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCParfor2ForCompileTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/jmlc/JMLCParfor2ForCompileTest.java @@ -23,10 +23,10 @@ import java.io.IOException; import org.junit.Assert; import org.junit.Test; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.jmlc.Connection; import org.apache.sysml.api.jmlc.PreparedScript; import org.apache.sysml.conf.CompilerConfig.ConfigType; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.utils.Statistics; @@ -49,7 +49,7 @@ public class JMLCParfor2ForCompileTest extends AutomatedTestBase private void runJMLCParFor2ForTest(boolean par) throws IOException - { + { try { Connection conn = !par ? new Connection() : new Connection(ConfigType.PARALLEL_LOCAL_OR_REMOTE_PARFOR); @@ -59,19 +59,20 @@ public class JMLCParfor2ForCompileTest extends AutomatedTestBase + "parfor(i in 1:nrow(X))" + " R[i,] = sum(X[i,])" + "print(sum(R))"; - DMLScript.STATISTICS = true; Statistics.reset(); PreparedScript pscript = conn.prepareScript( script, new String[]{}, new String[]{}, false); + ConfigurationManager.setStatistics(true); pscript.executeScript(); conn.close(); + //check for existing or non-existing parfor + Assert.assertTrue(Statistics.getParforOptCount()==(par?1:0)); } catch(Exception ex) { Assert.fail("JMLC parfor test failed: "+ex.getMessage()); + } finally { + ConfigurationManager.resetStatistics(); } - - //check for existing or non-existing parfor - Assert.assertTrue(Statistics.getParforOptCount()==(par?1:0)); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameMatrixConversionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameMatrixConversionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameMatrixConversionTest.java index ec946fb..a446573 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameMatrixConversionTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameMatrixConversionTest.java @@ -193,12 +193,12 @@ public class DataFrameMatrixConversionTest extends AutomatedTestBase private void testDataFrameConversion(boolean vector, int cols, boolean dense, boolean unknownDims) { boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; - RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform; + RUNTIME_PLATFORM oldPlatform = ConfigurationManager.getExecutionMode(); try { DMLScript.USE_LOCAL_SPARK_CONFIG = true; - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.HYBRID_SPARK); //generate input data and setup metadata int rows = (cols == cols3) ? rows3 : rows1; @@ -229,18 +229,18 @@ public class DataFrameMatrixConversionTest extends AutomatedTestBase } finally { DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; - DMLScript.rtplatform = oldPlatform; + ConfigurationManager.getDMLOptions().setExecutionMode(oldPlatform); } } private void testDataFrameConversionUltraSparse(boolean vector, boolean unknownDims) { boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; - RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform; + RUNTIME_PLATFORM oldPlatform = ConfigurationManager.getExecutionMode(); try { DMLScript.USE_LOCAL_SPARK_CONFIG = true; - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.HYBRID_SPARK); //generate input data and setup metadata double[][] A = getRandomMatrix(rows1, 1, -10, 10, 0.7, 2373); @@ -271,7 +271,7 @@ public class DataFrameMatrixConversionTest extends AutomatedTestBase } finally { DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; - DMLScript.rtplatform = oldPlatform; + ConfigurationManager.getDMLOptions().setExecutionMode(oldPlatform); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameRowFrameConversionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameRowFrameConversionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameRowFrameConversionTest.java index 452c1e1..4d130a7 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameRowFrameConversionTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameRowFrameConversionTest.java @@ -199,12 +199,12 @@ public class DataFrameRowFrameConversionTest extends AutomatedTestBase private void testDataFrameConversion(ValueType vt, boolean singleColBlock, boolean dense, boolean unknownDims) { boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; - RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform; + RUNTIME_PLATFORM oldPlatform = ConfigurationManager.getExecutionMode(); try { DMLScript.USE_LOCAL_SPARK_CONFIG = true; - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.HYBRID_SPARK); //generate input data and setup metadata int cols = singleColBlock ? cols1 : cols2; @@ -238,7 +238,7 @@ public class DataFrameRowFrameConversionTest extends AutomatedTestBase } finally { DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; - DMLScript.rtplatform = oldPlatform; + ConfigurationManager.getDMLOptions().setExecutionMode(oldPlatform); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorFrameConversionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorFrameConversionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorFrameConversionTest.java index d5484ce..36fb103 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorFrameConversionTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/DataFrameVectorFrameConversionTest.java @@ -249,12 +249,12 @@ public class DataFrameVectorFrameConversionTest extends AutomatedTestBase private void testDataFrameConversion(ValueType[] schema, boolean containsID, boolean dense, boolean unknownDims) { boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; - RUNTIME_PLATFORM oldPlatform = DMLScript.rtplatform; + RUNTIME_PLATFORM oldPlatform = ConfigurationManager.getExecutionMode(); try { DMLScript.USE_LOCAL_SPARK_CONFIG = true; - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.HYBRID_SPARK); //generate input data and setup metadata int cols = schema.length + colsVector - 1; @@ -285,7 +285,7 @@ public class DataFrameVectorFrameConversionTest extends AutomatedTestBase } finally { DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; - DMLScript.rtplatform = oldPlatform; + ConfigurationManager.getDMLOptions().setExecutionMode(oldPlatform); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java index 60a285a..10ccc0b 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/FrameTest.java @@ -41,6 +41,7 @@ import org.apache.sysml.api.mlcontext.FrameSchema; import org.apache.sysml.api.mlcontext.MLResults; import org.apache.sysml.api.mlcontext.Script; import org.apache.sysml.api.mlcontext.ScriptFactory; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.DataExpression; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.parser.ParseException; @@ -148,8 +149,8 @@ public class FrameTest extends MLContextTestBase boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; DMLScript.USE_LOCAL_SPARK_CONFIG = true; - RUNTIME_PLATFORM oldRT = DMLScript.rtplatform; - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + RUNTIME_PLATFORM oldRT = ConfigurationManager.getExecutionMode(); + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.HYBRID_SPARK); int rowstart = 234, rowend = 1478, colstart = 125, colend = 568; int bRows = rowend-rowstart+1, bCols = colend-colstart+1; @@ -340,7 +341,7 @@ public class FrameTest extends MLContextTestBase System.out.println("Frame MLContext test completed successfully."); } finally { - DMLScript.rtplatform = oldRT; + ConfigurationManager.getDMLOptions().setExecutionMode(oldRT); DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java index 2f9489b..97e2fbb 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/mlcontext/GNMFTest.java @@ -145,11 +145,11 @@ public class GNMFTest extends MLContextTestBase boolean oldConfig = DMLScript.USE_LOCAL_SPARK_CONFIG; DMLScript.USE_LOCAL_SPARK_CONFIG = true; - RUNTIME_PLATFORM oldRT = DMLScript.rtplatform; + RUNTIME_PLATFORM oldRT = ConfigurationManager.getExecutionMode(); try { - DMLScript.rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(RUNTIME_PLATFORM.HYBRID_SPARK); Script script = ScriptFactory.dmlFromFile(fullDMLScriptName); // set positional argument values @@ -235,7 +235,7 @@ public class GNMFTest extends MLContextTestBase TestUtils.compareMatrices(hmHDML, hmHR, 0.000001, "hmHDML", "hmHR"); } finally { - DMLScript.rtplatform = oldRT; + ConfigurationManager.getDMLOptions().setExecutionMode(oldRT); DMLScript.USE_LOCAL_SPARK_CONFIG = oldConfig; } } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/functions/paramserv/SparkDataPartitionerTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/SparkDataPartitionerTest.java b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/SparkDataPartitionerTest.java index 8cae4a4..2928e06 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/paramserv/SparkDataPartitionerTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/paramserv/SparkDataPartitionerTest.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.stream.IntStream; import org.apache.sysml.api.DMLScript; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.parser.Statement; import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; @@ -40,7 +41,7 @@ public class SparkDataPartitionerTest extends BaseDataPartitionerTest { static { DMLScript.USE_LOCAL_SPARK_CONFIG = true; - DMLScript.rtplatform = DMLScript.RUNTIME_PLATFORM.SPARK; + ConfigurationManager.getDMLOptions().setExecutionMode(DMLScript.RUNTIME_PLATFORM.SPARK); _sec = (SparkExecutionContext) ExecutionContextFactory.createContext(null); } http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextMultipleScriptsTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextMultipleScriptsTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextMultipleScriptsTest.java index bdc33bf..ac87c4a 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextMultipleScriptsTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextMultipleScriptsTest.java @@ -24,11 +24,11 @@ import static org.apache.sysml.api.mlcontext.ScriptFactory.dmlFromFile; import java.io.File; import org.apache.spark.sql.SparkSession; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.api.mlcontext.MLContext; import org.apache.sysml.api.mlcontext.Matrix; import org.apache.sysml.api.mlcontext.Script; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.utils.TestUtils; import org.junit.After; @@ -82,8 +82,8 @@ public class MLContextMultipleScriptsTest extends AutomatedTestBase private static void runMLContextTestMultipleScript(RUNTIME_PLATFORM platform, boolean wRead) { - RUNTIME_PLATFORM oldplatform = DMLScript.rtplatform; - DMLScript.rtplatform = platform; + RUNTIME_PLATFORM oldplatform = ConfigurationManager.getExecutionMode(); + ConfigurationManager.getDMLOptions().setExecutionMode(platform); //create mlcontext SparkSession spark = createSystemMLSparkSession("MLContextMultipleScriptsTest", "local"); @@ -109,7 +109,7 @@ public class MLContextMultipleScriptsTest extends AutomatedTestBase System.out.println(z); } finally { - DMLScript.rtplatform = oldplatform; + ConfigurationManager.getDMLOptions().setExecutionMode(oldplatform); // stop underlying spark context to allow single jvm tests (otherwise the // next test that tries to create a SparkContext would fail) http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextScratchCleanupTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextScratchCleanupTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextScratchCleanupTest.java index de3ab3c..eaee0e2 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextScratchCleanupTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextScratchCleanupTest.java @@ -24,11 +24,11 @@ import static org.apache.sysml.api.mlcontext.ScriptFactory.dmlFromFile; import java.io.File; import org.apache.spark.sql.SparkSession; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.api.mlcontext.MLContext; import org.apache.sysml.api.mlcontext.Matrix; import org.apache.sysml.api.mlcontext.Script; +import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.utils.TestUtils; import org.junit.After; @@ -82,8 +82,8 @@ public class MLContextScratchCleanupTest extends AutomatedTestBase private static void runMLContextTestMultipleScript(RUNTIME_PLATFORM platform, boolean wRead) { - RUNTIME_PLATFORM oldplatform = DMLScript.rtplatform; - DMLScript.rtplatform = platform; + RUNTIME_PLATFORM oldplatform = ConfigurationManager.getExecutionMode(); + ConfigurationManager.getDMLOptions().setExecutionMode(platform); //create mlcontext SparkSession spark = createSystemMLSparkSession("MLContextScratchCleanupTest", "local"); @@ -110,7 +110,7 @@ public class MLContextScratchCleanupTest extends AutomatedTestBase throw new RuntimeException(ex); } finally { - DMLScript.rtplatform = oldplatform; + ConfigurationManager.getDMLOptions().setExecutionMode(oldplatform); // stop underlying spark context to allow single jvm tests (otherwise the // next test that tries to create a SparkContext would fail) http://git-wip-us.apache.org/repos/asf/systemml/blob/ae268a9e/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java b/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java index 95b671a..3d94532 100644 --- a/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java +++ b/src/test/java/org/apache/sysml/test/unit/CLIOptionsParserTest.java @@ -24,9 +24,9 @@ import java.util.Map; import org.apache.commons.cli.AlreadySelectedException; import org.apache.commons.cli.MissingOptionException; import org.apache.commons.cli.ParseException; -import org.apache.sysml.api.DMLOptions; import org.apache.sysml.api.DMLScript; import org.apache.sysml.api.mlcontext.ScriptType; +import org.apache.sysml.conf.DMLOptions; import org.apache.sysml.utils.Explain; import org.junit.Assert; import org.junit.Test;
