[SYSTEMML-593] Fix mlcontext test (missing stop for single jvm test) Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/d6990dcc Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/d6990dcc Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/d6990dcc
Branch: refs/heads/master Commit: d6990dccd7688baa97b31a25f9a6c430e9477e9d Parents: cc5d22c Author: Matthias Boehm <[email protected]> Authored: Sat Aug 6 14:46:19 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sun Aug 7 12:30:43 2016 -0700 ---------------------------------------------------------------------- .../apache/sysml/api/mlcontext/MLContext.java | 14 +++++++- .../context/SparkExecutionContext.java | 10 ++++-- .../integration/mlcontext/MLContextTest.java | 38 +++++++++++++------- 3 files changed, 47 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d6990dcc/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java index 05deec2..85e2143 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/MLContext.java @@ -42,6 +42,7 @@ import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.controlprogram.LocalVariableMap; import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; +import org.apache.sysml.runtime.controlprogram.context.SparkExecutionContext; import org.apache.sysml.runtime.instructions.Instruction; import org.apache.sysml.runtime.instructions.cp.Data; import org.apache.sysml.runtime.instructions.cp.ScalarObject; @@ -56,7 +57,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo; * */ public class MLContext { - /** * Minimum Spark version supported by SystemML. */ @@ -502,4 +502,16 @@ public class MLContext { clearCache(); } + public void close() { + //reset static status (refs to sc / mlcontext) + SparkExecutionContext.resetSparkContextStatic(); + MLContextProxy.setActive(false); + activeMLContext = null; + + //clear local status, but do not stop sc as it + //may be used or stopped externally + clear(); + resetConfig(); + sc = null; + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d6990dcc/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java index c715331..02c900d 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/context/SparkExecutionContext.java @@ -144,8 +144,7 @@ public class SparkExecutionContext extends ExecutionContext * * @return */ - public static JavaSparkContext getSparkContextStatic() - { + public static JavaSparkContext getSparkContextStatic() { initSparkContext(); return _spctx; } @@ -153,6 +152,13 @@ public class SparkExecutionContext extends ExecutionContext /** * */ + public static void resetSparkContextStatic() { + _spctx = null; + } + + /** + * + */ public void close() { synchronized( SparkExecutionContext.class ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/d6990dcc/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java index 5c0eb48..f80b2a2 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java @@ -66,7 +66,9 @@ import org.apache.sysml.api.mlcontext.ScriptExecutor; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.test.integration.AutomatedTestBase; import org.junit.After; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import scala.Tuple2; @@ -79,23 +81,23 @@ public class MLContextTest extends AutomatedTestBase { protected final static String TEST_DIR = "org/apache/sysml/api/mlcontext"; protected final static String TEST_NAME = "MLContext"; - static SparkConf conf; - static JavaSparkContext sc; - MLContext ml; + private static SparkConf conf; + private static JavaSparkContext sc; + private static MLContext ml; + @BeforeClass + public static void setUpClass() { + if (conf == null) + conf = new SparkConf().setAppName("MLContextTest").setMaster("local"); + if (sc == null) + sc = new JavaSparkContext(conf); + ml = new MLContext(sc); + } + @Override public void setUp() { addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); - - if (conf == null) { - conf = new SparkConf().setAppName("MLContextTest").setMaster("local"); - } - if (sc == null) { - sc = new JavaSparkContext(conf); - } - ml = new MLContext(sc); - // ml.setExplain(true); } @Test @@ -1710,4 +1712,16 @@ public class MLContextTest extends AutomatedTestBase { super.tearDown(); } + @AfterClass + public static void tearDownClass() { + //stop spark context to allow single jvm tests (otherwise the + //next test that tries to create a SparkContext would fail) + sc.stop(); + sc = null; + conf = null; + + //clear status mlcontext and spark exec context + ml.close(); + ml = null; + } }
