Repository: incubator-systemml Updated Branches: refs/heads/master fb5309808 -> 9cdce10d5
[SYSTEMML-1564] Add a Java test suite wrapper around `nn` DML test suite The `nn` library contains it's own DML test suite for gradient checks and unit tests that produces "ERROR: ..." messages if any of the mathematical operations return incorrect results. Note that this has helped to find mathematical bugs in the library & engine that do not result in JVM exceptions. This commit simply adds a Java test harness that wraps these DML tests, parsing the standard out for "ERROR:" messages. A new `scripts` test package has been created, with the aim of eventually testing all of our DML algorithms. Additionally, the ParserWrapper has been updated to look in the `scripts` folder as a fallback, which allows for easy testing of our actual scripts. Closes #474. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/9cdce10d Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/9cdce10d Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/9cdce10d Branch: refs/heads/master Commit: 9cdce10d5ec414dfdf1c90afb7eb6170799a3ac2 Parents: 42ebdef Author: Mike Dusenberry <[email protected]> Authored: Fri Apr 28 17:51:23 2017 -0700 Committer: Mike Dusenberry <[email protected]> Committed: Fri Apr 28 17:54:14 2017 -0700 ---------------------------------------------------------------------- .../org/apache/sysml/parser/ParserWrapper.java | 7 +- .../test/integration/AutomatedTestBase.java | 45 +++++++++-- .../test/integration/scripts/nn/NNTest.java | 81 ++++++++++++++++++++ .../integration/scripts/nn/ZPackageSuite.java | 37 +++++++++ 4 files changed, 163 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/main/java/org/apache/sysml/parser/ParserWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ParserWrapper.java b/src/main/java/org/apache/sysml/parser/ParserWrapper.java index ec479e2..2711b6a 100644 --- a/src/main/java/org/apache/sysml/parser/ParserWrapper.java +++ b/src/main/java/org/apache/sysml/parser/ParserWrapper.java @@ -24,6 +24,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -110,7 +112,10 @@ public abstract class ParserWrapper { LOG.debug("Looking for the following file in the local file system: " + script); if( !LocalFileUtils.validateExternalFilename(script, false) ) throw new LanguageException("Invalid (non-trustworthy) local filename."); - in = new BufferedReader(new FileReader(script)); + if (Files.exists(Paths.get(script))) + in = new BufferedReader(new FileReader(script)); + else // check in scripts/ directory for file (useful for tests) + in = new BufferedReader(new FileReader("scripts/" + script)); } //core script reading http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index a822ae7..9bf227d 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -20,6 +20,7 @@ package org.apache.sysml.test.integration; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -228,6 +229,8 @@ public abstract class AutomatedTestBase private String expectedStdOut; private int iExpectedStdOutState = 0; + private String unexpectedStdOut; + private int iUnexpectedStdOutState = 0; private PrintStream originalPrintStreamStd = null; private String expectedStdErr; @@ -839,7 +842,7 @@ public abstract class AutomatedTestBase String testDirectory = config.getTestDirectory(); if (testDirectory != null) { if (isTargetTestDirectory(testDirectory)) { - baseDirectory = TEST_DATA_DIR + testDirectory; + baseDirectory = TEST_DATA_DIR + testDirectory; sourceDirectory = SCRIPT_DIR + getSourceDirectory(testDirectory); } else { @@ -1465,6 +1468,7 @@ public abstract class AutomatedTestBase || iExpectedStdOutState == 2); assertTrue("expected String did not occur (stderr): " + expectedStdErr, iExpectedStdErrState == 0 || iExpectedStdErrState == 2); + assertFalse("unexpected String occurred: " + unexpectedStdOut, iUnexpectedStdOutState == 1); TestUtils.displayAssertionBuffer(); @@ -1495,7 +1499,7 @@ public abstract class AutomatedTestBase } /** - * Enables expection of a line in standard output stream. + * Enables detection of expected output of a line in standard output stream. * * @param expectedLine */ @@ -1510,8 +1514,6 @@ public abstract class AutomatedTestBase * This class is used to compare the standard output stream against an * expected string. * - * - * */ class ExpectedOutputStream extends OutputStream { private String line = ""; @@ -1544,8 +1546,6 @@ public abstract class AutomatedTestBase * This class is used to compare the standard error stream against an * expected string. * - * - * */ class ExpectedErrorStream extends OutputStream { private String line = ""; @@ -1568,6 +1568,39 @@ public abstract class AutomatedTestBase } /** + * Enables detection of unexpected output of a line in standard output stream. + * + * @param unexpectedLine String that should not occur in stdout. + */ + public void setUnexpectedStdOut(String unexpectedLine) { + this.unexpectedStdOut = unexpectedLine; + originalPrintStreamStd = System.out; + System.setOut(new PrintStream(new UnexpectedOutputStream())); + } + + /** + * This class is used to compare the standard output stream against + * an unexpected string. + */ + class UnexpectedOutputStream extends OutputStream { + private String line = ""; + + @Override + public void write(int b) throws IOException { + line += String.valueOf((char) b); + if (((char) b) == '\n') { + /** new line */ + if (line.contains(unexpectedStdOut)) { + iUnexpectedStdOutState = 1; // error! + } else { + line = ""; // reset buffer + } + } + originalPrintStreamStd.write(b); + } + } + + /** * <p> * Generates a matrix containing easy to debug values in its cells. * </p> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java b/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java new file mode 100644 index 0000000..d86b707 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/scripts/nn/NNTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.scripts.nn; + +import org.apache.spark.sql.SparkSession; +import org.apache.sysml.api.mlcontext.MLContext; +import org.apache.sysml.api.mlcontext.Script; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import static org.apache.sysml.api.mlcontext.ScriptFactory.dmlFromFile; + +/** + * Test the SystemML deep learning library, `nn`. + */ +public class NNTest extends AutomatedTestBase { + + private static final String TEST_NAME = "NNTest"; + private static final String TEST_DIR = "scripts/"; + private static final String TEST_SCRIPT = "scripts/nn/test/run_tests.dml"; + private static final String ERROR_STRING = "ERROR:"; + + private static SparkSession spark; + private static MLContext ml; + + @BeforeClass + public static void setUpClass() { + spark = createSystemMLSparkSession("MLContextTest", "local"); + ml = new MLContext(spark); + } + + @Override + public void setUp() { + addTestConfiguration(TEST_DIR, TEST_NAME); + getAndLoadTestConfiguration(TEST_NAME); + } + + @Test + public void testNNLibrary() { + Script script = dmlFromFile(TEST_SCRIPT); + setUnexpectedStdOut(ERROR_STRING); + ml.execute(script); + } + + @After + public void tearDown() { + super.tearDown(); + } + + @AfterClass + public static void tearDownClass() { + // stop underlying spark context to allow single jvm tests (otherwise the + // next test that tries to create a SparkContext would fail) + spark.stop(); + spark = null; + + // clear status mlcontext and spark exec context + ml.close(); + ml = null; + } +} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/9cdce10d/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java new file mode 100644 index 0000000..145b954 --- /dev/null +++ b/src/test_suites/java/org/apache/sysml/test/integration/scripts/nn/ZPackageSuite.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.scripts.nn; + +import org.junit.runner.RunWith; +import org.junit.runners.Suite; + +/** Group together the tests in this package/related subpackages into a single suite so that the Maven build + * won't run two of them at once. Since the DML and PyDML equivalent tests currently share the same directories, + * they should not be run in parallel. */ +@RunWith(Suite.class) [email protected]({ + NNTest.class, +}) + + +/** This class is just a holder for the above JUnit annotations. */ +public class ZPackageSuite { + +}
