Repository: systemml Updated Branches: refs/heads/master aaaa903cb -> 8c11b5d82
[HOTFIX] Disable transposed matrix multiplication optimization as cusparseDcsrmm2 failures. - Also, added minor fixes for GPU tests. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/8c11b5d8 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/8c11b5d8 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/8c11b5d8 Branch: refs/heads/master Commit: 8c11b5d828c1ec7aa6cb4d668ec793f571bff7cb Parents: aaaa903 Author: Niketan Pansare <npan...@us.ibm.com> Authored: Thu Dec 7 07:43:04 2017 -0800 Committer: Niketan Pansare <npan...@us.ibm.com> Committed: Thu Dec 7 07:44:56 2017 -0800 ---------------------------------------------------------------------- .../java/org/apache/sysml/hops/AggBinaryOp.java | 10 ++-- .../runtime/matrix/data/LibMatrixCUDA.java | 1 + .../org/apache/sysml/api/ml/ScriptsUtils.scala | 2 +- .../sysml/test/gpu/AggregateUnaryOpTests.java | 1 + .../org/apache/sysml/test/gpu/AppendTest.java | 1 + .../apache/sysml/test/gpu/BinaryOpTests.java | 1 + .../org/apache/sysml/test/gpu/GPUTests.java | 52 +++++++++++++++----- .../gpu/MatrixMatrixElementWiseOpTests.java | 1 + .../test/gpu/MatrixMultiplicationOpTest.java | 1 + .../sysml/test/gpu/NeuralNetworkOpTests.java | 1 + .../org/apache/sysml/test/gpu/ReorgOpTests.java | 1 + .../sysml/test/gpu/RightIndexingTests.java | 1 + .../gpu/ScalarMatrixElementwiseOpTests.java | 1 + .../org/apache/sysml/test/gpu/UnaryOpTests.java | 1 + 14 files changed, 58 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/main/java/org/apache/sysml/hops/AggBinaryOp.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java index 30dc5e7..a7b6599 100644 --- a/src/main/java/org/apache/sysml/hops/AggBinaryOp.java +++ b/src/main/java/org/apache/sysml/hops/AggBinaryOp.java @@ -157,7 +157,7 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop input2.getDim1(), input2.getDim2(), mmtsj, chain, _hasLeftPMInput ); switch( _method ){ case TSMM: - return true; + return false; // TODO: Disabling any fused transa optimization in 1.0 release. case MAPMM_CHAIN: return false; case PMM: @@ -668,8 +668,12 @@ public class AggBinaryOp extends Hop implements MultiThreadedHop if (et == ExecType.GPU) { Hop h1 = getInput().get(0); Hop h2 = getInput().get(1); - boolean leftTrans = HopRewriteUtils.isTransposeOperation(h1); - boolean rightTrans = HopRewriteUtils.isTransposeOperation(h2); + // Since GPU backend is in experimental mode, rewrite optimization can be skipped. + // CuSPARSE's cusparsecsrmm2 fails with only following parameters, but passes for all other settings: + // transa=1 transb=1 m=300 n=300 k=300 ldb=300 ldc=300 + // Hence, we disable hope rewrite optimization. + boolean leftTrans = false; // HopRewriteUtils.isTransposeOperation(h1); + boolean rightTrans = false; // HopRewriteUtils.isTransposeOperation(h2); Lop left = !leftTrans ? h1.constructLops() : h1.getInput().get(0).constructLops(); Lop right = !rightTrans ? h2.constructLops() : http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java index bbc4727..1433b5a 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibMatrixCUDA.java @@ -480,6 +480,7 @@ public class LibMatrixCUDA { } if (ec.getGPUContext(0) != gCtx) throw new DMLRuntimeException("GPU : Invalid internal state, the GPUContext set with the ExecutionContext is not the same used to run this LibMatrixCUDA function"); + if(isInSparseFormat(gCtx, left)) { // For sparse TSMM, invoke matmult (TODO: possible performance improvement) LibMatrixCuMatMult.matmult(ec, gCtx, instName, left, left, outputName, isLeftTransposed, !isLeftTransposed); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala ---------------------------------------------------------------------- diff --git a/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala b/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala index 016457e..ffbdf2f 100644 --- a/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala +++ b/src/main/scala/org/apache/sysml/api/ml/ScriptsUtils.scala @@ -25,7 +25,7 @@ import java.io.InputStreamReader import org.apache.sysml.runtime.DMLRuntimeException object ScriptsUtils { - var systemmlHome = System.getenv("SYSTEMML_HOME") + var systemmlHome:String = null // System.getenv("SYSTEMML_HOME") type SparkDataType = org.apache.spark.sql.Dataset[_] // org.apache.spark.sql.DataFrame for Spark 1.x http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/AggregateUnaryOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/AggregateUnaryOpTests.java b/src/test/java/org/apache/sysml/test/gpu/AggregateUnaryOpTests.java index 59e9cb1..78a7c1b 100644 --- a/src/test/java/org/apache/sysml/test/gpu/AggregateUnaryOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/AggregateUnaryOpTests.java @@ -31,6 +31,7 @@ public class AggregateUnaryOpTests extends UnaryOpTestsBase { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/AppendTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/AppendTest.java b/src/test/java/org/apache/sysml/test/gpu/AppendTest.java index 930241b..f359d48 100644 --- a/src/test/java/org/apache/sysml/test/gpu/AppendTest.java +++ b/src/test/java/org/apache/sysml/test/gpu/AppendTest.java @@ -41,6 +41,7 @@ public class AppendTest extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/BinaryOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/BinaryOpTests.java b/src/test/java/org/apache/sysml/test/gpu/BinaryOpTests.java index f3d2b21..7949fcb 100644 --- a/src/test/java/org/apache/sysml/test/gpu/BinaryOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/BinaryOpTests.java @@ -37,6 +37,7 @@ public class BinaryOpTests extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/GPUTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/GPUTests.java b/src/test/java/org/apache/sysml/test/gpu/GPUTests.java index a83b110..501e545 100644 --- a/src/test/java/org/apache/sysml/test/gpu/GPUTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/GPUTests.java @@ -24,6 +24,8 @@ import java.util.Formatter; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; import org.apache.spark.sql.SparkSession; import org.apache.sysml.api.mlcontext.MLContext; @@ -79,9 +81,17 @@ public abstract class GPUTests extends AutomatedTestBase { else if(FLOATING_POINT_PRECISION.equals("single")) return SINGLE_PRECISION_THRESHOLD; else throw new RuntimeException("Unsupported precision:" + FLOATING_POINT_PRECISION); } - + + // force junit to run only 1 GPU test at a time to avoid cascading failures. + Lock sequential = new ReentrantLock(); + @Override + public void setUp() { + sequential.lock(); + + } @After public void tearDown() { + sequential.unlock(); clearGPUMemory(); super.tearDown(); } @@ -242,6 +252,16 @@ public abstract class GPUTests extends AutomatedTestBase { */ private void assertEqualMatrices(Matrix expected, Matrix actual) { try { + // Faster way to compare two matrices + MLContext cpuMLC = new MLContext(spark); + String scriptStr = "num_mismatch = sum((abs(X - Y) / X) > " + getTHRESHOLD() + ");"; + Script script = ScriptFactory.dmlFromString(scriptStr).in("X", expected).in("Y", actual).out("num_mismatch"); + long num_mismatch = cpuMLC.execute(script).getLong("num_mismatch"); + cpuMLC.close(); + if(num_mismatch == 0) + return; + + // If error, print the actual incorrect values MatrixBlock expectedMB = expected.toMatrixObject().acquireRead(); MatrixBlock actualMB = actual.toMatrixObject().acquireRead(); @@ -323,19 +343,25 @@ public abstract class GPUTests extends AutomatedTestBase { */ protected List<Object> runOnGPU(SparkSession spark, String scriptStr, Map<String, Object> inputs, List<String> outStrs) { - MLContext gpuMLC = new MLContext(spark); - gpuMLC.setConfigProperty("sysml.floating.point.precision", FLOATING_POINT_PRECISION); - gpuMLC.setGPU(true); - gpuMLC.setForceGPU(true); - gpuMLC.setStatistics(true); - List<Object> outputs = new ArrayList<>(); - Script script = ScriptFactory.dmlFromString(scriptStr).in(inputs).out(outStrs); - for (String outStr : outStrs) { - Object output = gpuMLC.execute(script).get(outStr); - outputs.add(output); + // Ensure that only one instance of ml.execute runs at a time to avoid incorrect memory estimates + // and other side effects. + synchronized(GPUTests.class) { + MLContext gpuMLC = new MLContext(spark); + gpuMLC.setConfigProperty("sysml.floating.point.precision", FLOATING_POINT_PRECISION); + if(IGNORE_CLEAR_MEMORY_BUG) + gpuMLC.setConfigProperty("sysml.gpu.eager.cudaFree", "true"); + gpuMLC.setGPU(true); + gpuMLC.setForceGPU(true); + gpuMLC.setStatistics(true); + List<Object> outputs = new ArrayList<>(); + Script script = ScriptFactory.dmlFromString(scriptStr).in(inputs).out(outStrs); + for (String outStr : outStrs) { + Object output = gpuMLC.execute(script).get(outStr); + outputs.add(output); + } + gpuMLC.close(); + return outputs; } - gpuMLC.close(); - return outputs; } /** http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/MatrixMatrixElementWiseOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/MatrixMatrixElementWiseOpTests.java b/src/test/java/org/apache/sysml/test/gpu/MatrixMatrixElementWiseOpTests.java index 490befa..c949924 100644 --- a/src/test/java/org/apache/sysml/test/gpu/MatrixMatrixElementWiseOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/MatrixMatrixElementWiseOpTests.java @@ -42,6 +42,7 @@ public class MatrixMatrixElementWiseOpTests extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/MatrixMultiplicationOpTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/MatrixMultiplicationOpTest.java b/src/test/java/org/apache/sysml/test/gpu/MatrixMultiplicationOpTest.java index cbc3563..227d0e8 100644 --- a/src/test/java/org/apache/sysml/test/gpu/MatrixMultiplicationOpTest.java +++ b/src/test/java/org/apache/sysml/test/gpu/MatrixMultiplicationOpTest.java @@ -36,6 +36,7 @@ public class MatrixMultiplicationOpTest extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/NeuralNetworkOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/NeuralNetworkOpTests.java b/src/test/java/org/apache/sysml/test/gpu/NeuralNetworkOpTests.java index c57e997..d11af5e 100644 --- a/src/test/java/org/apache/sysml/test/gpu/NeuralNetworkOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/NeuralNetworkOpTests.java @@ -92,6 +92,7 @@ public class NeuralNetworkOpTests extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/ReorgOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/ReorgOpTests.java b/src/test/java/org/apache/sysml/test/gpu/ReorgOpTests.java index b5b71f8..6a11c9c 100644 --- a/src/test/java/org/apache/sysml/test/gpu/ReorgOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/ReorgOpTests.java @@ -40,6 +40,7 @@ public class ReorgOpTests extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java b/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java index 90b08ae..63cbdd1 100644 --- a/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/RightIndexingTests.java @@ -39,6 +39,7 @@ public class RightIndexingTests extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/ScalarMatrixElementwiseOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/ScalarMatrixElementwiseOpTests.java b/src/test/java/org/apache/sysml/test/gpu/ScalarMatrixElementwiseOpTests.java index 7ceeb0f..2e3321f 100644 --- a/src/test/java/org/apache/sysml/test/gpu/ScalarMatrixElementwiseOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/ScalarMatrixElementwiseOpTests.java @@ -42,6 +42,7 @@ public class ScalarMatrixElementwiseOpTests extends GPUTests { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME); http://git-wip-us.apache.org/repos/asf/systemml/blob/8c11b5d8/src/test/java/org/apache/sysml/test/gpu/UnaryOpTests.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/gpu/UnaryOpTests.java b/src/test/java/org/apache/sysml/test/gpu/UnaryOpTests.java index 510766e..9b1f79e 100644 --- a/src/test/java/org/apache/sysml/test/gpu/UnaryOpTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/UnaryOpTests.java @@ -31,6 +31,7 @@ public class UnaryOpTests extends UnaryOpTestsBase { @Override public void setUp() { + super.setUp(); TestUtils.clearAssertionInformation(); addTestConfiguration(TEST_DIR, TEST_NAME); getAndLoadTestConfiguration(TEST_NAME);
