[SYSTEMML-2271] Include scalable decompositions in integration tests Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/d368644a Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/d368644a Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/d368644a
Branch: refs/heads/master Commit: d368644a2341ead8d5dcb47b6290fe7c7f833c7a Parents: 0209edd Author: Matthias Boehm <[email protected]> Authored: Sat Apr 21 22:17:55 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sat Apr 21 22:20:11 2018 -0700 ---------------------------------------------------------------------- .../runtime/instructions/InstructionUtils.java | 6 + .../test/integration/AutomatedTestBase.java | 6 + .../dml/ScalableDecompositionTest.java | 252 +++++++++++++++++++ .../org/apache/sysml/test/utils/TestUtils.java | 17 +- .../decomp/ScalableDecomposition.dml | 47 ++++ .../integration/applications/ZPackageSuite.java | 2 +- 6 files changed, 319 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java index 3b531dd..588b8e6 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/InstructionUtils.java @@ -87,6 +87,7 @@ import org.apache.sysml.runtime.instructions.gpu.GPUInstruction.GPUINSTRUCTION_T import org.apache.sysml.runtime.instructions.mr.MRInstruction.MRType; import org.apache.sysml.runtime.instructions.spark.SPInstruction.SPType; import org.apache.sysml.runtime.matrix.data.LibCommonsMath; +import org.apache.sysml.runtime.matrix.operators.AggregateBinaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateOperator; import org.apache.sysml.runtime.matrix.operators.AggregateTernaryOperator; import org.apache.sysml.runtime.matrix.operators.AggregateUnaryOperator; @@ -865,4 +866,9 @@ public class InstructionUtils || WeightedUnaryMM.OPCODE.equalsIgnoreCase(opcode) //mapwumm || WeightedUnaryMMR.OPCODE.equalsIgnoreCase(opcode); //redwumm } + + public static AggregateBinaryOperator getMatMultOperator(int k) { + AggregateOperator agg = new AggregateOperator(0, Plus.getPlusFnObject()); + return new AggregateBinaryOperator(Multiply.getMultiplyFnObject(), agg, k); + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index 78f8059..48f6428 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -53,6 +53,7 @@ import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.util.DataConverter; @@ -504,6 +505,11 @@ public abstract class AutomatedTestBase return matrix; } + protected double[][] writeInputMatrixWithMTD(String name, MatrixBlock matrix, boolean bIncludeR) { + double[][] data = DataConverter.convertToDoubleMatrix(matrix); + return writeInputMatrixWithMTD(name, data, bIncludeR); + } + protected double[][] writeInputMatrixWithMTD(String name, double[][] matrix, boolean bIncludeR) { MatrixCharacteristics mc = new MatrixCharacteristics(matrix.length, matrix[0].length, OptimizerUtils.DEFAULT_BLOCKSIZE, OptimizerUtils.DEFAULT_BLOCKSIZE, -1); return writeInputMatrixWithMTD(name, matrix, bIncludeR, mc); http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java b/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java new file mode 100644 index 0000000..607cac0 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/applications/dml/ScalableDecompositionTest.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.applications.dml; + +import java.util.HashMap; + +import org.junit.Test; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.lops.MMTSJ.MMTSJType; +import org.apache.sysml.runtime.instructions.InstructionUtils; +import org.apache.sysml.runtime.matrix.data.LibCommonsMath; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class ScalableDecompositionTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "ScalableDecomposition"; + private final static String TEST_DIR = "applications/decomp/"; + private final static String TEST_CLASS_DIR = + TEST_DIR + ScalableDecompositionTest.class.getSimpleName() + "/"; + + private final static int rows = 1362; + private final static int cols = 1362; + private final static int blen = 200; + private final static double eps = 1e-7; + + private enum DecompType { + CHOLESKY, LU, QR, SOLVE, INVERSE + } + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "C","D","E" })); + } + + @Test + public void testCholeskyCP() { + runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, false, ExecType.CP); + } + + @Test + public void testCholeskyRewritesCP() { + runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, true, ExecType.CP); + } + +// @Test +// public void testCholeskySP() { +// runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, false, ExecType.SPARK); +// } +// +// @Test +// public void testCholeskyRewritesSP() { +// runKMeansTest(TEST_NAME1, DecompType.CHOLESKY, true, ExecType.SPARK); +// } + +// @Test +// public void testLUDecompCP() { +// runKMeansTest(TEST_NAME1, DecompType.LU, false, ExecType.CP); +// } +// +// @Test +// public void testLUDecompRewritesCP() { +// runKMeansTest(TEST_NAME1, DecompType.LU, true, ExecType.CP); +// } + +// @Test +// public void testLUDecompSP() { +// runKMeansTest(TEST_NAME1, DecompType.LU, false, ExecType.SPARK); +// } +// +// @Test +// public void testLUDecompRewritesSP() { +// runKMeansTest(TEST_NAME1, DecompType.LU, true, ExecType.SPARK); +// } + +// @Test +// public void testQRDecompCP() { +// runKMeansTest(TEST_NAME1, DecompType.QR, false, ExecType.CP); +// } +// +// @Test +// public void testQRDecompRewritesCP() { +// runKMeansTest(TEST_NAME1, DecompType.QR, true, ExecType.CP); +// } + +// @Test +// public void testQRDecompSP() { +// runKMeansTest(TEST_NAME1, DecompType.QR, false, ExecType.SPARK); +// } +// +// @Test +// public void testQRDecompRewritesSP() { +// runKMeansTest(TEST_NAME1, DecompType.QR, true, ExecType.SPARK); +// } + +// @Test +// public void testSolveCP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.CP); +// } +// +// @Test +// public void testSolveRewritesCP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.CP); +// } + +// @Test +// public void testSolveSP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.SPARK); +// } +// +// @Test +// public void testSolveRewritesSP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.SPARK); +// } + +// @Test +// public void testInverseCP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.CP); +// } +// +// @Test +// public void testInverseRewritesCP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.CP); +// } + +// @Test +// public void testInverseSP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, false, ExecType.SPARK); +// } +// +// @Test +// public void testInverseRewritesSP() { +// runKMeansTest(TEST_NAME1, DecompType.SOLVE, true, ExecType.SPARK); +// } + + + private void runKMeansTest(String testname, DecompType type, boolean rewrites, ExecType instType) + { + boolean oldFlag1 = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; + boolean oldFlag2 = OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS; + RUNTIME_PLATFORM platformOld = rtplatform; + switch( instType ){ + case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; + default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break; + } + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + try + { + TestConfiguration config = getTestConfiguration(testname); + loadTestConfiguration(config); + + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = rewrites; + OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = rewrites; + + fullDMLScriptName = SCRIPT_DIR + TEST_DIR + testname + ".dml"; + programArgs = new String[]{"-explain","-args", + String.valueOf(type.ordinal()), String.valueOf(blen), + input("A"), input("B"), output("C"), output("D"), output("E") }; + + switch( type ) { + case CHOLESKY: { + MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7); + MatrixBlock AtA = A.transposeSelfMatrixMultOperations(new MatrixBlock(), MMTSJType.LEFT); + writeInputMatrixWithMTD("A", AtA, false); + runTest(true, false, null, -1); + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C"); + MatrixBlock C2 = LibCommonsMath.unaryOperations(AtA, "cholesky"); + TestUtils.compareMatrices(dmlfile, C2, eps); + break; + } + case SOLVE: { + MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7); + MatrixBlock b = MatrixBlock.randOperations(cols, 1, 1.0, -1, 1, "uniform", 3); + MatrixBlock y = A.aggregateBinaryOperations(A, b, new MatrixBlock(), InstructionUtils.getMatMultOperator(1)); + writeInputMatrixWithMTD("A", A, false); + writeInputMatrixWithMTD("B", y, false); + runTest(true, false, null, -1); + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C"); + MatrixBlock C2 = LibCommonsMath.matrixMatrixOperations(A, b, "solve"); + TestUtils.compareMatrices(dmlfile, C2, eps); + break; + } + case LU: { + MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7); + writeInputMatrixWithMTD("A", A, false); + runTest(true, false, null, -1); + MatrixBlock[] C = LibCommonsMath.multiReturnOperations(A, "lu"); + String[] outputs = new String[]{"C","D","E"}; + for(int i=0; i<outputs.length; i++) { + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(outputs[i]); + TestUtils.compareMatrices(dmlfile, C[i], eps); + } + break; + } + case QR: { + MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7); + writeInputMatrixWithMTD("A", A, false); + runTest(true, false, null, -1); + MatrixBlock[] C = LibCommonsMath.multiReturnOperations(A, "qr"); + String[] outputs = new String[]{"C","D","E"}; + for(int i=0; i<outputs.length; i++) { + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS(outputs[i]); + TestUtils.compareMatrices(dmlfile, C[i], eps); + } + break; + } + case INVERSE: { + MatrixBlock A = MatrixBlock.randOperations(rows, cols, 1.0, -5, 10, "uniform", 7); + writeInputMatrixWithMTD("A", A, false); + runTest(true, false, null, -1); + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("C"); + MatrixBlock C2 = LibCommonsMath.unaryOperations(A, "inverse"); + TestUtils.compareMatrices(dmlfile, C2, eps); + break; + } + } + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag1; + OptimizerUtils.ALLOW_INTER_PROCEDURAL_ANALYSIS = oldFlag2; + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/java/org/apache/sysml/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java index 5357be0..545ae17 100644 --- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java +++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java @@ -62,6 +62,7 @@ import org.apache.sysml.runtime.matrix.data.MatrixCell; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.runtime.util.UtilFunctions; import org.apache.sysml.test.integration.AutomatedTestBase; @@ -693,22 +694,18 @@ public class TestUtils assertEquals(expected, actual); } - /** - * - * @param m1 - * @param m2 - * @param tolerance - * @param name1 - * @param name2 - * @param ignoreNaN - * @return - */ public static boolean compareMatrices(HashMap<CellIndex, Double> m1, HashMap<CellIndex, Double> m2, double tolerance, String name1, String name2) { return compareMatrices(m1, m2, tolerance, name1, name2, false); } + public static void compareMatrices(HashMap<CellIndex, Double> m1, MatrixBlock m2, double tolerance) { + double[][] ret1 = convertHashMapToDoubleArray(m1); + double[][] ret2 = DataConverter.convertToDoubleMatrix(m2); + compareMatrices(ret1, ret2, m2.getNumRows(), m2.getNumColumns(), tolerance); + } + /** * Compares two matrices given as HashMaps. The matrix containing more nnz * is iterated and each cell value compared against the corresponding cell http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test/scripts/applications/decomp/ScalableDecomposition.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/applications/decomp/ScalableDecomposition.dml b/src/test/scripts/applications/decomp/ScalableDecomposition.dml new file mode 100644 index 0000000..009ddf1 --- /dev/null +++ b/src/test/scripts/applications/decomp/ScalableDecomposition.dml @@ -0,0 +1,47 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +source("scripts/staging/scalable_linalg/linalg_decomp.dml") as decomp + +A = read($3); + +if( $1 == 0 ) { + C = decomp::Cholesky(A, $2); +} +else if( $1 == 1 ) { + [C, D, E] = decomp::LU(A, $2); +} +else if( $1 == 2 ) { + [C, D] = decomp::QR(A, $2); +} +else if( $1 == 3 ) { + B = read($4); + C = decomp::Solve(A, B, $2); +} +else if( $1 == 4 ) { + C = decomp::Inverse(A, $2); +} + +write(C, $5); +if( exists(D) ) + write(D, $6) +if( exists(E) ) + write(E, $7) http://git-wip-us.apache.org/repos/asf/systemml/blob/d368644a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java index f9d1e91..084b5c6 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/applications/ZPackageSuite.java @@ -27,7 +27,6 @@ import org.junit.runners.Suite; * they should not be run in parallel. */ @RunWith(Suite.class) @Suite.SuiteClasses({ - // .applications.dml package org.apache.sysml.test.integration.applications.dml.ApplyTransformDMLTest.class, org.apache.sysml.test.integration.applications.dml.ArimaDMLTest.class, @@ -45,6 +44,7 @@ import org.junit.runners.Suite; org.apache.sysml.test.integration.applications.dml.NaiveBayesDMLTest.class, org.apache.sysml.test.integration.applications.dml.NaiveBayesParforDMLTest.class, org.apache.sysml.test.integration.applications.dml.PageRankDMLTest.class, + org.apache.sysml.test.integration.applications.dml.ScalableDecompositionTest.class, org.apache.sysml.test.integration.applications.dml.WelchTDMLTest.class, // .applications.pydml package
