Repository: systemml Updated Branches: refs/heads/master 8d8004121 -> 581077712
[MINOR] New tests for cholesky builtin function, incl. related fixes Closes #710. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/58107771 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/58107771 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/58107771 Branch: refs/heads/master Commit: 581077712cb1749db67859d4c8a9332ce95c5384 Parents: 8d80041 Author: Janardhan Pulivarthi <[email protected]> Authored: Sun Dec 24 06:13:53 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sun Dec 24 07:10:33 2017 -0800 ---------------------------------------------------------------------- .../IPAPassRemoveUnnecessaryCheckpoints.java | 13 +- .../cp/BuiltinUnaryCPInstruction.java | 15 +-- .../cp/MatrixBuiltinCPInstruction.java | 2 +- .../runtime/matrix/data/LibCommonsMath.java | 3 +- .../functions/unary/matrix/CholeskyTest.java | 128 +++++++++++++++++++ .../scripts/functions/unary/matrix/cholesky.dml | 50 ++++++++ .../functions/unary/matrix/ZPackageSuite.java | 1 + 7 files changed, 195 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java b/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java index d90b4b6..11bce2f 100644 --- a/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java +++ b/src/main/java/org/apache/sysml/hops/ipa/IPAPassRemoveUnnecessaryCheckpoints.java @@ -214,15 +214,14 @@ public class IPAPassRemoveUnnecessaryCheckpoints extends IPAPass throws HopsException { List<StatementBlock> sbs = dmlp.getStatementBlocks(); - - if( sbs.size()==1 & !(sbs.get(0) instanceof IfStatementBlock - || sbs.get(0) instanceof WhileStatementBlock - || sbs.get(0) instanceof ForStatementBlock) ) - { + + if (sbs.size() == 1 && !(sbs.get(0) instanceof IfStatementBlock + || sbs.get(0) instanceof WhileStatementBlock + || sbs.get(0) instanceof ForStatementBlock)) { //recursively process all dag roots - if( sbs.get(0).getHops()!=null ) { + if (sbs.get(0).getHops() != null) { Hop.resetVisitStatus(sbs.get(0).getHops()); - for( Hop root : sbs.get(0).getHops() ) + for (Hop root : sbs.get(0).getHops()) rRemoveCheckpointReadWrite(root); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java index 6b055a4..9f72d62 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/BuiltinUnaryCPInstruction.java @@ -27,19 +27,19 @@ import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.functionobjects.Builtin; import org.apache.sysml.runtime.functionobjects.ValueFunction; import org.apache.sysml.runtime.instructions.InstructionUtils; +import org.apache.sysml.runtime.matrix.data.LibCommonsMath; import org.apache.sysml.runtime.matrix.operators.Operator; import org.apache.sysml.runtime.matrix.operators.SimpleOperator; import org.apache.sysml.runtime.matrix.operators.UnaryOperator; public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction { - protected BuiltinUnaryCPInstruction(Operator op, CPOperand in, CPOperand out, String opcode, - String istr) { + protected BuiltinUnaryCPInstruction(Operator op, CPOperand in, CPOperand out, String opcode, String istr) { super(CPType.BuiltinUnary, op, in, out, opcode, istr); } public static BuiltinUnaryCPInstruction parseInstruction ( String str ) - throws DMLRuntimeException + throws DMLRuntimeException { CPOperand in = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN); CPOperand out = new CPOperand("", ValueType.UNKNOWN, DataType.UNKNOWN); @@ -49,8 +49,7 @@ public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction { ValueFunction func = null; //print or stop or cumulative aggregates - if( parts.length==4 ) - { + if( parts.length==4 ) { opcode = parts[0]; in.split(parts[1]); out.split(parts[2]); @@ -61,15 +60,15 @@ public abstract class BuiltinUnaryCPInstruction extends UnaryCPInstruction { else return new ScalarBuiltinCPInstruction(new SimpleOperator(func), in, out, opcode, str); } - else //2+1, general case - { + else { //2+1, general case opcode = parseUnaryInstruction(str, in, out); func = Builtin.getBuiltinFnObject(opcode); if(in.getDataType() == DataType.SCALAR) return new ScalarBuiltinCPInstruction(new SimpleOperator(func), in, out, opcode, str); else if(in.getDataType() == DataType.MATRIX) - return new MatrixBuiltinCPInstruction(new UnaryOperator(func), in, out, opcode, str); + return new MatrixBuiltinCPInstruction(LibCommonsMath.isSupportedUnaryOperation(opcode) ? + null : new UnaryOperator(func), in, out, opcode, str); } return null; http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java index 84e3ebb..b120dab 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/MatrixBuiltinCPInstruction.java @@ -35,7 +35,6 @@ public class MatrixBuiltinCPInstruction extends BuiltinUnaryCPInstruction { public void processInstruction(ExecutionContext ec) throws DMLRuntimeException { - UnaryOperator u_op = (UnaryOperator) _optr; String output_name = output.getName(); String opcode = getOpcode(); @@ -44,6 +43,7 @@ public class MatrixBuiltinCPInstruction extends BuiltinUnaryCPInstruction { ec.setMatrixOutput(output_name, retBlock, getExtendedOpcode()); } else { + UnaryOperator u_op = (UnaryOperator) _optr; MatrixBlock inBlock = ec.getMatrixInput(input1.getName(), getExtendedOpcode()); MatrixBlock retBlock = (MatrixBlock) (inBlock.unaryOperations(u_op, new MatrixBlock())); http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java index 30aec56..2755c28 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/data/LibCommonsMath.java @@ -281,7 +281,8 @@ public class LibCommonsMath if ( !in.isSquare() ) throw new DMLRuntimeException("Input to cholesky() must be square matrix -- given: a " + in.getRowDimension() + "x" + in.getColumnDimension() + " matrix."); - CholeskyDecomposition cholesky = new CholeskyDecomposition(in); + CholeskyDecomposition cholesky = new CholeskyDecomposition(in, 1e-14, + CholeskyDecomposition.DEFAULT_ABSOLUTE_POSITIVITY_THRESHOLD); RealMatrix rmL = cholesky.getL(); return DataConverter.convertToMatrixBlock(rmL.getData()); http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java new file mode 100644 index 0000000..71b50d6 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/unary/matrix/CholeskyTest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.unary.matrix; + +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for the Cholesky matrix factorization + * Note: + * 1. Only tested for dense configuration of matrices (small, & large) + * 2. The remaining tests for matrix dim check, positive definiteness + * already tested at commons-math. + */ + +public class CholeskyTest extends AutomatedTestBase +{ + private final static String TEST_NAME = "cholesky"; + private final static String TEST_DIR = "functions/unary/matrix/"; + private static final String TEST_CLASS_DIR = TEST_DIR + CholeskyTest.class.getSimpleName() + "/"; + + private final static int rows1 = 500; + private final static int rows2 = 2500; + private final static int cols1 = 500; + private final static int cols2 = 2500; + private final static double sparsity = 0.9; + + @Override + public void setUp() { + addTestConfiguration( + TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, + new String[] { "D" }) ); + } + + @Test + public void testCholeskyDenseCP() { + runTestCholesky( rows1, cols1, DMLScript.RUNTIME_PLATFORM.SINGLE_NODE ); + } + + @Test + public void testCholeskyDenseSP() { + runTestCholesky( rows1, cols1, RUNTIME_PLATFORM.SPARK ); + } + + @Test + public void testCholeskyDenseMR() { + runTestCholesky( rows1, cols1, RUNTIME_PLATFORM.HADOOP ); + } + + @Test + public void testCholeskyDenseHybrid() { + runTestCholesky( rows1, cols1, RUNTIME_PLATFORM.HYBRID ); + } + + @Test + public void testLargeCholeskyDenseCP() { + runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.SINGLE_NODE ); + } + + @Test + public void testLargeCholeskyDenseSP() { + runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.SPARK ); + } + + @Test + public void testLargeCholeskyDenseMR() { + runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.HADOOP ); + } + + @Test + public void testLargeCholeskyDenseHybrid() { + runTestCholesky( rows2, cols2, RUNTIME_PLATFORM.HYBRID ); + } + + private void runTestCholesky( int rows, int cols, RUNTIME_PLATFORM rt) { + RUNTIME_PLATFORM rtold = rtplatform; + rtplatform = rt; + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + try { + getAndLoadTestConfiguration(TEST_NAME); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME+ TEST_NAME + ".dml"; + programArgs = new String[]{"-args", input("A"), output("D") }; + + double[][] A = getRandomMatrix(rows, cols, 0, 1, sparsity, 10); + MatrixCharacteristics mc = new MatrixCharacteristics(rows, cols, -1, -1, -1); + writeInputMatrixWithMTD("A", A, false, mc); + + //run tests and compare results + runTest(true, false, null, -1); + Assert.assertEquals(0, readDMLMatrixFromHDFS("D") + .get(new CellIndex(1,1)), 1e-5); + } + finally { + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + rtplatform = rtold; + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/test/scripts/functions/unary/matrix/cholesky.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/unary/matrix/cholesky.dml b/src/test/scripts/functions/unary/matrix/cholesky.dml new file mode 100644 index 0000000..8be1e23 --- /dev/null +++ b/src/test/scripts/functions/unary/matrix/cholesky.dml @@ -0,0 +1,50 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +/* + * DML script to test Cholesky Factorization (cholesky), as a Builtin function. + * + * Reference: + * Theorem 4.2.7, page No. 163 + * "Matrix Computations" by Golub & Van Loan (4th Edition) + * + * Cholesky Factorization: + * If A belongs to R(nxn), is symmetric positive definite, + * then there exists a unique lower triangular G belongs to R(nxn), with positive + * diagonal entries such that. + * A = G %*% t(G); + * + */ + + +X = read($1); + +#1. generate a symmetric positive definite matrix +A = X %*% t(X); + +#2. obtain unique lower triangular matrix, G +G = cholesky(A); + +#3. Recompute the positive definite matrix +B = G %*% t(G); +D = as.matrix(sum(A-B)); + +write(D, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/58107771/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java index c556138..09f4113 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/unary/matrix/ZPackageSuite.java @@ -32,6 +32,7 @@ import org.junit.runners.Suite; ATanTest.class, CastAsScalarTest.class, CosTest.class, + CholeskyTest.class, DiagTest.class, EigenFactorizeTest.class, FullCummaxTest.class,
