Repository: systemml Updated Branches: refs/heads/master c13a1b04c -> 11d119877
[SYSTEMML-2436] New tests for all matrix market formats/fields/symmetry This patch is a preparation step for supporting matrix market in the general case, that is all formats (arrays, coordinate), fields (real, integer, pattern), and symmetry (general, symmetric, skew-symmetric). Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/f2a413a0 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/f2a413a0 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/f2a413a0 Branch: refs/heads/master Commit: f2a413a0279d19a3abc4734f0d0902b92fa937ea Parents: c13a1b0 Author: Matthias Boehm <mboe...@gmail.com> Authored: Fri Jul 13 18:06:13 2018 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Fri Jul 13 18:06:13 2018 -0700 ---------------------------------------------------------------------- .../functions/data/MatrixMarketFormatTest.java | 395 +++++++++++++++++++ .../scripts/functions/data/MatrixMarketFormat.R | 30 ++ .../functions/data/MatrixMarketFormat.dml | 26 ++ 3 files changed, 451 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/f2a413a0/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java new file mode 100644 index 0000000..911f35f --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/data/MatrixMarketFormatTest.java @@ -0,0 +1,395 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.data; + +import org.junit.Test; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.util.Iterator; + +import org.apache.commons.lang.NotImplementedException; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.runtime.io.IOUtilFunctions; +import org.apache.sysml.runtime.matrix.data.IJV; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class MatrixMarketFormatTest extends AutomatedTestBase +{ + private final static String TEST_NAME = "MatrixMarketFormat"; + private final static String TEST_DIR = "functions/data/"; + private final static String TEST_CLASS_DIR = TEST_DIR + MatrixMarketFormatTest.class.getSimpleName() + "/"; + + private final static int dim = 1200; + private final static double sparsity = 0.1; + + private enum MMFormat { + COORDINATE, + ARRAY, + } + + private enum MMField { + REAL, + INTEGER, + COMPLEX, + PATTERN, + } + + private enum MMSymmetry { + GENERAL, + SYMMETRIC, + SKEW_SYMMETRIC, //- instead _ + } + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration( TEST_NAME, + new TestConfiguration(TEST_CLASS_DIR, TEST_NAME, new String[] { "R", "C" }) ); + } + + @Test + public void testMMCooRealGeneralCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.GENERAL, ExecType.CP); + } + + @Test + public void testMMCooRealGeneralSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.GENERAL, ExecType.SPARK); + } + + @Test + public void testMMCooRealGeneralMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.GENERAL, ExecType.MR); + } + + @Test + public void testMMCooRealSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMCooRealSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMCooRealSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMCooRealSkewSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMCooRealSkewSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMCooRealSkewSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMCooIntegerGeneralCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.CP); + } + + @Test + public void testMMCooIntegerGeneralSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.SPARK); + } + + @Test + public void testMMCooIntegerGeneralMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.MR); + } + + @Test + public void testMMCooIntegerSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMCooIntegerSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMCooIntegerSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMCooIntegerSkewSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMCooIntegerSkewSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMCooIntegerSkewSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMCooPatternGeneralCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.CP); + } + + @Test + public void testMMCooPatternGeneralSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.SPARK); + } + + @Test + public void testMMCooPatternGeneralMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.GENERAL, ExecType.MR); + } + + @Test + public void testMMCooPatternSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMCooPatternSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMCooPatternSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.COORDINATE, MMField.PATTERN, MMSymmetry.SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMArrRealGeneralCP() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.CP); + } + + @Test + public void testMMArrRealGeneralSp() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.SPARK); + } + + @Test + public void testMMArrRealGeneralMR() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.GENERAL, ExecType.MR); + } + + @Test + public void testMMArrRealSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMArrRealSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMArrRealSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMArrRealSkewSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMArrRealSkewSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMArrRealSkewSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.REAL, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMArrIntegerGeneralCP() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.CP); + } + + @Test + public void testMMArrIntegerGeneralSp() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.SPARK); + } + + @Test + public void testMMArrIntegerGeneralMR() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.GENERAL, ExecType.MR); + } + + @Test + public void testMMArrIntegerSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMArrIntegerSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMArrIntegerSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SYMMETRIC, ExecType.MR); + } + + @Test + public void testMMArrIntegerSkewSymmetricCP() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.CP); + } + + @Test + public void testMMArrIntegerSkewSymmetricSp() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.SPARK); + } + + @Test + public void testMMArrIntegerSkewSymmetricMR() { + runMatrixMarketFormatTest(MMFormat.ARRAY, MMField.INTEGER, MMSymmetry.SKEW_SYMMETRIC, ExecType.MR); + } + + private void runMatrixMarketFormatTest(MMFormat fmt, MMField field, MMSymmetry symmetry, ExecType et) + { + //rtplatform for MR + RUNTIME_PLATFORM platformOld = rtplatform; + switch( et ){ + case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; + case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; + default: rtplatform = RUNTIME_PLATFORM.SINGLE_NODE; break; + } + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + try + { + TestConfiguration config = getTestConfiguration(TEST_NAME); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullRScriptName = HOME + TEST_NAME + ".R"; + fullDMLScriptName = HOME + TEST_NAME + ".dml"; + programArgs = new String[]{"-args", input("X"), output("R"), output("C") }; + rCmd = "Rscript" + " " + fullRScriptName + " " + + input("X") + " " + expected("R") + " " + expected("C"); + + generateAndWriteMMInput(input("X"), fmt, field, symmetry); + + runTest(true, false, null, -1); + runRScript(true); + + //compare row and column aggregates + TestUtils.compareMatrices(readDMLMatrixFromHDFS("R"), + readRMatrixFromFS("R"), 1e-10, "Stat-DML", "Stat-R"); + TestUtils.compareMatrices(readDMLMatrixFromHDFS("C"), + readRMatrixFromFS("C"), 1e-10, "Stat-DML", "Stat-R"); + } + catch (IOException e) { + throw new RuntimeException(e); + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + } + } + + private void generateAndWriteMMInput(String fname, MMFormat fmt, MMField field, MMSymmetry symmetry) + throws IOException + { + int rows = dim; + int cols = (symmetry==MMSymmetry.GENERAL) ? dim/3 : dim; + MatrixBlock tmp = MatrixBlock.randOperations( + rows, cols, sparsity, -10, 10, "uniform", 7); + + String header = "%%MatrixMarket matrix " + + fmt.name().toLowerCase() + " " + + field.name().toLowerCase() + " " + + symmetry.name().toLowerCase().replace("_", "-") + "\n"; + String meta = rows + " " + cols + ((fmt == MMFormat.COORDINATE) ? + " " + tmp.getNonZeros() : "") + "\n"; + + Path path = new Path( fname ); + FileSystem fs = IOUtilFunctions.getFileSystem(path); + + try( BufferedWriter br = new BufferedWriter(new OutputStreamWriter(fs.create(path,true))) ) + { + br.write(header); + br.write(meta); + + if( fmt == MMFormat.ARRAY ) { + for(int j=0; j<tmp.getNumColumns(); j++) { + int bi = (symmetry == MMSymmetry.GENERAL) ? 0 : + (symmetry == MMSymmetry.SYMMETRIC) ? j : j+1; + for(int i=bi; i<tmp.getNumRows(); i++) { + double val = tmp.quickGetValue(i, j); + br.write(String.valueOf((field == MMField.INTEGER) ? + (int) val : val) + "\n" ); + } + } + } + else { //COORDINATE + if( tmp.isInSparseFormat() ) { + StringBuilder sb = new StringBuilder(); + Iterator<IJV> iter = tmp.getSparseBlockIterator(); + while( iter.hasNext() ) { + IJV cell = iter.next(); + if( (symmetry == MMSymmetry.SYMMETRIC && cell.getJ() > cell.getI()) + || (symmetry == MMSymmetry.SKEW_SYMMETRIC && cell.getJ() >= cell.getI())) + continue; + sb.append(cell.getI()+1); + sb.append(' '); + sb.append(cell.getJ()+1); + if( field != MMField.PATTERN ) { + sb.append(' '); + sb.append(String.valueOf((field == MMField.INTEGER) ? + (int) cell.getV() : cell.getV())); + } + sb.append('\n'); + br.write( sb.toString() ); //same as append + sb.setLength(0); + } + } + else { + //always sparse in above used setup + throw new NotImplementedException(); + } + } + } + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/f2a413a0/src/test/scripts/functions/data/MatrixMarketFormat.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/data/MatrixMarketFormat.R b/src/test/scripts/functions/data/MatrixMarketFormat.R new file mode 100644 index 0000000..870cb73 --- /dev/null +++ b/src/test/scripts/functions/data/MatrixMarketFormat.R @@ -0,0 +1,30 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +A = as.matrix(readMM(args[1])) +R = as.matrix(rowSums(A)); +C = t(as.matrix(colSums(A))); +writeMM(as(R,"CsparseMatrix"), args[2]) +writeMM(as(C,"CsparseMatrix"), args[3]) http://git-wip-us.apache.org/repos/asf/systemml/blob/f2a413a0/src/test/scripts/functions/data/MatrixMarketFormat.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/data/MatrixMarketFormat.dml b/src/test/scripts/functions/data/MatrixMarketFormat.dml new file mode 100644 index 0000000..5535e4c --- /dev/null +++ b/src/test/scripts/functions/data/MatrixMarketFormat.dml @@ -0,0 +1,26 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = read($1); +R = rowSums(A); +C = colSums(A); +write(R, $2); +write(C, $3);
