[SYSTEMML-2020] Fix codegen outer handling of sparse side inputs The codegen outer template uses cache blocking for uncompressed input data and column-wise access for compressed data. Both render the use of side inputs with sparse cursors (row-major) invalid, leading to incorrect results. This patch address this issue by using simple dense/sparse side inputs only.
Furthermore, this patch also fixes the selection of the main driver for codegen outer operations according to the involved operations. This fixes issues with sparse side inputs that have less non-zeros than the main driver input. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/9a6a9bd7 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/9a6a9bd7 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/9a6a9bd7 Branch: refs/heads/master Commit: 9a6a9bd762820e55a845f78ff1e25278bdfc01c9 Parents: 079b493 Author: Matthias Boehm <[email protected]> Authored: Sun Nov 19 22:56:59 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Sun Nov 19 22:56:59 2017 -0800 ---------------------------------------------------------------------- .../codegen/template/TemplateOuterProduct.java | 10 +- .../sysml/runtime/codegen/SpoofCellwise.java | 6 + .../runtime/codegen/SpoofOuterProduct.java | 51 +++-- .../functions/codegen/SparseSideInputTest.java | 206 +++++++++++++++++++ .../functions/codegen/SparseSideInput1.R | 31 +++ .../functions/codegen/SparseSideInput1.dml | 27 +++ .../functions/codegen/SparseSideInput2.R | 31 +++ .../functions/codegen/SparseSideInput2.dml | 27 +++ .../functions/codegen/SparseSideInput3.R | 31 +++ .../functions/codegen/SparseSideInput3.dml | 27 +++ .../functions/codegen/SparseSideInput4.R | 33 +++ .../functions/codegen/SparseSideInput4.dml | 29 +++ .../functions/codegen/ZPackageSuite.java | 1 + 13 files changed, 487 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java index 188bac2..12b8795 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateOuterProduct.java @@ -178,10 +178,12 @@ public class TemplateOuterProduct extends TemplateBase { CNode cdata2 = tmp.get(hop.getInput().get(1).getHopID()); String primitiveOpName = ((BinaryOp)hop).getOp().toString(); - if( TemplateUtils.isMatrix(hop.getInput().get(0)) && cdata1 instanceof CNodeData ) - inHops2.put("_X", hop.getInput().get(0)); - if( TemplateUtils.isMatrix(hop.getInput().get(1)) && cdata2 instanceof CNodeData ) - inHops2.put("_X", hop.getInput().get(1)); + if( HopRewriteUtils.isBinarySparseSafe(hop) ) { + if( TemplateUtils.isMatrix(hop.getInput().get(0)) && cdata1 instanceof CNodeData ) + inHops2.put("_X", hop.getInput().get(0)); + if( TemplateUtils.isMatrix(hop.getInput().get(1)) && cdata2 instanceof CNodeData ) + inHops2.put("_X", hop.getInput().get(1)); + } //add lookups if required cdata1 = TemplateUtils.wrapLookupIfNecessary(cdata1, hop.getInput().get(0)); http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java index 86c4ece..5acd5b4 100644 --- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java +++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofCellwise.java @@ -375,6 +375,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl MatrixBlock out, int m, int n, boolean sparseSafe, int rl, int ru) throws DMLRuntimeException { + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //compressed data is access in a column-major order + if( _type == CellType.NO_AGG ) { long lnnz = executeCompressedNoAgg(a, b, scalars, out, m, n, sparseSafe, rl, ru); if( out.isInSparseFormat() ) @@ -401,6 +404,9 @@ public abstract class SpoofCellwise extends SpoofOperator implements Serializabl private double executeCompressedAndAgg(CompressedMatrixBlock a, SideInput[] b, double[] scalars, int m, int n, boolean sparseSafe, int rl, int ru) throws DMLRuntimeException { + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //compressed data is access in a column-major order + //numerically stable aggregation for sum/sum_sq if( _aggOp == AggOp.SUM || _aggOp == AggOp.SUM_SQ ) return executeCompressedAggSum(a, b, scalars, m, n, sparseSafe, rl, ru); http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java index 4d4da51..e99ebe1 100644 --- a/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java +++ b/src/main/java/org/apache/sysml/runtime/codegen/SpoofOuterProduct.java @@ -280,7 +280,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator MatrixBlock a = inputs.get(0); try - { + { ExecutorService pool = Executors.newFixedThreadPool(numThreads); ArrayList<ParExecTask> tasks = new ArrayList<>(); //create tasks (for wdivmm-left, parallelization over columns; @@ -345,7 +345,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator //cache-conscious blocking: due to blocksize constraint (default 1000), //a blocksize of 16 allows to fit blocks of UV into L2 cache (256KB) - SideInput[] lb = createSparseSideInputs(b); + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //cache blocking would lead to non-sequential access + final int blocksizeIJ = 16; //u/v block (max at typical L2 size) int cix = 0; //blocked execution @@ -359,7 +361,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator for( int j=bj, vix=bj*k; j<bjmin; j++, vix+=k) if( a[ix+j] != 0 ) { cix = (type == OutProdType.LEFT_OUTER_PRODUCT) ? vix : uix; - genexecDense( a[ix+j], u, uix, v, vix, lb, scalars, c, cix, m, n, k, i, j); + genexecDense( a[ix+j], u, uix, v, vix, b, scalars, c, cix, m, n, k, i, j); } } } @@ -371,7 +373,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator //cache-conscious blocking: due to blocksize constraint (default 1000), //a blocksize of 16 allows to fit blocks of UV into L2 cache (256KB) - SideInput[] lb = createSparseSideInputs(b); + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //cache blocking would lead to non-sequential access + final int blocksizeIJ = 16; //u/v block (max at typical L2 size) //blocked execution double sum = 0; @@ -385,9 +389,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator for( int j=bj, vix=bj*k; j<bjmin; j++, vix+=k) if( a[ix+j] != 0 ) { if(type == OutProdType.CELLWISE_OUTER_PRODUCT) - c[ix+j] = genexecCellwise( a[ix+j], u, uix, v, vix, lb, scalars, m, n, k, i, j ); + c[ix+j] = genexecCellwise( a[ix+j], u, uix, v, vix, b, scalars, m, n, k, i, j ); else - sum += genexecCellwise( a[ix+j], u, uix, v, vix, lb, scalars, m, n, k, i, j); + sum += genexecCellwise( a[ix+j], u, uix, v, vix, b, scalars, m, n, k, i, j); } } if( type != OutProdType.CELLWISE_OUTER_PRODUCT ) @@ -397,7 +401,6 @@ public abstract class SpoofOuterProduct extends SpoofOperator private void executeSparse(SparseBlock sblock, double[] u, double[] v, SideInput[] b, double[] scalars, double[] c, int m, int n, int k, long nnz, OutProdType type, int rl, int ru, int cl, int cu) { - SideInput[] lb = createSparseSideInputs(b); boolean left = (_outerProductType== OutProdType.LEFT_OUTER_PRODUCT); //approach: iterate over non-zeros of w, selective mm computation @@ -410,7 +413,8 @@ public abstract class SpoofOuterProduct extends SpoofOperator { //for ultra-sparse matrices, we do not allocate the index array because //its allocation and maintenance can dominate the total runtime. - + SideInput[] lb = createSparseSideInputs(b); + //core wdivmm block matrix mult for( int i=rl, uix=rl*k; i<ru; i++, uix+=k ) { if( sblock.isEmpty(i) ) continue; @@ -430,6 +434,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator } else //sparse { + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //cache blocking would lead to non-sequential access + final int blocksizeJ = left ? Math.max(8,Math.min(L2_CACHESIZE/(k*8), blocksizeI)) : blocksizeI; int[] curk = new int[Math.min(blocksizeI,ru-rl)]; @@ -457,7 +464,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator int index = wpos + curk[i-bi]; for( ; index<wpos+wlen && wix[index]<bjmin; index++ ) { - genexecDense(wval[index], u, uix, v, wix[index]*k, lb, scalars, c, + genexecDense(wval[index], u, uix, v, wix[index]*k, b, scalars, c, (left ? wix[index]*k : uix), m, n, k, i, wix[index]); } curk[i-bi] = index - wpos; @@ -470,7 +477,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator private void executeCellwiseSparse(SparseBlock sblock, double[] u, double[] v, SideInput[] b, double[] scalars, MatrixBlock out, int m, int n, int k, long nnz, OutProdType type, int rl, int ru, int cl, int cu ) { - SideInput[] lb = createSparseSideInputs(b); + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //cache blocking would lead to non-sequential access + final int blocksizeIJ = (int) (8L*m*n/nnz); int[] curk = new int[Math.min(blocksizeIJ, ru-rl)]; @@ -495,11 +504,11 @@ public abstract class SpoofOuterProduct extends SpoofOperator if( type == OutProdType.CELLWISE_OUTER_PRODUCT ) for( ; index<wpos+wlen && wix[index]<bjmin; index++ ) c[wix[index]] = genexecCellwise( wval[index], - u, uix, v, wix[index]*k, lb, scalars, m, n, k, i, wix[index] ); + u, uix, v, wix[index]*k, b, scalars, m, n, k, i, wix[index] ); else for( ; index<wpos+wlen && wix[index]<bjmin; index++ ) tmp += genexecCellwise( wval[index], - u, uix, v, wix[index]*k, lb, scalars, m, n, k, i, wix[index]); + u, uix, v, wix[index]*k, b, scalars, m, n, k, i, wix[index]); curk[i-bi] = index - wpos; } } @@ -526,7 +535,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator int index = wpos + curk[i-bi]; for( ; index<wpos+wlen && wix[index]<bjmin; index++ ) { c.append(i, wix[index], genexecCellwise( wval[index], u, uix, v, - wix[index]*k, lb, scalars, m, n, k, i, wix[index] )); + wix[index]*k, b, scalars, m, n, k, i, wix[index] )); } curk[i-bi] = index - wpos; } @@ -538,7 +547,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator private void executeCompressed(CompressedMatrixBlock a, double[] u, double[] v, SideInput[] b, double[] scalars, double[] c, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu) { - SideInput[] lb = createSparseSideInputs(b); + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //compressed data is access in a column-major order + boolean left = (_outerProductType==OutProdType.LEFT_OUTER_PRODUCT); Iterator<IJV> iter = !left ? a.getIterator(rl, ru, false) : @@ -547,7 +558,7 @@ public abstract class SpoofOuterProduct extends SpoofOperator IJV cell = iter.next(); int uix = cell.getI() * k; int vix = cell.getJ() * k; - genexecDense(cell.getV(), u, uix, v, vix, lb, scalars, c, + genexecDense(cell.getV(), u, uix, v, vix, b, scalars, c, left ? vix : uix, m, n, k, cell.getI(), cell.getJ()); } } @@ -555,7 +566,9 @@ public abstract class SpoofOuterProduct extends SpoofOperator private void executeCellwiseCompressed(CompressedMatrixBlock a, double[] u, double[] v, SideInput[] b, double[] scalars, MatrixBlock out, int m, int n, int k, OutProdType type, int rl, int ru, int cl, int cu ) { - SideInput[] lb = createSparseSideInputs(b); + //NOTE: we don't create sparse side inputs w/ row-major cursors because + //compressed data is access in a column-major order + double[] c = out.getDenseBlock(); SparseBlock csblock = out.getSparseBlock(); @@ -568,15 +581,15 @@ public abstract class SpoofOuterProduct extends SpoofOperator if( out.isInSparseFormat() ) { csblock.allocate(cell.getI()); csblock.append(cell.getI(), cell.getJ(), - genexecCellwise(cell.getV(), u, uix, v, vix, lb, scalars, m, n, k, cell.getI(), cell.getJ())); + genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ())); } else { c[cell.getI()*n+cell.getJ()] = - genexecCellwise(cell.getV(), u, uix, v, vix, lb, scalars, m, n, k, cell.getI(), cell.getJ()); + genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ()); } } else { - c[0] += genexecCellwise(cell.getV(), u, uix, v, vix, lb, scalars, m, n, k, cell.getI(), cell.getJ()); + c[0] += genexecCellwise(cell.getV(), u, uix, v, vix, b, scalars, m, n, k, cell.getI(), cell.getJ()); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java new file mode 100644 index 0000000..508cd52 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/SparseSideInputTest.java @@ -0,0 +1,206 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.codegen; + +import java.io.File; +import java.util.HashMap; + +import org.junit.Assert; +import org.junit.Test; +import org.apache.sysml.api.DMLScript; +import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; +import org.apache.sysml.hops.OptimizerUtils; +import org.apache.sysml.lops.LopProperties.ExecType; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class SparseSideInputTest extends AutomatedTestBase +{ + private static final String TEST_NAME = "sparseSideInput"; + private static final String TEST_NAME1 = TEST_NAME+"1"; //row sum(X/rowSums(X)+Y) + private static final String TEST_NAME2 = TEST_NAME+"2"; //cell sum(abs(X^2)+Y) + private static final String TEST_NAME3 = TEST_NAME+"3"; //magg sum(X^2), sum(X+Y) + private static final String TEST_NAME4 = TEST_NAME+"4"; //outer sum((X!=0) * (U %*% t(V) - Y)) + + private static final String TEST_DIR = "functions/codegen/"; + private static final String TEST_CLASS_DIR = TEST_DIR + SparseSideInputTest.class.getSimpleName() + "/"; + private static String TEST_CONF1 = "SystemML-config-codegen.xml"; + private static String TEST_CONF2 = "SystemML-config-codegen-compress.xml"; + private static String TEST_CONF = TEST_CONF1; + + private static final int rows = 1798; + private static final int cols = 784; + private static final double sparsity = 0.1; + private static final double eps = Math.pow(10, -7); + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + for(int i=1; i<=4; i++) + addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) ); + } + + @Test + public void testCodegenRowULASparseSideInputCP() { + testCodegenIntegration( TEST_NAME1, false, ExecType.CP ); + } + + @Test + public void testCodegenRowCLASparseSideInputCP() { + testCodegenIntegration( TEST_NAME1, true, ExecType.CP ); + } + + @Test + public void testCodegenRowULASparseSideInputSP() { + testCodegenIntegration( TEST_NAME1, false, ExecType.SPARK ); + } + + @Test + public void testCodegenRowCLASparseSideInputSP() { + testCodegenIntegration( TEST_NAME1, true, ExecType.SPARK ); + } + + @Test + public void testCodegenCellULASparseSideInputCP() { + testCodegenIntegration( TEST_NAME2, false, ExecType.CP ); + } + + @Test + public void testCodegenCellCLASparseSideInputCP() { + testCodegenIntegration( TEST_NAME2, true, ExecType.CP ); + } + + @Test + public void testCodegenCellULASparseSideInputSP() { + testCodegenIntegration( TEST_NAME2, false, ExecType.SPARK ); + } + + @Test + public void testCodegenCellCLASparseSideInputSP() { + testCodegenIntegration( TEST_NAME2, true, ExecType.SPARK ); + } + + @Test + public void testCodegenMaggULASparseSideInputCP() { + testCodegenIntegration( TEST_NAME3, false, ExecType.CP ); + } + + @Test + public void testCodegenMaggCLASparseSideInputCP() { + testCodegenIntegration( TEST_NAME3, true, ExecType.CP ); + } + + @Test + public void testCodegenMaggULASparseSideInputSP() { + testCodegenIntegration( TEST_NAME3, false, ExecType.SPARK ); + } + + @Test + public void testCodegenMaggCLASparseSideInputSP() { + testCodegenIntegration( TEST_NAME3, true, ExecType.SPARK ); + } + + @Test + public void testCodegenOuterULASparseSideInputCP() { + testCodegenIntegration( TEST_NAME4, false, ExecType.CP ); + } + + @Test + public void testCodegenOuterCLASparseSideInputCP() { + testCodegenIntegration( TEST_NAME4, true, ExecType.CP ); + } + + @Test + public void testCodegenOuterULASparseSideInputSP() { + testCodegenIntegration( TEST_NAME4, false, ExecType.SPARK ); + } + + @Test + public void testCodegenOuterCLASparseSideInputSP() { + testCodegenIntegration( TEST_NAME4, true, ExecType.SPARK ); + } + + private void testCodegenIntegration( String testname, boolean compress, ExecType instType ) + { + boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; + RUNTIME_PLATFORM platformOld = rtplatform; + switch( instType ) { + case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; + case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; + default: rtplatform = RUNTIME_PLATFORM.HYBRID_SPARK; break; + } + + boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; + if( rtplatform == RUNTIME_PLATFORM.SPARK || rtplatform == RUNTIME_PLATFORM.HYBRID_SPARK ) + DMLScript.USE_LOCAL_SPARK_CONFIG = true; + + try + { + TEST_CONF = compress ? TEST_CONF2 : TEST_CONF1; + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = false; + + TestConfiguration config = getTestConfiguration(testname); + loadTestConfiguration(config); + + String HOME = SCRIPT_DIR + TEST_DIR; + fullDMLScriptName = HOME + testname + ".dml"; + programArgs = new String[]{"-explain", "-stats", "-args", + input("X"), input("Y"), output("R") }; + + fullRScriptName = HOME + testname + ".R"; + rCmd = getRCmd(inputDir(), expectedDir()); + + //generate inputs + double[][] X = getRandomMatrix(rows, cols, 0, 1, sparsity, 7); + double[][] Y = getRandomMatrix(rows, cols, 0, 1, sparsity, 3); + writeInputMatrixWithMTD("X", X, true); + writeInputMatrixWithMTD("Y", Y, true); + + //run dml and r scripts + runTest(true, false, null, -1); + runRScript(true); + + //compare matrices + HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("R"); + HashMap<CellIndex, Double> rfile = readRMatrixFromFS("R"); + TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); + Assert.assertTrue(heavyHittersContainsSubString("spoof") + || heavyHittersContainsSubString("sp_spoof")); + } + finally { + rtplatform = platformOld; + DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; + OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION = oldFlag; + OptimizerUtils.ALLOW_AUTO_VECTORIZATION = true; + OptimizerUtils.ALLOW_OPERATOR_FUSION = true; + TEST_CONF = TEST_CONF2; + } + } + + @Override + protected File getConfigTemplateFile() { + // Instrumentation in this test's output log to show custom configuration file used for template. + File f = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); + System.out.println("This test case overrides default configuration with " + f.getPath()); + return f; + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput1.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput1.R b/src/test/scripts/functions/codegen/SparseSideInput1.R new file mode 100644 index 0000000..5acdb01 --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput1.R @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = readMM(paste(args[1], "X.mtx", sep="")) +Y = readMM(paste(args[1], "Y.mtx", sep="")) + +R = as.matrix(sum(X/rowSums(X)+Y)); + +writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text") http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput1.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput1.dml b/src/test/scripts/functions/codegen/SparseSideInput1.dml new file mode 100644 index 0000000..1e0b241 --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput1.dml @@ -0,0 +1,27 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); +Y = read($2); + +R = as.matrix(sum(X/rowSums(X)+Y)); + +write(R, $3) http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput2.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput2.R b/src/test/scripts/functions/codegen/SparseSideInput2.R new file mode 100644 index 0000000..55872a4 --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput2.R @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = readMM(paste(args[1], "X.mtx", sep="")) +Y = readMM(paste(args[1], "Y.mtx", sep="")) + +R = as.matrix(sum(abs(X^2)+Y)); + +writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text") http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput2.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput2.dml b/src/test/scripts/functions/codegen/SparseSideInput2.dml new file mode 100644 index 0000000..9c0ec59 --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput2.dml @@ -0,0 +1,27 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); +Y = read($2); + +R = as.matrix(sum(abs(X^2)+Y)); + +write(R, $3) http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput3.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput3.R b/src/test/scripts/functions/codegen/SparseSideInput3.R new file mode 100644 index 0000000..106e59c --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput3.R @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = readMM(paste(args[1], "X.mtx", sep="")) +Y = readMM(paste(args[1], "Y.mtx", sep="")) + +R = as.matrix(sum(X^2)+sum(X+Y)); + +writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text") http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput3.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput3.dml b/src/test/scripts/functions/codegen/SparseSideInput3.dml new file mode 100644 index 0000000..253ba49 --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput3.dml @@ -0,0 +1,27 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); +Y = read($2); + +R = as.matrix(sum(X^2)+sum(X+Y)); + +write(R, $3) http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput4.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput4.R b/src/test/scripts/functions/codegen/SparseSideInput4.R new file mode 100644 index 0000000..a109983 --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput4.R @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = readMM(paste(args[1], "X.mtx", sep="")) +Y = readMM(paste(args[1], "Y.mtx", sep="")) + +U = matrix(1, nrow(X), 10); +V = matrix(1, ncol(X), 10); +R = as.matrix(sum((X!=0)*((U%*%t(V))/10+Y))); + +writeMM(as(R,"CsparseMatrix"), paste(args[2], "R", sep=""), format="text") http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test/scripts/functions/codegen/SparseSideInput4.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/SparseSideInput4.dml b/src/test/scripts/functions/codegen/SparseSideInput4.dml new file mode 100644 index 0000000..37c25ce --- /dev/null +++ b/src/test/scripts/functions/codegen/SparseSideInput4.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = read($1); +Y = read($2); + +U = matrix(1, nrow(X), 10); +V = matrix(1, ncol(X), 10); +R = as.matrix(sum((X!=0)*((U%*%t(V))/10+Y))); + +write(R, $3) http://git-wip-us.apache.org/repos/asf/systemml/blob/9a6a9bd7/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java index 75b66a1..596d673 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/codegen/ZPackageSuite.java @@ -51,6 +51,7 @@ import org.junit.runners.Suite; OuterProdTmplTest.class, RowAggTmplTest.class, RowVectorComparisonTest.class, + SparseSideInputTest.class, SumProductChainTest.class, })
