Repository: systemml Updated Branches: refs/heads/master 140f1e1b7 -> 215a1cc6b
http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java deleted file mode 100644 index df40724..0000000 --- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicBlocksizeOrFirst.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.hops.globalopt.gdfresolve; - -import org.apache.sysml.hops.globalopt.RewriteConfig; - -public class GDFMismatchHeuristicBlocksizeOrFirst extends GDFMismatchHeuristic -{ - - @Override - public String getName(){ - return "BLOCKSIZE_OR_FIRST"; - } - - @Override - public boolean resolveMismatch( RewriteConfig currRc, RewriteConfig newRc ) - { - //check for blocksize mismatch - if( currRc.getBlockSize() != newRc.getBlockSize() ) - { - //choose the new rewrite config if its blocksize is larger than - //the current (intuition: we generally prefer larger blocksizes - //because this often enables better physical operators with constraints - //like ncol(X) <= blocksize) - return (currRc.getBlockSize() < newRc.getBlockSize()); - } - - //return the current rewrite configuration (first come first served) - //if the previous check for blocksize mismatch failed - return false; - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java deleted file mode 100644 index a96bc33..0000000 --- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/GDFMismatchHeuristicFirst.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.hops.globalopt.gdfresolve; - -import org.apache.sysml.hops.globalopt.RewriteConfig; - -public class GDFMismatchHeuristicFirst extends GDFMismatchHeuristic -{ - - @Override - public String getName(){ - return "FIRST"; - } - - @Override - public boolean resolveMismatch( RewriteConfig currRc, RewriteConfig newRc ) - { - //always return the current rewrite configuration (first come first served) - return false; - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java b/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java deleted file mode 100644 index 9846850..0000000 --- a/src/main/java/org/apache/sysml/hops/globalopt/gdfresolve/MismatchHeuristicFactory.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.hops.globalopt.gdfresolve; - -import org.apache.sysml.hops.globalopt.gdfresolve.GDFMismatchHeuristic.MismatchHeuristicType; -import org.apache.sysml.runtime.DMLRuntimeException; - -public abstract class MismatchHeuristicFactory -{ - - public static GDFMismatchHeuristic createMismatchHeuristic( MismatchHeuristicType type ) - throws DMLRuntimeException - { - switch( type ) { - case FIRST: - return new GDFMismatchHeuristicFirst(); - case BLOCKSIZE_OR_FIRST: - return new GDFMismatchHeuristicBlocksizeOrFirst(); - - default: - throw new DMLRuntimeException("Unsupported mismatch heuristic: "+type); - } - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/main/java/org/apache/sysml/utils/Explain.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/Explain.java b/src/main/java/org/apache/sysml/utils/Explain.java index b72fe99..e0a9c92 100644 --- a/src/main/java/org/apache/sysml/utils/Explain.java +++ b/src/main/java/org/apache/sysml/utils/Explain.java @@ -39,9 +39,6 @@ import org.apache.sysml.hops.UnaryOp; import org.apache.sysml.hops.codegen.cplan.CNode; import org.apache.sysml.hops.codegen.cplan.CNodeMultiAgg; import org.apache.sysml.hops.codegen.cplan.CNodeTpl; -import org.apache.sysml.hops.globalopt.gdfgraph.GDFLoopNode; -import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode; -import org.apache.sysml.hops.globalopt.gdfgraph.GDFNode.NodeType; import org.apache.sysml.hops.ipa.FunctionCallGraph; import org.apache.sysml.lops.Lop; import org.apache.sysml.parser.DMLProgram; @@ -474,22 +471,6 @@ public class Explain return explainCNode(node, level); } - public static String explainGDFNodes( ArrayList<GDFNode> gdfnodes ) - throws DMLRuntimeException - { - return explainGDFNodes(gdfnodes, 0); - } - - public static String explainGDFNodes( ArrayList<GDFNode> gdfnodes, int level ) - throws DMLRuntimeException - { - StringBuilder sb = new StringBuilder(); - HashSet<Long> memo = new HashSet<>(); - for( GDFNode gnode : gdfnodes ) - sb.append(explainGDFNode(gnode, level, memo)); - return sb.toString(); - } - /** * Counts the number of compiled MRJob/Spark instructions in the * given runtime program. @@ -980,104 +961,7 @@ public class Explain return sb.toString(); } - - ////////////// - // internal explain GDFNODE - /** - * Do a post-order traverse through the GDFNode DAG and explain each GDFNode. - * Note: nodes referring to literalops are suppressed. - * - * @param gnode GDF node - * @param level offset - * @param memo memoization table - * @return string explanation - * @throws DMLRuntimeException if DMLRuntimeException occurs - */ - private static String explainGDFNode(GDFNode gnode, int level, HashSet<Long> memo) - throws DMLRuntimeException - { - //basic memoization via memo table since gnode has no visit status - if( memo.contains(gnode.getID()) || - gnode.getNodeType()==NodeType.HOP_NODE && gnode.getHop() instanceof LiteralOp ) - { - return ""; - } - - StringBuilder sb = new StringBuilder(); - String offset = createOffset(level); - - for( GDFNode input : gnode.getInputs() ) - sb.append(explainGDFNode(input, level, memo)); - - //indentation - sb.append(offset); - - //hop id - String deps = null; - if( SHOW_DATA_DEPENDENCIES ) { - sb.append("("+gnode.getID()+") "); - - StringBuilder childs = new StringBuilder(); - childs.append(" ("); - boolean childAdded = false; - for( GDFNode input : gnode.getInputs() ) { - childs.append(childAdded?",":""); - childs.append(input.getID()); - childAdded = true; - } - childs.append(")"); - if( childAdded ) - deps = childs.toString(); - } - - //operation string - if( gnode instanceof GDFLoopNode ) //LOOP NODES - { - GDFLoopNode lgnode = (GDFLoopNode) gnode; - String offset2 = createOffset(level+1); - sb.append(lgnode.explain(deps)+"\n"); //loop header - sb.append(offset2+"PRED:\n"); - sb.append(explainGDFNode(lgnode.getLoopPredicate(),level+2, memo)); - sb.append(offset2+"BODY:\n"); - //note: memo table and already done child explain prevents redundancy - for( Entry<String,GDFNode> root : lgnode.getLoopOutputs().entrySet() ) { - sb.append(explainGDFNode(root.getValue(), level+2, memo)); - } - } - else //GENERAL CASE (BASIC/CROSSBLOCK NODES) - { - sb.append(gnode.explain(deps)); - sb.append('\n'); - } - - /* - //matrix characteristics - sb.append(" [" + hop.getDim1() + "," - + hop.getDim2() + "," - + hop.getRowsInBlock() + "," - + hop.getColsInBlock() + "," - + hop.getNnz() + "]"); - - //memory estimates - sb.append(" [" + showMem(hop.getInputMemEstimate(), false) + "," - + showMem(hop.getIntermediateMemEstimate(), false) + "," - + showMem(hop.getOutputMemEstimate(), false) + " -> " - + showMem(hop.getMemEstimate(), true) + "]"); - - //exec type - if (hop.getExecType() != null) - sb.append(", " + hop.getExecType()); - */ - - - //memoization - memo.add(gnode.getID()); - - return sb.toString(); - } - - ////////////// // internal explain RUNTIME http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java deleted file mode 100644 index af5be94..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregCG.java +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.gdfo; - -import java.io.File; -import java.util.HashMap; - -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.lops.LopProperties.ExecType; -import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - */ -public class GDFOLinregCG extends AutomatedTestBase -{ - - private final static String TEST_NAME1 = "LinregCG"; - private final static String TEST_DIR = "functions/gdfo/"; - private final static String TEST_CLASS_DIR = TEST_DIR + GDFOLinregCG.class.getSimpleName() + "/"; - private final static String TEST_CONF = "SystemML-config-globalopt.xml"; - private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); - - private final static double eps = 1e-5; - - private final static int rows = 1468; - private final static int cols = 1007; - - private final static double sparsity1 = 0.7; //dense - private final static double sparsity2 = 0.1; //sparse - - private final static int intercept = 0; - private final static double epsilon = 0.000000001; - private final static double maxiter = 10; - - @Override - public void setUp() - { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "w" })); - } - - @Test - public void testGDFOLinregCGDenseCP() - { - runGDFOTest(TEST_NAME1, false, ExecType.CP); - } - - @Test - public void testGDFOLinregCGSparseCP() - { - runGDFOTest(TEST_NAME1, true, ExecType.CP); - } - - /* - @Test - public void testGDFOLinregCGDenseMR() - { - runGDFOTest(TEST_NAME1, false, ExecType.MR); - } - - @Test - public void testGDFOLinregCGSparseMR() - { - runGDFOTest(TEST_NAME1, true, ExecType.MR); - } - - @Test - public void testGDFOLinregCGDenseSP() - { - runGDFOTest(TEST_NAME1, false, ExecType.SPARK); - } - - @Test - public void testGDFOLinregCGSparseSP() - { - runGDFOTest(TEST_NAME1, true, ExecType.SPARK); - } - */ - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - */ - private void runGDFOTest( String testname,boolean sparse, ExecType instType) - { - //rtplatform for MR - RUNTIME_PLATFORM platformOld = rtplatform; - switch( instType ){ - case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; - case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; - default: rtplatform = RUNTIME_PLATFORM.HYBRID; break; - } - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK ) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - String TEST_NAME = testname; - TestConfiguration config = getTestConfiguration(TEST_NAME); - loadTestConfiguration(config); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{ "-explain", //"hops", - "-args", input("X"), input("y"), - String.valueOf(intercept), String.valueOf(epsilon), - String.valueOf(maxiter), output("w")}; - - rCmd = getRCmd(inputDir(), String.valueOf(intercept),String.valueOf(epsilon), - String.valueOf(maxiter), expectedDir()); - - //generate actual datasets - double[][] X = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity2:sparsity1, 7); - writeInputMatrixWithMTD("X", X, true); - double[][] y = getRandomMatrix(rows, 1, 0, 10, 1.0, 3); - writeInputMatrixWithMTD("y", y, true); - - runTest(true, false, null, -1); - runRScript(true); - - //compare matrices - HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("w"); - HashMap<CellIndex, Double> rfile = readRMatrixFromFS("w"); - TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); - } - finally - { - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } - - /** - * Override default configuration with custom test configuration to ensure - * scratch space and local temporary directory locations are also updated. - */ - @Override - protected File getConfigTemplateFile() { - // Instrumentation in this test's output log to show custom configuration file used for template. - System.out.println("This test case overrides default configuration with " + TEST_CONF_FILE.getPath()); - return TEST_CONF_FILE; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java deleted file mode 100644 index 61abfea..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDS.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.gdfo; - -import java.io.File; -import java.util.HashMap; - -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.lops.LopProperties.ExecType; -import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - */ -public class GDFOLinregDS extends AutomatedTestBase -{ - - private final static String TEST_NAME1 = "LinregDS"; - private final static String TEST_DIR = "functions/gdfo/"; - private final static String TEST_CLASS_DIR = TEST_DIR + GDFOLinregDS.class.getSimpleName() + "/"; - private final static String TEST_CONF = "SystemML-config-globalopt.xml"; - private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); - - private final static double eps = 1e-8; - - private final static int rows = 1468; - private final static int cols = 1007; - - private final static double sparsity1 = 0.7; //dense - private final static double sparsity2 = 0.1; //sparse - - private final static int intercept = 0; - private final static double lambda = 0.01; - - @Override - public void setUp() - { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "B" })); - } - - @Test - public void testGDFOLinregDSDenseCP() - { - runGDFOTest(TEST_NAME1, false, ExecType.CP); - } - - @Test - public void testGDFOLinregDSSparseCP() - { - runGDFOTest(TEST_NAME1, true, ExecType.CP); - } - - @Test - public void testGDFOLinregDSDenseMR() - { - runGDFOTest(TEST_NAME1, false, ExecType.MR); - } - - @Test - public void testGDFOLinregDSSparseMR() - { - runGDFOTest(TEST_NAME1, true, ExecType.MR); - } - - @Test - public void testGDFOLinregDSDenseSP() - { - runGDFOTest(TEST_NAME1, false, ExecType.SPARK); - } - - @Test - public void testGDFOLinregDSSparseSP() - { - runGDFOTest(TEST_NAME1, true, ExecType.SPARK); - } - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - */ - private void runGDFOTest( String testname,boolean sparse, ExecType instType) - { - //rtplatform for MR - RUNTIME_PLATFORM platformOld = rtplatform; - switch( instType ){ - case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; - case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; - default: rtplatform = RUNTIME_PLATFORM.HYBRID; break; - } - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK ) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - String TEST_NAME = testname; - TestConfiguration config = getTestConfiguration(TEST_NAME); - loadTestConfiguration(config); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{ "-explain","hops", - "-args", input("X"), input("y"), - String.valueOf(intercept), String.valueOf(lambda), output("B")}; - - rCmd = getRCmd(inputDir(), String.valueOf(intercept), String.valueOf(lambda), expectedDir()); - - //generate actual datasets - double[][] X = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity2:sparsity1, 7); - writeInputMatrixWithMTD("X", X, true); - double[][] y = getRandomMatrix(rows, 1, 0, 10, 1.0, 3); - writeInputMatrixWithMTD("y", y, true); - - runTest(true, false, null, -1); - runRScript(true); - - //compare matrices - HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B"); - HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B"); - TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); - } - finally - { - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } - - /** - * Override default configuration with custom test configuration to ensure - * scratch space and local temporary directory locations are also updated. - */ - @Override - protected File getConfigTemplateFile() { - // Instrumentation in this test's output log to show custom configuration file used for template. - System.out.println("This test case overrides default configuration with " + TEST_CONF_FILE.getPath()); - return TEST_CONF_FILE; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java deleted file mode 100644 index f095844..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOLinregDSsimpl.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.gdfo; - -import java.io.File; -import java.util.HashMap; - -import org.junit.Test; -import org.apache.sysml.api.DMLScript; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.lops.LopProperties.ExecType; -import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - */ -public class GDFOLinregDSsimpl extends AutomatedTestBase -{ - - private final static String TEST_NAME1 = "LinregDSsimpl"; - private final static String TEST_DIR = "functions/gdfo/"; - private final static String TEST_CLASS_DIR = TEST_DIR + GDFOLinregDSsimpl.class.getSimpleName() + "/"; - private final static String TEST_CONF = "SystemML-config-globalopt.xml"; - private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); - - private final static double eps = 1e-8; - - private final static int rows = 1468; - private final static int cols = 1007; - - private final static double sparsity1 = 0.7; //dense - private final static double sparsity2 = 0.1; //sparse - - private final static int intercept = 0; - private final static double lambda = 0.01; - - @Override - public void setUp() - { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "B" })); - } - - @Test - public void testGDFOLinregDSDenseCP() - { - runGDFOTest(TEST_NAME1, false, ExecType.CP); - } - - @Test - public void testGDFOLinregDSSparseCP() - { - runGDFOTest(TEST_NAME1, true, ExecType.CP); - } - - @Test - public void testGDFOLinregDSDenseMR() - { - runGDFOTest(TEST_NAME1, false, ExecType.MR); - } - - @Test - public void testGDFOLinregDSSparseMR() - { - runGDFOTest(TEST_NAME1, true, ExecType.MR); - } - - @Test - public void testGDFOLinregDSDenseSP() - { - runGDFOTest(TEST_NAME1, false, ExecType.SPARK); - } - - @Test - public void testGDFOLinregDSSparseSP() - { - runGDFOTest(TEST_NAME1, true, ExecType.SPARK); - } - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - */ - private void runGDFOTest( String testname,boolean sparse, ExecType instType) - { - //rtplatform for MR - RUNTIME_PLATFORM platformOld = rtplatform; - switch( instType ){ - case MR: rtplatform = RUNTIME_PLATFORM.HADOOP; break; - case SPARK: rtplatform = RUNTIME_PLATFORM.SPARK; break; - default: rtplatform = RUNTIME_PLATFORM.HYBRID; break; - } - - boolean sparkConfigOld = DMLScript.USE_LOCAL_SPARK_CONFIG; - if( rtplatform == RUNTIME_PLATFORM.SPARK ) - DMLScript.USE_LOCAL_SPARK_CONFIG = true; - - try - { - String TEST_NAME = testname; - TestConfiguration config = getTestConfiguration(TEST_NAME); - loadTestConfiguration(config); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{ "-explain","hops", - "-args", input("X"), input("y"), - String.valueOf(intercept), String.valueOf(lambda), output("B")}; - - rCmd = getRCmd(inputDir(), String.valueOf(intercept), String.valueOf(lambda), expectedDir()); - - //generate actual datasets - double[][] X = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity2:sparsity1, 7); - writeInputMatrixWithMTD("X", X, true); - double[][] y = getRandomMatrix(rows, 1, 0, 10, 1.0, 3); - writeInputMatrixWithMTD("y", y, true); - - runTest(true, false, null, -1); - runRScript(true); - - //compare matrices - HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("B"); - HashMap<CellIndex, Double> rfile = readRMatrixFromFS("B"); - TestUtils.compareMatrices(dmlfile, rfile, eps, "Stat-DML", "Stat-R"); - } - finally - { - rtplatform = platformOld; - DMLScript.USE_LOCAL_SPARK_CONFIG = sparkConfigOld; - } - } - - /** - * Override default configuration with custom test configuration to ensure - * scratch space and local temporary directory locations are also updated. - */ - @Override - protected File getConfigTemplateFile() { - // Instrumentation in this test's output log to show custom configuration file used for template. - System.out.println("This test case overrides default configuration with " + TEST_CONF_FILE.getPath()); - return TEST_CONF_FILE; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java deleted file mode 100644 index 7f29b9a..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/GDFOMMChainLoop.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.gdfo; - -import java.io.File; -import java.util.HashMap; - -import org.junit.Test; -import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; -import org.apache.sysml.lops.LopProperties.ExecType; -import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - -/** - * - */ -public class GDFOMMChainLoop extends AutomatedTestBase -{ - - private final static String TEST_NAME1 = "MMChainLoop"; - private final static String TEST_DIR = "functions/gdfo/"; - private final static String TEST_CLASS_DIR = TEST_DIR + GDFOMMChainLoop.class.getSimpleName() + "/"; - private final static String TEST_CONF = "SystemML-config-globalopt.xml"; - private final static File TEST_CONF_FILE = new File(SCRIPT_DIR + TEST_DIR, TEST_CONF); - - private final static double eps = 1e-10; - - private final static int rows = 1468; - private final static int cols = 1007; - - private final static double sparsity1 = 0.7; //dense - private final static double sparsity2 = 0.1; //sparse - - private final static double maxiter = 10; - - @Override - public void setUp() - { - TestUtils.clearAssertionInformation(); - addTestConfiguration(TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[] { "w" })); - } - - @Test - public void testMMChainLoopDenseCP() - { - runGDFOTest(TEST_NAME1, false, ExecType.CP); - } - - @Test - public void testMMChainLoopSparseCP() - { - runGDFOTest(TEST_NAME1, true, ExecType.CP); - } - - /** - * - * @param sparseM1 - * @param sparseM2 - * @param instType - */ - private void runGDFOTest( String testname,boolean sparse, ExecType instType) - { - //rtplatform for MR - RUNTIME_PLATFORM platformOld = rtplatform; - rtplatform = (instType==ExecType.MR) ? RUNTIME_PLATFORM.HADOOP : RUNTIME_PLATFORM.HYBRID; - - try - { - String TEST_NAME = testname; - TestConfiguration config = getTestConfiguration(TEST_NAME); - loadTestConfiguration(config); - - /* This is for running the junit test the new way, i.e., construct the arguments directly */ - String HOME = SCRIPT_DIR + TEST_DIR; - fullDMLScriptName = HOME + TEST_NAME + ".dml"; - programArgs = new String[]{ "-explain", //"hops", - "-args", input("X"), input("v"), - String.valueOf(maxiter), output("w")}; - - rCmd = getRCmd(inputDir(), String.valueOf(maxiter), expectedDir()); - - //generate actual datasets - double[][] X = getRandomMatrix(rows, cols, 0, 1, sparse?sparsity2:sparsity1, 7); - writeInputMatrixWithMTD("X", X, true); - double[][] y = getRandomMatrix(cols, 1, 0, 1, 1.0, 3); - writeInputMatrixWithMTD("v", y, true); - - runTest(true, false, null, -1); - runRScript(true); - - //compare matrices - HashMap<CellIndex, Double> dmlfile = readDMLMatrixFromHDFS("w"); - HashMap<CellIndex, Double> rfile = readRMatrixFromFS("w"); - double absEps = rfile.get(new CellIndex(1,1)).doubleValue() * eps; - TestUtils.compareMatrices(dmlfile, rfile, absEps, "Stat-DML", "Stat-R"); - } - finally - { - rtplatform = platformOld; - } - } - - /** - * Override default configuration with custom test configuration to ensure - * scratch space and local temporary directory locations are also updated. - */ - @Override - protected File getConfigTemplateFile() { - // Instrumentation in this test's output log to show custom configuration file used for template. - System.out.println("This test case overrides default configuration with " + TEST_CONF_FILE.getPath()); - return TEST_CONF_FILE; - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java b/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java deleted file mode 100644 index 50b34dd..0000000 --- a/src/test/java/org/apache/sysml/test/integration/functions/gdfo/HashInterestingPropertiesTest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.gdfo; - - -import org.junit.Assert; -import org.junit.Test; - -import org.apache.sysml.hops.globalopt.InterestingProperties; -import org.apache.sysml.hops.globalopt.InterestingProperties.Format; -import org.apache.sysml.hops.globalopt.InterestingProperties.Location; -import org.apache.sysml.hops.globalopt.InterestingProperties.Partitioning; -import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.utils.TestUtils; - -public class HashInterestingPropertiesTest extends AutomatedTestBase -{ - - @Override - public void setUp() - { - TestUtils.clearAssertionInformation(); - } - - @Test - public void testCopyEquivalence() - { - InterestingProperties ips1 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips2 = new InterestingProperties(ips1); - InterestingProperties ips3 = new InterestingProperties(ips2); - - //test hash functions - Assert.assertEquals(ips1.hashCode(), ips2.hashCode()); - Assert.assertEquals(ips2.hashCode(), ips3.hashCode()); - - //test equals (used inside hash maps) - Assert.assertTrue(ips1.equals(ips2)); - Assert.assertTrue(ips2.equals(ips3)); - } - - @Test - public void testValueEquivalence() - { - InterestingProperties ips1 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips2 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips3 = new InterestingProperties(2000, Format.BINARY_BLOCK, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips4 = new InterestingProperties(1000, Format.TEXT_CELL, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips5 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.HDFS, Partitioning.NONE, 1, true); - InterestingProperties ips6 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, Partitioning.COL_WISE, 1, true); - InterestingProperties ips7 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, Partitioning.NONE, 1, false); - - //test hash functions - Assert.assertEquals(ips1.hashCode(), ips2.hashCode()); - - //test equals (used inside hash maps) - Assert.assertTrue(ips1.equals(ips2)); - Assert.assertFalse(ips2.equals(ips3)); - Assert.assertFalse(ips2.equals(ips4)); - Assert.assertFalse(ips2.equals(ips5)); - Assert.assertFalse(ips2.equals(ips6)); - Assert.assertFalse(ips2.equals(ips7)); - } - - @Test - public void testValueEquivalenceWithNull() - { - InterestingProperties ips1 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips2 = new InterestingProperties(1000, null, Location.MEM, Partitioning.NONE, 1, true); - InterestingProperties ips3 = new InterestingProperties(1000, Format.BINARY_BLOCK, null, Partitioning.NONE, 1, true); - InterestingProperties ips4 = new InterestingProperties(1000, Format.BINARY_BLOCK, Location.MEM, null, 1, true); - InterestingProperties ips5 = new InterestingProperties(1000, Format.ANY, Location.MEM, Partitioning.NONE, 1, true); - - //test equals (used inside hash maps) - Assert.assertFalse(ips1.equals(ips2)); - Assert.assertFalse(ips1.equals(ips3)); - Assert.assertFalse(ips1.equals(ips4)); - Assert.assertFalse(ips1.equals(ips5)); - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java index a6b2ea2..39b9652 100644 --- a/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java +++ b/src/test/java/org/apache/sysml/test/integration/mlcontext/MLContextTest.java @@ -407,11 +407,8 @@ public class MLContextTest extends MLContextTestBase { Script script = new Script("print('" + testString + "');", org.apache.sysml.api.mlcontext.ScriptType.DML); ScriptExecutor scriptExecutor = new ScriptExecutor() { - // turn off global data flow optimization check @Override - protected void globalDataFlowOptimization() { - return; - } + protected void showExplanation() {} }; ml.execute(script, scriptExecutor); } @@ -424,11 +421,8 @@ public class MLContextTest extends MLContextTestBase { Script script = new Script("print('" + testString + "')", org.apache.sysml.api.mlcontext.ScriptType.PYDML); ScriptExecutor scriptExecutor = new ScriptExecutor() { - // turn off global data flow optimization check @Override - protected void globalDataFlowOptimization() { - return; - } + protected void showExplanation() {} }; ml.execute(script, scriptExecutor); } http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/LinregCG.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/LinregCG.R b/src/test/scripts/functions/gdfo/LinregCG.R deleted file mode 100644 index 5dcad95..0000000 --- a/src/test/scripts/functions/gdfo/LinregCG.R +++ /dev/null @@ -1,57 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -args <- commandArgs(TRUE) -options(digits=22) -library("Matrix") - -X = readMM(paste(args[1], "X.mtx", sep="")) -y = readMM(paste(args[1], "y.mtx", sep="")) - -intercept = as.integer(args[2]); -eps = as.double(args[3]); -maxiter = as.double(args[4]); - -if( intercept == 1 ){ - ones = matrix(1, nrow(X), 1); - X = cbind(X, ones); -} - -r = -(t(X) %*% y); -p = -r; -norm_r2 = sum(r * r); -w = matrix(0, ncol(X), 1); - -i = 0; -while(i < maxiter) { - q = ((t(X) %*% (X %*% p)) + eps * p); - alpha = norm_r2 / ((t(p) %*% q)[1:1]); - w = w + alpha * p; - old_norm_r2 = norm_r2; - r = r + alpha * q; - norm_r2 = sum(r * r); - beta = norm_r2 / old_norm_r2; - p = -r + beta * p; - i = i + 1; -} - -writeMM(as(w,"CsparseMatrix"), paste(args[5], "w", sep="")) http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/LinregCG.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/LinregCG.dml b/src/test/scripts/functions/gdfo/LinregCG.dml deleted file mode 100644 index 01dcae7..0000000 --- a/src/test/scripts/functions/gdfo/LinregCG.dml +++ /dev/null @@ -1,56 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -X = read($1); -y = read($2); -intercept = $3; -eps = $4; -maxiter = $5; - -if( intercept == 1 ){ - ones = matrix(1, nrow(X), 1); - X = cbind(X, ones); -} - -r = -(t(X) %*% y); -p = -r; -norm_r2 = sum(r * r); -w = matrix(0, rows = ncol(X), cols = 1); - -i = 0; -while(i < maxiter) { - q = ((t(X) %*% (X %*% p)) + eps * p); - alpha = norm_r2 / as.scalar(t(p) %*% q); - w = w + alpha * p; - old_norm_r2 = norm_r2; - r = r + alpha * q; - norm_r2 = sum(r * r); - beta = norm_r2 / old_norm_r2; - p = -r + beta * p; - i = i + 1; -} - -write(w, $6); - - - - http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/LinregDS.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/LinregDS.R b/src/test/scripts/functions/gdfo/LinregDS.R deleted file mode 100644 index 752e862..0000000 --- a/src/test/scripts/functions/gdfo/LinregDS.R +++ /dev/null @@ -1,43 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -args <- commandArgs(TRUE) -options(digits=22) -library("Matrix") - -X = as.matrix(readMM(paste(args[1], "X.mtx", sep=""))) -y = as.matrix(readMM(paste(args[1], "y.mtx", sep=""))) -I = as.vector(matrix(1, ncol(X), 1)); -intercept = as.integer(args[2]) -lambda = as.double(args[3]); - -if( intercept == 1 ){ - ones = matrix(1, nrow(X), 1); - X = cbind(X, ones); - I = as.vector(matrix(1, ncol(X), 1)); -} - -A = t(X) %*% X + diag(I)*lambda; -b = t(X) %*% y; -beta = solve(A, b); - -writeMM(as(beta,"CsparseMatrix"), paste(args[4], "B", sep="")) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/LinregDS.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/LinregDS.dml b/src/test/scripts/functions/gdfo/LinregDS.dml deleted file mode 100644 index a985340..0000000 --- a/src/test/scripts/functions/gdfo/LinregDS.dml +++ /dev/null @@ -1,39 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -X = read($1); -y = read($2); -I = matrix(1, ncol(X), 1); -intercept = $3; -lambda = $4; - -if( intercept == 1 ){ - ones = matrix(1, nrow(X), 1); - X = cbind(X, ones); - I = matrix(1, ncol(X), 1); -} - -A = t(X) %*% X + diag(I)*lambda; -b = t(X) %*% y; -beta = solve(A, b); - -write(beta, $5); http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/LinregDSsimpl.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/LinregDSsimpl.R b/src/test/scripts/functions/gdfo/LinregDSsimpl.R deleted file mode 100644 index 5b90875..0000000 --- a/src/test/scripts/functions/gdfo/LinregDSsimpl.R +++ /dev/null @@ -1,35 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -args <- commandArgs(TRUE) -options(digits=22) -library("Matrix") - -X = as.matrix(readMM(paste(args[1], "X.mtx", sep=""))) -y = as.matrix(readMM(paste(args[1], "y.mtx", sep=""))) -I = as.vector(matrix(1, ncol(X), 1)); -lambda = as.double(args[3]); -A = t(X) %*% X + diag(I)*lambda; -b = t(X) %*% y; -beta = solve(A, b); - -writeMM(as(beta,"CsparseMatrix"), paste(args[4], "B", sep="")) \ No newline at end of file http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/LinregDSsimpl.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/LinregDSsimpl.dml b/src/test/scripts/functions/gdfo/LinregDSsimpl.dml deleted file mode 100644 index 576d1ce..0000000 --- a/src/test/scripts/functions/gdfo/LinregDSsimpl.dml +++ /dev/null @@ -1,31 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -X = read($1); -y = read($2); -I = matrix(1, ncol(X), 1); -lambda = $4; -A = t(X) %*% X + diag(I)*lambda; -b = t(X) %*% y; -beta = solve(A, b); - -write(beta, $5); http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/MMChainLoop.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/MMChainLoop.R b/src/test/scripts/functions/gdfo/MMChainLoop.R deleted file mode 100644 index e2177f7..0000000 --- a/src/test/scripts/functions/gdfo/MMChainLoop.R +++ /dev/null @@ -1,37 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -args <- commandArgs(TRUE) -options(digits=22) -library("Matrix") - -X = readMM(paste(args[1], "X.mtx", sep="")) -v = readMM(paste(args[1], "v.mtx", sep="")) -maxiter = as.double(args[2]); - -i = 0; -while(i < maxiter) { - v = t(X) %*% (X %*% v); - i = i + 1; -} - -writeMM(as(v,"CsparseMatrix"), paste(args[3], "w", sep="")) http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/MMChainLoop.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/MMChainLoop.dml b/src/test/scripts/functions/gdfo/MMChainLoop.dml deleted file mode 100644 index a68c79b..0000000 --- a/src/test/scripts/functions/gdfo/MMChainLoop.dml +++ /dev/null @@ -1,37 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - - -X = read($1); -v = read($2); -maxiter = $3; - -i = 0; -while(i < maxiter) { - v = t(X) %*% (X %*% v); - i = i + 1; -} - -write(v, $4); - - - - http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml b/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml deleted file mode 100644 index 01c4c9a..0000000 --- a/src/test/scripts/functions/gdfo/SystemML-config-globalopt.xml +++ /dev/null @@ -1,26 +0,0 @@ -<!-- - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. ---> - -<root> - <sysml.localtmpdir>/tmp/systemml</sysml.localtmpdir> - <sysml.scratch>scratch_space</sysml.scratch> - <sysml.optlevel>4</sysml.optlevel> - <!-- piggybacked test for custom mapred/mapreduce configurations --> - <mapreduce.task.io.sort.mb>50</mapreduce.task.io.sort.mb> -</root> http://git-wip-us.apache.org/repos/asf/systemml/blob/b3419b09/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java deleted file mode 100644 index 074aec8..0000000 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/gdfo/ZPackageSuite.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.functions.gdfo; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -/** Group together the tests in this package into a single suite so that the Maven build - * won't run two of them at once. */ -@RunWith(Suite.class) [email protected]({ - GDFOLinregCG.class, - GDFOLinregDS.class, - GDFOLinregDSsimpl.class, - GDFOMMChainLoop.class, - HashInterestingPropertiesTest.class -}) - - -/** This class is just a holder for the above JUnit annotations. */ -public class ZPackageSuite { - -}
