Repository: systemml Updated Branches: refs/heads/master 6f9e1cf9c -> 4eb1b935b
[SYSTEMML-2210] Improved exists builtin function (data-dep ordering) This patch improves the exists builtin function for scenarios where the input matrix is created in the same DAG (or ends of in the same DAG after rewrites) by ordering the instruction schedule according to existing data dependencies and generalizing the runtime. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/0d858347 Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/0d858347 Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/0d858347 Branch: refs/heads/master Commit: 0d858347d5a8d1630b78bb353361b9fa7ea6d629 Parents: 6f9e1cf Author: Matthias Boehm <[email protected]> Authored: Mon Mar 26 22:03:03 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Mon Mar 26 22:03:03 2018 -0700 ---------------------------------------------------------------------- .../org/apache/sysml/parser/DMLTranslator.java | 4 +-- .../cp/AggregateUnaryCPInstruction.java | 7 +++-- .../runtime/instructions/cp/CPOperand.java | 4 +++ .../instructions/cp/VariableCPInstruction.java | 8 ++---- .../functions/misc/ExistsVariableTest.java | 20 ++++++++++--- src/test/scripts/functions/misc/Exists.dml | 30 -------------------- src/test/scripts/functions/misc/Exists1.dml | 30 ++++++++++++++++++++ src/test/scripts/functions/misc/Exists2.dml | 27 ++++++++++++++++++ 8 files changed, 86 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index dd41a03..e0510b5 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -2443,9 +2443,9 @@ public class DMLTranslator case EXISTS: currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), - target.getValueType(), Hop.OpOp1.EXISTS, new LiteralOp(expr.getName())); + target.getValueType(), Hop.OpOp1.EXISTS, expr); break; - + case SUM: currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM, Direction.RowCol, expr); http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java index 249e567..2c74a10 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java @@ -122,9 +122,10 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction ec.setScalarOutput(output_name, new IntObject(rval)); } else if( _type == AUType.EXISTS ) { - //probe existing of variable in symbol table w/o error - boolean rval = ec.getVariables().keySet() - .contains(ec.getScalarInput(input1).getStringValue()); + //probe existence of variable in symbol table w/o error + String varName = input1.isMatrix() ? input1.getName() : + ec.getScalarInput(input1).getStringValue(); + boolean rval = ec.getVariables().keySet().contains(varName); //create and set output scalar ec.setScalarOutput(output_name, new BooleanObject(rval)); } http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java index 2f9b64d..1ca8eab 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/CPOperand.java @@ -69,6 +69,10 @@ public class CPOperand return _dataType.isMatrix(); } + public boolean isScalar() { + return _dataType.isScalar(); + } + public boolean isLiteral() { return _isLiteral; } http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java index 99292ec..e9a6fd0 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/VariableCPInstruction.java @@ -802,12 +802,10 @@ public class VariableCPInstruction extends CPInstruction { */ public static void processRemoveVariableInstruction( ExecutionContext ec, String varname ) { // remove variable from symbol table - Data input1_data = ec.removeVariable(varname); - if ( input1_data == null ) - throw new DMLRuntimeException("Unexpected error: could not find a data object for variable name:" + varname + ", while processing rmvar instruction."); + Data dat = ec.removeVariable(varname); //cleanup matrix data on fs/hdfs (if necessary) - if ( input1_data instanceof CacheableData ) { - ec.cleanupCacheableData( (CacheableData<?>) input1_data ); + if ( dat != null && dat instanceof CacheableData ) { + ec.cleanupCacheableData((CacheableData<?>) dat); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java index 1bf0f34..bdc0497 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java @@ -29,27 +29,39 @@ import org.apache.sysml.test.utils.TestUtils; public class ExistsVariableTest extends AutomatedTestBase { - private final static String TEST_NAME1 = "Exists"; + private final static String TEST_NAME1 = "Exists1"; //for var names + private final static String TEST_NAME2 = "Exists2"; //for vars + private final static String TEST_DIR = "functions/misc/"; private final static String TEST_CLASS_DIR = TEST_DIR + ExistsVariableTest.class.getSimpleName() + "/"; - //TODO additional test with variable creation in same DAG, requires better data dependency handling @Override public void setUp() { TestUtils.clearAssertionInformation(); addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[]{"R"})); + addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[]{"R"})); } @Test - public void testExistsPositive() { + public void testExistsVarnamePositive() { runExistsTest(TEST_NAME1, true); } @Test - public void testExistsNegative() { + public void testExistsVarnameNegative() { runExistsTest(TEST_NAME1, false); } + @Test + public void testExistsVarPositive() { + runExistsTest(TEST_NAME2, true); + } + + @Test + public void testExistsVarNegative() { + runExistsTest(TEST_NAME2, false); + } + private void runExistsTest(String testName, boolean pos) { TestConfiguration config = getTestConfiguration(testName); loadTestConfiguration(config); http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/test/scripts/functions/misc/Exists.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/Exists.dml b/src/test/scripts/functions/misc/Exists.dml deleted file mode 100644 index 4ff2b7c..0000000 --- a/src/test/scripts/functions/misc/Exists.dml +++ /dev/null @@ -1,30 +0,0 @@ -#------------------------------------------------------------- -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -#------------------------------------------------------------- - -Y = matrix(7, 10, 10) -if( $1 == 1 & sum(Y)>7 ) - X = matrix(1,10,10); - -R = as.matrix(as.double(exists("X"))); - -if( exists(X) ) - print("X exists: "+sum(X)); -write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/test/scripts/functions/misc/Exists1.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/Exists1.dml b/src/test/scripts/functions/misc/Exists1.dml new file mode 100644 index 0000000..4ff2b7c --- /dev/null +++ b/src/test/scripts/functions/misc/Exists1.dml @@ -0,0 +1,30 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +Y = matrix(7, 10, 10) +if( $1 == 1 & sum(Y)>7 ) + X = matrix(1,10,10); + +R = as.matrix(as.double(exists("X"))); + +if( exists(X) ) + print("X exists: "+sum(X)); +write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/0d858347/src/test/scripts/functions/misc/Exists2.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/Exists2.dml b/src/test/scripts/functions/misc/Exists2.dml new file mode 100644 index 0000000..8dbbb83 --- /dev/null +++ b/src/test/scripts/functions/misc/Exists2.dml @@ -0,0 +1,27 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +if( $1 == 1 ) + X = matrix(1,10,10); + +R = as.matrix(as.double(exists(X))); + +write(R, $2);
