[SYSTEMML-2210] New exists builtin function to check existence of vars This patch introduces an initial version of the new exists builtin function that allows to check for the existence of variables by explicit references to data identifiers or by variable names.
Note that this initial version requires that the referenced variable is already bound to its logical variable name and that the variable is used after the existence check (which guarantees its still available as live variable). In another patch, we will generalize the data dependency handling to remove these limitation.s Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/53b489ce Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/53b489ce Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/53b489ce Branch: refs/heads/master Commit: 53b489cedaf4fa48eaa25cb9bd93e449e198286f Parents: 8693ae6 Author: Matthias Boehm <[email protected]> Authored: Sun Mar 25 22:40:48 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sun Mar 25 22:40:48 2018 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/hops/Hop.java | 3 +- .../hops/rewrite/RewriteConstantFolding.java | 1 + .../java/org/apache/sysml/lops/UnaryCP.java | 14 ++-- .../sysml/parser/BuiltinFunctionExpression.java | 21 +++++- .../org/apache/sysml/parser/DMLTranslator.java | 7 +- .../org/apache/sysml/parser/Expression.java | 1 + .../instructions/CPInstructionParser.java | 1 + .../cp/AggregateUnaryCPInstruction.java | 14 +++- .../functions/misc/ExistsVariableTest.java | 70 ++++++++++++++++++++ src/test/scripts/functions/misc/Exists.dml | 30 +++++++++ .../functions/misc/ZPackageSuite.java | 1 + 11 files changed, 147 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/hops/Hop.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/Hop.java b/src/main/java/org/apache/sysml/hops/Hop.java index 8d799f9..dd168d7 100644 --- a/src/main/java/org/apache/sysml/hops/Hop.java +++ b/src/main/java/org/apache/sysml/hops/Hop.java @@ -1051,7 +1051,7 @@ public abstract class Hop implements ParseInfo NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SINH, COSH, TANH, SIGN, SQRT, LOG, EXP, CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_FRAME, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, PRINT, ASSERT, EIGEN, NROW, NCOL, LENGTH, ROUND, IQM, STOP, CEIL, FLOOR, MEDIAN, INVERSE, CHOLESKY, - SVD, + SVD, EXISTS, //cumulative sums, products, extreme values CUMSUM, CUMPROD, CUMMIN, CUMMAX, //fused ML-specific operators for performance @@ -1344,6 +1344,7 @@ public abstract class Hop implements ParseInfo HopsOpOp1LopsUS.put(OpOp1.NROW, org.apache.sysml.lops.UnaryCP.OperationTypes.NROW); HopsOpOp1LopsUS.put(OpOp1.NCOL, org.apache.sysml.lops.UnaryCP.OperationTypes.NCOL); HopsOpOp1LopsUS.put(OpOp1.LENGTH, org.apache.sysml.lops.UnaryCP.OperationTypes.LENGTH); + HopsOpOp1LopsUS.put(OpOp1.EXISTS, org.apache.sysml.lops.UnaryCP.OperationTypes.EXISTS); HopsOpOp1LopsUS.put(OpOp1.PRINT, org.apache.sysml.lops.UnaryCP.OperationTypes.PRINT); HopsOpOp1LopsUS.put(OpOp1.ASSERT, org.apache.sysml.lops.UnaryCP.OperationTypes.ASSERT); HopsOpOp1LopsUS.put(OpOp1.ROUND, org.apache.sysml.lops.UnaryCP.OperationTypes.ROUND); http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java index 9a1e76d..ed34956 100644 --- a/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java +++ b/src/main/java/org/apache/sysml/hops/rewrite/RewriteConstantFolding.java @@ -238,6 +238,7 @@ public class RewriteConstantFolding extends HopRewriteRule ArrayList<Hop> in = hop.getInput(); return ( hop instanceof UnaryOp && in.get(0) instanceof LiteralOp + && ((UnaryOp)hop).getOp() != OpOp1.EXISTS && ((UnaryOp)hop).getOp() != OpOp1.PRINT && ((UnaryOp)hop).getOp() != OpOp1.ASSERT && ((UnaryOp)hop).getOp() != OpOp1.STOP http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/lops/UnaryCP.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/lops/UnaryCP.java b/src/main/java/org/apache/sysml/lops/UnaryCP.java index 26e1731..0208018 100644 --- a/src/main/java/org/apache/sysml/lops/UnaryCP.java +++ b/src/main/java/org/apache/sysml/lops/UnaryCP.java @@ -36,7 +36,7 @@ public class UnaryCP extends Lop public enum OperationTypes { NOT, ABS, SIN, COS, TAN, ASIN, ACOS, ATAN, SQRT, LOG, EXP, SINH, COSH, TANH, CAST_AS_SCALAR, CAST_AS_MATRIX, CAST_AS_FRAME, CAST_AS_DOUBLE, CAST_AS_INT, CAST_AS_BOOLEAN, - PRINT, ASSERT, NROW, NCOL, LENGTH, ROUND, STOP, CEIL, FLOOR, CUMSUM, SOFTMAX + PRINT, ASSERT, NROW, NCOL, LENGTH, EXISTS, ROUND, STOP, CEIL, FLOOR, CUMSUM, SOFTMAX } public static final String CAST_AS_SCALAR_OPCODE = "castdts"; @@ -170,14 +170,10 @@ public class UnaryCP extends Lop case CAST_AS_BOOLEAN: return CAST_AS_BOOLEAN_OPCODE; - case NROW: - return "nrow"; - - case NCOL: - return "ncol"; - - case LENGTH: - return "length"; + case NROW: return "nrow"; + case NCOL: return "ncol"; + case LENGTH: return "length"; + case EXISTS: return "exists"; case SOFTMAX: return "softmax"; http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java index b597dd0..d7d3a4c 100644 --- a/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java +++ b/src/main/java/org/apache/sysml/parser/BuiltinFunctionExpression.java @@ -694,6 +694,15 @@ public class BuiltinFunctionExpression extends DataIdentifier output.setValueType(ValueType.INT); break; + case EXISTS: + checkNumParameters(1); + checkStringOrDataIdentifier(getFirstExpr()); + output.setDataType(DataType.SCALAR); + output.setDimensions(0, 0); + output.setBlockDimensions (0, 0); + output.setValueType(ValueType.BOOLEAN); + break; + // Contingency tables case TABLE: @@ -1514,6 +1523,13 @@ public class BuiltinFunctionExpression extends DataIdentifier } } + protected void checkStringOrDataIdentifier(Expression e) { //always unconditional + if( !(e.getOutput().getDataType().isScalar() && e.getOutput().getValueType()==ValueType.STRING) + && !(e instanceof DataIdentifier && !(e instanceof IndexedIdentifier)) ) { + raiseValidateError("Expecting variable name or data identifier "+ getOpCode(), false, LanguageErrorCodes.UNSUPPORTED_PARAMETERS); + } + } + private static boolean is1DMatrix(Expression e) { return (e.getOutput().getDim1() == 1 || e.getOutput().getDim2() == 1 ); } @@ -1777,9 +1793,10 @@ public class BuiltinFunctionExpression extends DataIdentifier bifop = Expression.BuiltinFunctionOp.BITWSHIFTR; else if ( functionName.equals("ifelse") ) bifop = Expression.BuiltinFunctionOp.IFELSE; - else if (functionName.equals("eval")) { + else if (functionName.equals("eval")) bifop = Expression.BuiltinFunctionOp.EVAL; - } + else if (functionName.equals("exists")) + bifop = Expression.BuiltinFunctionOp.EXISTS; else return null; http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/parser/DMLTranslator.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/DMLTranslator.java b/src/main/java/org/apache/sysml/parser/DMLTranslator.java index 9753465..dd41a03 100644 --- a/src/main/java/org/apache/sysml/parser/DMLTranslator.java +++ b/src/main/java/org/apache/sysml/parser/DMLTranslator.java @@ -2440,7 +2440,12 @@ public class DMLTranslator currBuiltinOp = new LiteralOp(lval); } break; - + + case EXISTS: + currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), + target.getValueType(), Hop.OpOp1.EXISTS, new LiteralOp(expr.getName())); + break; + case SUM: currBuiltinOp = new AggUnaryOp(target.getName(), target.getDataType(), target.getValueType(), AggOp.SUM, Direction.RowCol, expr); http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/parser/Expression.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/Expression.java b/src/main/java/org/apache/sysml/parser/Expression.java index 431e21d..dbbe8b8 100644 --- a/src/main/java/org/apache/sysml/parser/Expression.java +++ b/src/main/java/org/apache/sysml/parser/Expression.java @@ -88,6 +88,7 @@ public abstract class Expression implements ParseInfo DIAG, EIGEN, EVAL, + EXISTS, CONV2D, CONV2D_BACKWARD_FILTER, CONV2D_BACKWARD_DATA, BIAS_ADD, BIAS_MULTIPLY, MAX_POOL, AVG_POOL, MAX_POOL_BACKWARD, AVG_POOL_BACKWARD, EXP, http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java index c8d0d6c..24424ae 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/CPInstructionParser.java @@ -107,6 +107,7 @@ public class CPInstructionParser extends InstructionParser String2CPInstructionType.put( "nrow" ,CPType.AggregateUnary); String2CPInstructionType.put( "ncol" ,CPType.AggregateUnary); String2CPInstructionType.put( "length" ,CPType.AggregateUnary); + String2CPInstructionType.put( "exists" ,CPType.AggregateUnary); String2CPInstructionType.put( "uaggouterchain", CPType.UaggOuterChain); http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java index 65e6a97..249e567 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/cp/AggregateUnaryCPInstruction.java @@ -38,7 +38,7 @@ import org.apache.sysml.runtime.matrix.operators.SimpleOperator; public class AggregateUnaryCPInstruction extends UnaryCPInstruction { public enum AUType { - NROW, NCOL, LENGTH, + NROW, NCOL, LENGTH, EXISTS, DEFAULT; public boolean isMeta() { return this != DEFAULT; @@ -63,7 +63,8 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction CPOperand in1 = new CPOperand(parts[1]); CPOperand out = new CPOperand(parts[2]); - if(opcode.equalsIgnoreCase("nrow") || opcode.equalsIgnoreCase("ncol") || opcode.equalsIgnoreCase("length")){ + if(opcode.equalsIgnoreCase("nrow") || opcode.equalsIgnoreCase("ncol") + || opcode.equalsIgnoreCase("length") || opcode.equalsIgnoreCase("exists")){ return new AggregateUnaryCPInstruction(new SimpleOperator(Builtin.getBuiltinFnObject(opcode)), in1, out, AUType.valueOf(opcode.toUpperCase()), opcode, str); } @@ -79,7 +80,7 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction String output_name = output.getName(); String opcode = getOpcode(); - if( _type.isMeta() ) //nrow/ncol/length + if( _type.isMeta() && _type!=AUType.EXISTS ) //nrow/ncol/length { //check existence of input variable if( !ec.getVariables().keySet().contains(input1.getName()) ) @@ -120,6 +121,13 @@ public class AggregateUnaryCPInstruction extends UnaryCPInstruction //create and set output scalar ec.setScalarOutput(output_name, new IntObject(rval)); } + else if( _type == AUType.EXISTS ) { + //probe existing of variable in symbol table w/o error + boolean rval = ec.getVariables().keySet() + .contains(ec.getScalarInput(input1).getStringValue()); + //create and set output scalar + ec.setScalarOutput(output_name, new BooleanObject(rval)); + } else { //DEFAULT MatrixBlock matBlock = ec.getMatrixInput(input1.getName()); AggregateUnaryOperator au_op = (AggregateUnaryOperator) _optr; http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java new file mode 100644 index 0000000..1bf0f34 --- /dev/null +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ExistsVariableTest.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.sysml.test.integration.functions.misc; + + +import org.junit.Assert; +import org.junit.Test; +import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; +import org.apache.sysml.test.integration.AutomatedTestBase; +import org.apache.sysml.test.integration.TestConfiguration; +import org.apache.sysml.test.utils.TestUtils; + +public class ExistsVariableTest extends AutomatedTestBase +{ + private final static String TEST_NAME1 = "Exists"; + private final static String TEST_DIR = "functions/misc/"; + private final static String TEST_CLASS_DIR = TEST_DIR + ExistsVariableTest.class.getSimpleName() + "/"; + //TODO additional test with variable creation in same DAG, requires better data dependency handling + + @Override + public void setUp() { + TestUtils.clearAssertionInformation(); + addTestConfiguration( TEST_NAME1, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME1, new String[]{"R"})); + } + + @Test + public void testExistsPositive() { + runExistsTest(TEST_NAME1, true); + } + + @Test + public void testExistsNegative() { + runExistsTest(TEST_NAME1, false); + } + + private void runExistsTest(String testName, boolean pos) { + TestConfiguration config = getTestConfiguration(testName); + loadTestConfiguration(config); + String HOME = SCRIPT_DIR + TEST_DIR; + String param = pos ? "1" : "0"; + fullDMLScriptName = HOME + testName + ".dml"; + programArgs = new String[]{"-explain", "-stats", "-args", param, output("R") }; + + //run script and compare output + runTest(true, false, null, -1); + + //compare results + Double val = readDMLMatrixFromHDFS("R").get(new CellIndex(1,1)); + val = (val!=null) ? val : 0; + Assert.assertTrue("Wrong result: "+param+" vs "+val, + val==Double.parseDouble(param)); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/test/scripts/functions/misc/Exists.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/Exists.dml b/src/test/scripts/functions/misc/Exists.dml new file mode 100644 index 0000000..4ff2b7c --- /dev/null +++ b/src/test/scripts/functions/misc/Exists.dml @@ -0,0 +1,30 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +Y = matrix(7, 10, 10) +if( $1 == 1 & sum(Y)>7 ) + X = matrix(1,10,10); + +R = as.matrix(as.double(exists("X"))); + +if( exists(X) ) + print("X exists: "+sum(X)); +write(R, $2); http://git-wip-us.apache.org/repos/asf/systemml/blob/53b489ce/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java index 1849b51..46385c2 100644 --- a/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java +++ b/src/test_suites/java/org/apache/sysml/test/integration/functions/misc/ZPackageSuite.java @@ -29,6 +29,7 @@ import org.junit.runners.Suite; ConditionalValidateTest.class, DataTypeCastingTest.class, DataTypeChangeTest.class, + ExistsVariableTest.class, FunctionInExpressionTest.class, FunctionInliningTest.class, FunctionNamespaceTest.class,
