Repository: systemml Updated Branches: refs/heads/master a5a4d4d33 -> 79a5e80f6
[SYSTEMML-2327] Fix missing list support in parfor, incl tests This patch adds tests with lists and named lists for parfor loops and dependency analysis and fixes a minor issue with unbounded scoping of temporary list intermediates. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/79a5e80f Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/79a5e80f Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/79a5e80f Branch: refs/heads/master Commit: 79a5e80f62d387af8ad3a3cf1967ca8adc9882b9 Parents: a5a4d4d Author: Matthias Boehm <[email protected]> Authored: Thu May 17 18:13:43 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Thu May 17 18:13:43 2018 -0700 ---------------------------------------------------------------------- .../controlprogram/ParForProgramBlock.java | 5 +++ .../functions/misc/ListAndStructTest.java | 25 +++++++++++- .../parfor/ParForDependencyAnalysisTest.java | 11 +++++ .../scripts/functions/misc/ListNamedParfor.R | 42 ++++++++++++++++++++ .../scripts/functions/misc/ListNamedParfor.dml | 36 +++++++++++++++++ .../scripts/functions/misc/ListUnnamedParfor.R | 42 ++++++++++++++++++++ .../functions/misc/ListUnnamedParfor.dml | 37 +++++++++++++++++ src/test/scripts/functions/parfor/parfor54a.dml | 29 ++++++++++++++ src/test/scripts/functions/parfor/parfor54b.dml | 29 ++++++++++++++ src/test/scripts/functions/parfor/parfor54c.dml | 29 ++++++++++++++ 10 files changed, 284 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java index 732e59d..368737b 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/ParForProgramBlock.java @@ -26,6 +26,7 @@ import java.io.Serializable; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -103,6 +104,7 @@ import org.apache.sysml.runtime.instructions.cp.BooleanObject; import org.apache.sysml.runtime.instructions.cp.Data; import org.apache.sysml.runtime.instructions.cp.DoubleObject; import org.apache.sysml.runtime.instructions.cp.IntObject; +import org.apache.sysml.runtime.instructions.cp.ListObject; import org.apache.sysml.runtime.instructions.cp.StringObject; import org.apache.sysml.runtime.instructions.cp.VariableCPInstruction; import org.apache.sysml.runtime.io.IOUtilFunctions; @@ -1274,6 +1276,9 @@ public class ParForProgramBlock extends ForProgramBlock //currently we do not create any unscoped matrix or frame outputs //because metadata (e.g., outputinfo) not known at this place. break; + case LIST: + dataObj = new ListObject(Collections.emptyList()); + break; case UNKNOWN: break; default: http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java index 4c33847..b49f84c 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/misc/ListAndStructTest.java @@ -37,7 +37,8 @@ public class ListAndStructTest extends AutomatedTestBase private static final String TEST_NAME2 = "ListNamed"; private static final String TEST_NAME3 = "ListUnnamedFun"; private static final String TEST_NAME4 = "ListNamedFun"; - + private static final String TEST_NAME5 = "ListUnnamedParfor"; + private static final String TEST_NAME6 = "ListNamedParfor"; private static final String TEST_DIR = "functions/misc/"; private static final String TEST_CLASS_DIR = TEST_DIR + ListAndStructTest.class.getSimpleName() + "/"; @@ -49,6 +50,8 @@ public class ListAndStructTest extends AutomatedTestBase addTestConfiguration( TEST_NAME2, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME2, new String[] { "R" }) ); addTestConfiguration( TEST_NAME3, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME3, new String[] { "R" }) ); addTestConfiguration( TEST_NAME4, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME4, new String[] { "R" }) ); + addTestConfiguration( TEST_NAME5, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME5, new String[] { "R" }) ); + addTestConfiguration( TEST_NAME6, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME6, new String[] { "R" }) ); } @Test @@ -91,6 +94,26 @@ public class ListAndStructTest extends AutomatedTestBase runListStructTest(TEST_NAME4, true); } + @Test + public void testListUnnamedParFor() { + runListStructTest(TEST_NAME5, false); + } + + @Test + public void testListUnnamedParForRewrites() { + runListStructTest(TEST_NAME5, true); + } + + @Test + public void testListNamedParFor() { + runListStructTest(TEST_NAME6, false); + } + + @Test + public void testListNamedParForRewrites() { + runListStructTest(TEST_NAME6, true); + } + private void runListStructTest(String testname, boolean rewrites) { boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java index 26b3fc6..97f253b 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/parfor/ParForDependencyAnalysisTest.java @@ -66,6 +66,8 @@ import org.junit.Test; * 49a: dep, 49b: dep * * accumulators * 53a: no, 53b dep, 53c dep, 53d dep, 53e dep + * * lists + * 54a: no, 54b: dep, 54c: dep */ public class ParForDependencyAnalysisTest extends AutomatedTestBase { @@ -316,6 +318,15 @@ public class ParForDependencyAnalysisTest extends AutomatedTestBase @Test public void testDependencyAnalysis53e() { runTest("parfor53e.dml", true); } + @Test + public void testDependencyAnalysis54a() { runTest("parfor54a.dml", false); } + + @Test + public void testDependencyAnalysis54b() { runTest("parfor54b.dml", true); } + + @Test + public void testDependencyAnalysis54c() { runTest("parfor54c.dml", true); } + private void runTest( String scriptFilename, boolean expectedException ) { boolean raisedException = false; http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListNamedParfor.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ListNamedParfor.R b/src/test/scripts/functions/misc/ListNamedParfor.R new file mode 100644 index 0000000..abf01be --- /dev/null +++ b/src/test/scripts/functions/misc/ListNamedParfor.R @@ -0,0 +1,42 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +A = matrix(1, 10, 10); +B = matrix(2, 10, 10); +c = 3; +D = matrix(4, 10, 10); + +X = list(a=A, b=B, c=c, d=D, e=matrix(5, 3, 3), f=6); + +for( i in 1:length(X) ) { + tmp = X[i] + if( !exists("tmp") ) + print("ERROR: non-existing entry "+i ); +} + +R = as.matrix(sum(as.matrix(X[['e']]))); + +writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListNamedParfor.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ListNamedParfor.dml b/src/test/scripts/functions/misc/ListNamedParfor.dml new file mode 100644 index 0000000..a00df0d --- /dev/null +++ b/src/test/scripts/functions/misc/ListNamedParfor.dml @@ -0,0 +1,36 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = matrix(1, 10, 10); +B = matrix(2, 10, 10); +c = 3; +D = matrix(4, 10, 10); + +X = list(a=A, b=B, c=c, d=D, e=matrix(5, 3, 3), f=6); +for( i in 1:length(X) ) { + tmp = X[i]; + if( !exists(tmp) ) + print("ERROR: non-existing entry "+i ); +} + +R = as.matrix(sum(as.matrix(X['e']))); + +write(R, $1); http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListUnnamedParfor.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ListUnnamedParfor.R b/src/test/scripts/functions/misc/ListUnnamedParfor.R new file mode 100644 index 0000000..ede3c2b --- /dev/null +++ b/src/test/scripts/functions/misc/ListUnnamedParfor.R @@ -0,0 +1,42 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +args <- commandArgs(TRUE) +options(digits=22) +library("Matrix") + +A = matrix(1, 10, 10); +B = matrix(2, 10, 10); +c = 3; +D = matrix(4, 10, 10); + +X = list(A, B, c, D, matrix(5, 3, 3), 6); + +for( i in 1:length(X) ) { + tmp = X[i] + if( !exists("tmp") ) + print("ERROR: non-existing entry "+i ); +} + +R = as.matrix(sum(as.matrix(X[[5]]))); + +writeMM(as(R, "CsparseMatrix"), paste(args[1], "R", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/misc/ListUnnamedParfor.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/misc/ListUnnamedParfor.dml b/src/test/scripts/functions/misc/ListUnnamedParfor.dml new file mode 100644 index 0000000..9d38517 --- /dev/null +++ b/src/test/scripts/functions/misc/ListUnnamedParfor.dml @@ -0,0 +1,37 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +A = matrix(1, 10, 10); +B = matrix(2, 10, 10); +c = 3; +D = matrix(4, 10, 10); + +X = list(A, B, c, D, matrix(5, 3, 3), 6); + +parfor( i in 1:length(X) ) { + tmp = X[i]; + if( !exists(tmp) ) + print("ERROR: non-existing entry "+i ); +} + +R = as.matrix(sum(as.matrix(X[5]))); + +write(R, $1); http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/parfor/parfor54a.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/parfor/parfor54a.dml b/src/test/scripts/functions/parfor/parfor54a.dml new file mode 100644 index 0000000..1d3a75e --- /dev/null +++ b/src/test/scripts/functions/parfor/parfor54a.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +A = matrix(7, rows=2, cols=2); +B = matrix(3, rows=2, cols=2); +C = list(A, B); +parfor( i in 1:2 ) { + print(sum(as.matrix(C[i]))); +} +print(sum(as.matrix(C[1]))); http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/parfor/parfor54b.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/parfor/parfor54b.dml b/src/test/scripts/functions/parfor/parfor54b.dml new file mode 100644 index 0000000..3e2c6e8 --- /dev/null +++ b/src/test/scripts/functions/parfor/parfor54b.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +A = matrix(7, rows=2, cols=2); +B = matrix(3, rows=2, cols=2); +C = list(A, B); +parfor( i in 1:2 ) { + C[i] = as.matrix(C[i])+7; +} +print(sum(as.matrix(C[1]))); http://git-wip-us.apache.org/repos/asf/systemml/blob/79a5e80f/src/test/scripts/functions/parfor/parfor54c.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/parfor/parfor54c.dml b/src/test/scripts/functions/parfor/parfor54c.dml new file mode 100644 index 0000000..b4f96d8 --- /dev/null +++ b/src/test/scripts/functions/parfor/parfor54c.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + + +A = matrix(7, rows=2, cols=2); +B = matrix(3, rows=2, cols=2); +C = list(A, B); +parfor( i in 1:2 ) { + C = list(as.matrix(C[i])+7); +} +print(sum(as.matrix(C[1])));
