Repository: incubator-systemml Updated Branches: refs/heads/master c4259eb65 -> f788b42d0
[SYSTEMML-1478] Generalized multi-agg codegen tpl (partial shared reads) So far, we only supported multi-aggregate templates within a connected partition of fusion plans as well as multiple aggregates over a single shared input. This patch generalizes the compilation of multi-aggregates to partial reads across partitions, which allows the fusion of expressions like r1 = sum(X * Y); r2 = sum(X ^ 2); r3 = sum(Y ^ 2); into a single operator, reducing the number of reads from memory from 4 to 2. This also includes transitive partial reads for expressions like r1 = sum(V * X); r2 = sum(Y * Z); r3 = sum(X * Y * Z); where r1 does not have a shared read with r2 but when considering fusion plans we notice that both transitively share reads with r3, reducing the number of reads from memory from 7 to 4. Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/f788b42d Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/f788b42d Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/f788b42d Branch: refs/heads/master Commit: f788b42d09ff734d88d9c5b3bf78a5c526249f12 Parents: c4259eb Author: Matthias Boehm <[email protected]> Authored: Sun Apr 9 00:33:08 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sun Apr 9 00:36:07 2017 -0700 ---------------------------------------------------------------------- .../hops/codegen/template/CPlanMemoTable.java | 5 + .../template/PlanSelectionFuseCostBased.java | 166 ++++++++++++++++--- .../hops/codegen/template/TemplateCell.java | 3 +- .../hops/codegen/template/TemplateMultiAgg.java | 10 +- .../functions/codegen/MultiAggTmplTest.java | 66 +++++++- .../functions/codegen/multiAggPattern3.R | 33 ++++ .../functions/codegen/multiAggPattern3.dml | 29 ++++ .../functions/codegen/multiAggPattern4.R | 35 ++++ .../functions/codegen/multiAggPattern4.dml | 31 ++++ .../functions/codegen/multiAggPattern5.R | 37 +++++ .../functions/codegen/multiAggPattern5.dml | 33 ++++ .../functions/codegen/multiAggPattern6.R | 33 ++++ .../functions/codegen/multiAggPattern6.dml | 29 ++++ 13 files changed, 483 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java b/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java index 3aa94d5..d306f0f 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/CPlanMemoTable.java @@ -62,6 +62,11 @@ public class CPlanMemoTable return _plans.containsKey(hopID); } + public boolean contains(long hopID, TemplateType type) { + return contains(hopID) && get(hopID).stream() + .filter(p -> p.type==type).findAny().isPresent(); + } + public boolean containsTopLevel(long hopID) { return !_plansBlacklist.contains(hopID) && getBest(hopID) != null; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java b/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java index 7682336..a9cd90e 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/PlanSelectionFuseCostBased.java @@ -25,16 +25,17 @@ import java.util.Collection; import java.util.Comparator; import java.util.HashMap; import java.util.Map.Entry; +import java.util.stream.Collectors; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.sysml.hops.AggBinaryOp; import org.apache.sysml.hops.AggUnaryOp; import org.apache.sysml.hops.BinaryOp; -import org.apache.sysml.hops.DataOp; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.Hop.AggOp; import org.apache.sysml.hops.Hop.Direction; @@ -268,32 +269,91 @@ public class PlanSelectionFuseCostBased extends PlanSelection } } - //across-partition multi-agg templates - private static void createAndAddMultiAggPlans(CPlanMemoTable memo, ArrayList<Hop> roots) + //across-partition multi-agg templates with shared reads + private void createAndAddMultiAggPlans(CPlanMemoTable memo, ArrayList<Hop> roots) { - //#1: collect full aggregations over shared inputs (otherwise never fused) - HashMap<Long, ArrayList<Long>> fullAggs = new HashMap<Long, ArrayList<Long>>(); + //collect full aggregations as initial set of candidates + HashSet<Long> fullAggs = new HashSet<Long>(); Hop.resetVisitStatus(roots); for( Hop hop : roots ) - rCollectAggregatesSharedRead(hop, fullAggs); + rCollectFullAggregates(hop, fullAggs); + + //remove operators with assigned multi-agg plans + Iterator<Long> iter = fullAggs.iterator(); + while( iter.hasNext() ) { + if( memo.contains(iter.next(), TemplateType.MultiAggTpl) ) + iter.remove(); + } + + //check applicability for further analysis + if( fullAggs.size() <= 1 ) + return; + + if( LOG.isTraceEnabled() ) { + LOG.trace("Found across-partition ua(RC) aggregations: " + + Arrays.toString(fullAggs.toArray(new Long[0]))); + } + + //collect information for all candidates + //(subsumed aggregations, and inputs to fused operators) + List<AggregateInfo> aggInfos = new ArrayList<AggregateInfo>(); + for( Long hopID : fullAggs ) { + Hop aggHop = memo._hopRefs.get(hopID); + AggregateInfo tmp = new AggregateInfo(aggHop); + for( Hop c : aggHop.getInput() ) + rExtractAggregateInfo(memo, c, tmp, TemplateType.CellTpl); + if( tmp._fusedInputs.isEmpty() ) + tmp.addFusedInput(aggHop.getInput().get(0).getHopID()); + aggInfos.add(tmp); + } if( LOG.isTraceEnabled() ) { - for( Entry<Long, ArrayList<Long>> e : fullAggs.entrySet() ) - LOG.trace("Found across-partition ua(RC) aggregations for "+e.getKey()+": " + - Arrays.toString(e.getValue().toArray(new Long[0]))); + LOG.trace("Extracted across-partition ua(RC) aggregation info: "); + for( AggregateInfo info : aggInfos ) + LOG.trace(info); + } + + //sort aggregations by num dependencies to simplify merging + //clusters of aggregations with parallel dependencies + aggInfos = aggInfos.stream().sorted(Comparator.comparing( + a -> a._inputAggs.size())).collect(Collectors.toList()); + + //greedy grouping of multi-agg candidates + boolean converged = false; + while( !converged ) { + AggregateInfo merged = null; + for( int i=0; i<aggInfos.size(); i++ ) { + AggregateInfo current = aggInfos.get(i); + for( int j=i+1; j<aggInfos.size(); j++ ) { + AggregateInfo that = aggInfos.get(j); + if( current.isMergable(that) ) { + merged = current.merge(that); + aggInfos.remove(j); j--; + } + } + } + converged = (merged == null); + } + + if( LOG.isTraceEnabled() ) { + LOG.trace("Merged across-partition ua(RC) aggregation info: "); + for( AggregateInfo info : aggInfos ) + LOG.trace(info); } //construct and add multiagg template plans (w/ max 3 aggregations) - for( Entry<Long, ArrayList<Long>> e : fullAggs.entrySet() ) { - if( e.getValue().size()<=1 ) + for( AggregateInfo info : aggInfos ) { + if( info._aggregates.size()<=1 ) continue; - ArrayList<Long> aggs = e.getValue(); + Long[] aggs = info._aggregates.keySet().toArray(new Long[0]); MemoTableEntry me = new MemoTableEntry(TemplateType.MultiAggTpl, - aggs.get(0), aggs.get(1), (aggs.size()>2)?aggs.get(2):-1); - for( int i=0; i<aggs.size(); i++ ) { - memo.add(memo._hopRefs.get(aggs.get(i)), me); + aggs[0], aggs[1], (aggs.length>2)?aggs[2]:-1); + for( int i=0; i<aggs.length; i++ ) { + memo.add(memo._hopRefs.get(aggs[i]), me); + addBestPlan(aggs[i], me); if( LOG.isTraceEnabled() ) - LOG.trace("Added multiagg* plan: "+aggs.get(i)+" "+me); + LOG.trace("Added multiagg* plan: "+aggs[i]+" "+me); + } } } @@ -321,28 +381,46 @@ public class PlanSelectionFuseCostBased extends PlanSelection return ret; } - private static void rCollectAggregatesSharedRead(Hop current, HashMap<Long, ArrayList<Long>> aggs) { + private static void rCollectFullAggregates(Hop current, HashSet<Long> aggs) { if( current.isVisited() ) return; //collect all applicable full aggregations per read if( HopRewriteUtils.isAggUnaryOp(current, AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX) - && ((AggUnaryOp)current).getDirection()==Direction.RowCol - && current.getInput().get(0) instanceof DataOp ) + && ((AggUnaryOp)current).getDirection()==Direction.RowCol ) { - Hop input = current.getInput().get(0); - if( !aggs.containsKey(input.getHopID()) ) - aggs.put(input.getHopID(), new ArrayList<Long>()); - aggs.get(input.getHopID()).add(current.getHopID()); + aggs.add(current.getHopID()); } //recursively process children for( Hop c : current.getInput() ) - rCollectAggregatesSharedRead(c, aggs); + rCollectFullAggregates(c, aggs); current.setVisited(); } + private static void rExtractAggregateInfo(CPlanMemoTable memo, Hop current, AggregateInfo aggInfo, TemplateType type) { + //collect input aggregates (dependents) + if( HopRewriteUtils.isAggUnaryOp(current, AggOp.SUM, AggOp.SUM_SQ, AggOp.MIN, AggOp.MAX) + && ((AggUnaryOp)current).getDirection()==Direction.RowCol ) + { + aggInfo.addInputAggregate(current.getHopID()); + } + + //recursively process children + MemoTableEntry me = (type!=null) ? memo.getBest(current.getHopID()) : null; + for( int i=0; i< current.getInput().size(); i++ ) { + Hop c = current.getInput().get(i); + if( me != null && me.isPlanRef(i) ) + rExtractAggregateInfo(memo, c, aggInfo, type); + else { + if( type != null && c.getDataType().isMatrix() ) //add fused input + aggInfo.addFusedInput(c.getHopID()); + rExtractAggregateInfo(memo, c, aggInfo, null); + } + } + } + private void selectPlans(CPlanMemoTable memo, HashSet<Long> partition, HashSet<Long> R, ArrayList<Long> M) { //if no materialization points, use basic fuse-all w/ partition awareness @@ -721,4 +799,44 @@ public class PlanSelectionFuseCostBased extends PlanSelection +Arrays.toString(inSizes.values().toArray(new Double[0]))+"}]"; } } + + private static class AggregateInfo { + public final HashMap<Long,Hop> _aggregates; + public final HashSet<Long> _inputAggs = new HashSet<Long>(); + public final HashSet<Long> _fusedInputs = new HashSet<Long>(); + public AggregateInfo(Hop aggregate) { + _aggregates = new HashMap<Long, Hop>(); + _aggregates.put(aggregate.getHopID(), aggregate); + } + public void addInputAggregate(long hopID) { + _inputAggs.add(hopID); + } + public void addFusedInput(long hopID) { + _fusedInputs.add(hopID); + } + public boolean isMergable(AggregateInfo that) { + //check independence + boolean ret = _aggregates.size()<3 + && _aggregates.size()+that._aggregates.size()<=3; + for( Long hopID : that._aggregates.keySet() ) + ret &= !_inputAggs.contains(hopID); + for( Long hopID : _aggregates.keySet() ) + ret &= !that._inputAggs.contains(hopID); + //check partial shared reads + return ret && !CollectionUtils.intersection( + _fusedInputs, that._fusedInputs).isEmpty(); + } + public AggregateInfo merge(AggregateInfo that) { + _aggregates.putAll(that._aggregates); + _inputAggs.addAll(that._inputAggs); + _fusedInputs.addAll(that._fusedInputs); + return this; + } + @Override + public String toString() { + return "["+Arrays.toString(_aggregates.keySet().toArray(new Long[0]))+": " + +"{"+Arrays.toString(_inputAggs.toArray(new Long[0]))+"}," + +"{"+Arrays.toString(_fusedInputs.toArray(new Long[0]))+"}]"; + } + } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java index 885d3db..95f6643 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateCell.java @@ -150,7 +150,8 @@ public class TemplateCell extends TemplateBase MemoTableEntry me = memo.getBest(hop.getHopID(), TemplateType.CellTpl); for( int i=0; i<hop.getInput().size(); i++ ) { Hop c = hop.getInput().get(i); - if( me!=null && me.isPlanRef(i) && !(c instanceof DataOp) ) + if( me!=null && me.isPlanRef(i) && !(c instanceof DataOp) + && (me.type!=TemplateType.MultiAggTpl || memo.contains(c.getHopID(), TemplateType.CellTpl))) rConstructCplan(c, memo, tmp, inHops, compileLiterals); else { CNodeData cdata = TemplateUtils.createCNodeData(c, compileLiterals); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/main/java/org/apache/sysml/hops/codegen/template/TemplateMultiAgg.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateMultiAgg.java b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateMultiAgg.java index aaf00e7..56477da 100644 --- a/src/main/java/org/apache/sysml/hops/codegen/template/TemplateMultiAgg.java +++ b/src/main/java/org/apache/sysml/hops/codegen/template/TemplateMultiAgg.java @@ -28,9 +28,12 @@ import java.util.stream.Collectors; import org.apache.sysml.hops.Hop; import org.apache.sysml.hops.Hop.AggOp; import org.apache.sysml.hops.codegen.cplan.CNode; +import org.apache.sysml.hops.codegen.cplan.CNodeData; import org.apache.sysml.hops.codegen.template.CPlanMemoTable.MemoTableEntry; import org.apache.sysml.hops.codegen.cplan.CNodeMultiAgg; import org.apache.sysml.hops.codegen.cplan.CNodeTpl; +import org.apache.sysml.hops.codegen.cplan.CNodeUnary; +import org.apache.sysml.hops.codegen.cplan.CNodeUnary.UnaryType; import org.apache.sysml.runtime.matrix.data.Pair; public class TemplateMultiAgg extends TemplateCell @@ -96,7 +99,12 @@ public class TemplateMultiAgg extends TemplateCell ArrayList<CNode> outputs = new ArrayList<CNode>(); ArrayList<AggOp> aggOps = new ArrayList<AggOp>(); for( Hop root : roots ) { - outputs.add(tmp.get(root.getHopID())); + CNode node = tmp.get(root.getHopID()); + if( node instanceof CNodeData //add indexing ops for sideways data inputs + && ((CNodeData)inputs.get(0)).getHopID() != ((CNodeData)node).getHopID() ) + node = new CNodeUnary(node, (roots.get(0).getDim2()==1) ? + UnaryType.LOOKUP_R : UnaryType.LOOKUP_RC); + outputs.add(node); aggOps.add(TemplateUtils.getAggOp(root)); } CNodeMultiAgg tpl = new CNodeMultiAgg(inputs, outputs); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java index b25d496..c33d680 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/MultiAggTmplTest.java @@ -38,6 +38,10 @@ public class MultiAggTmplTest extends AutomatedTestBase private static final String TEST_NAME = "multiAggPattern"; private static final String TEST_NAME1 = TEST_NAME+"1"; //min(X>7), max(X>7) private static final String TEST_NAME2 = TEST_NAME+"2"; //sum(X>7), sum((X>7)^2) + private static final String TEST_NAME3 = TEST_NAME+"3"; //sum(X==7), sum(X==3) + private static final String TEST_NAME4 = TEST_NAME+"4"; //sum(X*Y), sum(X^2), sum(Y^2) + private static final String TEST_NAME5 = TEST_NAME+"5"; //sum(V*X), sum(Y*Z), sum(X+Y-Z) + private static final String TEST_NAME6 = TEST_NAME+"6"; //min(X), max(X), sum(X) private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + MultiAggTmplTest.class.getSimpleName() + "/"; @@ -49,7 +53,7 @@ public class MultiAggTmplTest extends AutomatedTestBase @Override public void setUp() { TestUtils.clearAssertionInformation(); - for(int i=1; i<=2; i++) + for(int i=1; i<=6; i++) addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) ); } @@ -83,6 +87,66 @@ public class MultiAggTmplTest extends AutomatedTestBase testCodegenIntegration( TEST_NAME2, false, ExecType.SPARK ); } + @Test + public void testCodegenMultiAggRewrite3CP() { + testCodegenIntegration( TEST_NAME3, true, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg3CP() { + testCodegenIntegration( TEST_NAME3, false, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg3Spark() { + testCodegenIntegration( TEST_NAME3, false, ExecType.SPARK ); + } + + @Test + public void testCodegenMultiAggRewrite4CP() { + testCodegenIntegration( TEST_NAME4, true, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg4CP() { + testCodegenIntegration( TEST_NAME4, false, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg4Spark() { + testCodegenIntegration( TEST_NAME4, false, ExecType.SPARK ); + } + + @Test + public void testCodegenMultiAggRewrite5CP() { + testCodegenIntegration( TEST_NAME5, true, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg5CP() { + testCodegenIntegration( TEST_NAME5, false, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg5Spark() { + testCodegenIntegration( TEST_NAME5, false, ExecType.SPARK ); + } + + @Test + public void testCodegenMultiAggRewrite6CP() { + testCodegenIntegration( TEST_NAME6, true, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg6CP() { + testCodegenIntegration( TEST_NAME6, false, ExecType.CP ); + } + + @Test + public void testCodegenMultiAgg6Spark() { + testCodegenIntegration( TEST_NAME6, false, ExecType.SPARK ); + } + private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType ) { boolean oldFlag = OptimizerUtils.ALLOW_ALGEBRAIC_SIMPLIFICATION; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern3.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern3.R b/src/test/scripts/functions/codegen/multiAggPattern3.R new file mode 100644 index 0000000..96f0eec --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern3.R @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = matrix(seq(1,15), 5, 3, byrow=TRUE); + +#disjoint partitions with shared read +r1 = sum(X == 7) +r2 = sum(X == 3) +S = as.matrix(r1+r2); + +writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep="")); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern3.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern3.dml b/src/test/scripts/functions/codegen/multiAggPattern3.dml new file mode 100644 index 0000000..5836d56 --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern3.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix(seq(1,15), rows=5, cols=3); + +#disjoint partitions with shared read +r1 = sum(X == 7) +r2 = sum(X == 3) +S = as.matrix(r1+r2); + +write(S,$1) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern4.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern4.R b/src/test/scripts/functions/codegen/multiAggPattern4.R new file mode 100644 index 0000000..2db1c42 --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern4.R @@ -0,0 +1,35 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = matrix(seq(1,15), 5, 3, byrow=TRUE); +Y = matrix(seq(2,16), 5, 3, byrow=TRUE); + +#disjoint partitions with partial shared reads +r1 = sum(X * Y); +r2 = sum(X ^ 2); +r3 = sum(Y ^ 2); +S = as.matrix(r1+r2+r3); + +writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep="")); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern4.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern4.dml b/src/test/scripts/functions/codegen/multiAggPattern4.dml new file mode 100644 index 0000000..0ca1a96 --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern4.dml @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix(seq(1,15), rows=5, cols=3); +Y = matrix(seq(2,16), rows=5, cols=3); + +#disjoint partitions with partial shared reads +r1 = sum(X * Y); +r2 = sum(X ^ 2); +r3 = sum(Y ^ 2); +S = as.matrix(r1+r2+r3); + +write(S,$1) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern5.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern5.R b/src/test/scripts/functions/codegen/multiAggPattern5.R new file mode 100644 index 0000000..e09d1c3 --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern5.R @@ -0,0 +1,37 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +V = matrix(seq(0,14), 5, 3, byrow=TRUE); +X = matrix(seq(1,15), 5, 3, byrow=TRUE); +Y = matrix(seq(2,16), 5, 3, byrow=TRUE); +Z = matrix(seq(3,17), 5, 3, byrow=TRUE); + +#disjoint partitions with transitive partial shared reads +r1 = sum(V * X); +r2 = sum(Y * Z); +r3 = sum(X * Y * Z); +S = as.matrix(r1+r2+r3); + +writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep="")); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern5.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern5.dml b/src/test/scripts/functions/codegen/multiAggPattern5.dml new file mode 100644 index 0000000..e4d5728 --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern5.dml @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +V = matrix(seq(0,14), rows=5, cols=3); +X = matrix(seq(1,15), rows=5, cols=3); +Y = matrix(seq(2,16), rows=5, cols=3); +Z = matrix(seq(3,17), rows=5, cols=3); + +#disjoint partitions with transitive partial shared reads +r1 = sum(V * X); +r2 = sum(Y * Z); +r3 = sum(X * Y * Z); +S = as.matrix(r1+r2+r3); + +write(S,$1) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern6.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern6.R b/src/test/scripts/functions/codegen/multiAggPattern6.R new file mode 100644 index 0000000..0c42f6d --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern6.R @@ -0,0 +1,33 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = matrix(seq(1,15), 5, 3, byrow=TRUE); + +r1 = min(X); +r2 = max(X); +r3 = sum(X); +S = as.matrix(r1+r2+r3); + +writeMM(as(S, "CsparseMatrix"), paste(args[2], "S", sep="")); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/f788b42d/src/test/scripts/functions/codegen/multiAggPattern6.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/multiAggPattern6.dml b/src/test/scripts/functions/codegen/multiAggPattern6.dml new file mode 100644 index 0000000..c5a315a --- /dev/null +++ b/src/test/scripts/functions/codegen/multiAggPattern6.dml @@ -0,0 +1,29 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix(seq(1,15), rows=5, cols=3); + +r1 = min(X); +r2 = max(X); +r3 = sum(X); +S = as.matrix(r1+r2+r3); + +write(S,$1)
