Repository: systemml Updated Branches: refs/heads/master 1a37cfad4 -> d0b4373fa
http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java index 314e118..a3e1dfe 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/spark/SpoofSPInstruction.java @@ -136,8 +136,8 @@ public class SpoofSPInstruction extends SPInstruction { if( _out.getDataType()==DataType.MATRIX ) { //execute codegen block operation - out = in.mapPartitionsToPair(new CellwiseFunction( - _class.getName(), _classBytes, bcVect2, bcMatrices, scalars), true); + out = in.mapPartitionsToPair(new CellwiseFunction(_class.getName(), + _classBytes, bcVect2, bcMatrices, scalars, mcIn.getRowsPerBlock()), true); if( (op.getCellType()==CellType.ROW_AGG && mcIn.getCols() > mcIn.getColsPerBlock()) || (op.getCellType()==CellType.COL_AGG && mcIn.getRows() > mcIn.getRowsPerBlock())) { @@ -153,8 +153,8 @@ public class SpoofSPInstruction extends SPInstruction { updateOutputMatrixCharacteristics(sec, op); } else { //SCALAR - out = in.mapPartitionsToPair(new CellwiseFunction( - _class.getName(), _classBytes, bcVect2, bcMatrices, scalars), true); + out = in.mapPartitionsToPair(new CellwiseFunction(_class.getName(), + _classBytes, bcVect2, bcMatrices, scalars, mcIn.getRowsPerBlock()), true); MatrixBlock tmpMB = RDDAggregateUtils.aggStable(out, aggop); sec.setVariable(_out.getName(), new DoubleObject(tmpMB.getValue(0, 0))); } @@ -163,11 +163,9 @@ public class SpoofSPInstruction extends SPInstruction { { SpoofMultiAggregate op = (SpoofMultiAggregate) CodegenUtils.createInstance(_class); AggOp[] aggOps = op.getAggOps(); - - MatrixBlock tmpMB = in.mapToPair(new MultiAggregateFunction( - _class.getName(), _classBytes, bcVect2, bcMatrices, scalars)) + MatrixBlock tmpMB = in.mapToPair(new MultiAggregateFunction(_class.getName(), + _classBytes, bcVect2, bcMatrices, scalars, mcIn.getRowsPerBlock())) .values().fold(new MatrixBlock(), new MultiAggAggregateFunction(aggOps) ); - sec.setMatrixOutput(_out.getName(), tmpMB, getExtendedOpcode()); } else if(_class.getSuperclass() == SpoofOuterProduct.class) //OUTER @@ -207,8 +205,8 @@ public class SpoofSPInstruction extends SPInstruction { SpoofRowwise op = (SpoofRowwise) CodegenUtils.createInstance(_class); long clen2 = op.getRowType().isConstDim2(op.getConstDim2()) ? op.getConstDim2() : op.getRowType().isRowTypeB1() ? sec.getMatrixCharacteristics(_in[1].getName()).getCols() : -1; - RowwiseFunction fmmc = new RowwiseFunction(_class.getName(), - _classBytes, bcVect2, bcMatrices, scalars, (int)mcIn.getCols(), (int)clen2); + RowwiseFunction fmmc = new RowwiseFunction(_class.getName(), _classBytes, bcVect2, + bcMatrices, scalars, mcIn.getRowsPerBlock(), (int)mcIn.getCols(), (int)clen2); out = in.mapPartitionsToPair(fmmc, op.getRowType()==RowType.ROW_AGG || op.getRowType() == RowType.NO_AGG); @@ -422,12 +420,15 @@ public class SpoofSPInstruction extends SPInstruction { { private static final long serialVersionUID = -7926980450209760212L; + private final int _brlen; private final int _clen; private final int _clen2; private SpoofRowwise _op = null; - public RowwiseFunction(String className, byte[] classBytes, boolean[] bcInd, ArrayList<PartitionedBroadcast<MatrixBlock>> bcMatrices, ArrayList<ScalarObject> scalars, int clen, int clen2) { + public RowwiseFunction(String className, byte[] classBytes, boolean[] bcInd, ArrayList<PartitionedBroadcast<MatrixBlock>> bcMatrices, + ArrayList<ScalarObject> scalars, int brlen, int clen, int clen2) { super(className, classBytes, bcInd, bcMatrices, scalars); + _brlen = brlen; _clen = clen; _clen2 = clen; } @@ -453,11 +454,12 @@ public class SpoofSPInstruction extends SPInstruction { Tuple2<MatrixIndexes,MatrixBlock[]> e = arg.next(); MatrixIndexes ixIn = e._1(); MatrixBlock[] blkIn = e._2(); + long rix = (ixIn.getRowIndex()-1) * _brlen; //0-based //prepare output and execute single-threaded operator ArrayList<MatrixBlock> inputs = getAllMatrixInputs(ixIn, blkIn); blkOut = aggIncr ? blkOut : new MatrixBlock(); - blkOut = _op.execute(inputs, _scalars, blkOut, false, aggIncr); + blkOut = _op.execute(inputs, _scalars, blkOut, false, aggIncr, rix); if( !aggIncr ) { MatrixIndexes ixOut = new MatrixIndexes(ixIn.getRowIndex(), _op.getRowType()!=RowType.NO_AGG ? 1 : ixIn.getColumnIndex()); @@ -482,10 +484,12 @@ public class SpoofSPInstruction extends SPInstruction { { private static final long serialVersionUID = -8209188316939435099L; - private SpoofOperator _op = null; + private SpoofCellwise _op = null; + private final int _brlen; - public CellwiseFunction(String className, byte[] classBytes, boolean[] bcInd, ArrayList<PartitionedBroadcast<MatrixBlock>> bcMatrices, ArrayList<ScalarObject> scalars) { + public CellwiseFunction(String className, byte[] classBytes, boolean[] bcInd, ArrayList<PartitionedBroadcast<MatrixBlock>> bcMatrices, ArrayList<ScalarObject> scalars, int brlen) { super(className, classBytes, bcInd, bcMatrices, scalars); + _brlen = brlen; } @Override @@ -495,7 +499,7 @@ public class SpoofSPInstruction extends SPInstruction { //lazy load of shipped class if( _op == null ) { Class<?> loadedClass = CodegenUtils.getClassSync(_className, _classBytes); - _op = (SpoofOperator) CodegenUtils.createInstance(loadedClass); + _op = (SpoofCellwise) CodegenUtils.createInstance(loadedClass); } List<Tuple2<MatrixIndexes, MatrixBlock>> ret = new ArrayList<>(); @@ -507,19 +511,20 @@ public class SpoofSPInstruction extends SPInstruction { MatrixIndexes ixOut = ixIn; MatrixBlock blkOut = new MatrixBlock(); ArrayList<MatrixBlock> inputs = getAllMatrixInputs(ixIn, blkIn); + long rix = (ixIn.getRowIndex()-1) * _brlen; //0-based //execute core operation - if(((SpoofCellwise)_op).getCellType()==CellType.FULL_AGG) { - ScalarObject obj = _op.execute(inputs, _scalars, 1); + if( _op.getCellType()==CellType.FULL_AGG ) { + ScalarObject obj = _op.execute(inputs, _scalars, 1, rix); blkOut.reset(1, 1); - blkOut.quickSetValue(0, 0, obj.getDoubleValue()); + blkOut.quickSetValue(0, 0, obj.getDoubleValue()); } else { - if(((SpoofCellwise)_op).getCellType()==CellType.ROW_AGG) + if( _op.getCellType()==CellType.ROW_AGG ) ixOut = new MatrixIndexes(ixOut.getRowIndex(), 1); else if(((SpoofCellwise)_op).getCellType()==CellType.COL_AGG) ixOut = new MatrixIndexes(1, ixOut.getColumnIndex()); - blkOut = _op.execute(inputs, _scalars, blkOut); + blkOut = _op.execute(inputs, _scalars, blkOut, 1, rix); } ret.add(new Tuple2<>(ixOut, blkOut)); } @@ -532,10 +537,12 @@ public class SpoofSPInstruction extends SPInstruction { { private static final long serialVersionUID = -5224519291577332734L; - private SpoofOperator _op = null; + private SpoofMultiAggregate _op = null; + private final int _brlen; - public MultiAggregateFunction(String className, byte[] classBytes, boolean[] bcInd, ArrayList<PartitionedBroadcast<MatrixBlock>> bcMatrices, ArrayList<ScalarObject> scalars) { + public MultiAggregateFunction(String className, byte[] classBytes, boolean[] bcInd, ArrayList<PartitionedBroadcast<MatrixBlock>> bcMatrices, ArrayList<ScalarObject> scalars, int brlen) { super(className, classBytes, bcInd, bcMatrices, scalars); + _brlen = brlen; } @Override @@ -545,13 +552,14 @@ public class SpoofSPInstruction extends SPInstruction { //lazy load of shipped class if( _op == null ) { Class<?> loadedClass = CodegenUtils.getClassSync(_className, _classBytes); - _op = (SpoofOperator) CodegenUtils.createInstance(loadedClass); + _op = (SpoofMultiAggregate) CodegenUtils.createInstance(loadedClass); } - + //execute core operation ArrayList<MatrixBlock> inputs = getAllMatrixInputs(arg._1(), arg._2()); MatrixBlock blkOut = new MatrixBlock(); - blkOut = _op.execute(inputs, _scalars, blkOut); + long rix = (arg._1().getRowIndex()-1) * _brlen; //0-based + blkOut = _op.execute(inputs, _scalars, blkOut, 1, rix); return new Tuple2<>(arg._1(), blkOut); } http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java index f9b4a4a..76469de 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/CellwiseTmplTest.java @@ -57,6 +57,7 @@ public class CellwiseTmplTest extends AutomatedTestBase private static final String TEST_NAME19 = TEST_NAME+19; //sum(ifelse(true,Y,Z))+sum(ifelse(false,Y,Z)) private static final String TEST_NAME20 = TEST_NAME+20; //bitwAnd() operation private static final String TEST_NAME21 = TEST_NAME+21; //relu operation, (X>0)*dout + private static final String TEST_NAME22 = TEST_NAME+22; //sum(X * seq(1,N) + t(seq(M,1))) private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + CellwiseTmplTest.class.getSimpleName() + "/"; @@ -69,7 +70,7 @@ public class CellwiseTmplTest extends AutomatedTestBase @Override public void setUp() { TestUtils.clearAssertionInformation(); - for( int i=1; i<=21; i++ ) { + for( int i=1; i<=22; i++ ) { addTestConfiguration( TEST_NAME+i, new TestConfiguration( TEST_CLASS_DIR, TEST_NAME+i, new String[] {String.valueOf(i)}) ); } @@ -366,6 +367,21 @@ public class CellwiseTmplTest extends AutomatedTestBase public void testCodegenCellwiseRewrite21_sp() { testCodegenIntegration( TEST_NAME21, true, ExecType.SPARK ); } + + @Test + public void testCodegenCellwiseRewrite22() { + testCodegenIntegration( TEST_NAME22, true, ExecType.CP ); + } + + @Test + public void testCodegenCellwise22() { + testCodegenIntegration( TEST_NAME22, false, ExecType.CP ); + } + + @Test + public void testCodegenCellwiseRewrite22_sp() { + testCodegenIntegration( TEST_NAME22, true, ExecType.SPARK ); + } private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType ) { @@ -433,6 +449,8 @@ public class CellwiseTmplTest extends AutomatedTestBase Assert.assertTrue(!heavyHittersContainsSubString("uack+")); else if( testname.equals(TEST_NAME17) ) Assert.assertTrue(!heavyHittersContainsSubString("xor")); + else if( testname.equals(TEST_NAME22) ) + Assert.assertTrue(!heavyHittersContainsSubString("seq")); } finally { rtplatform = platformOld; http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java index 6cb853d..e07c543 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/codegen/RowAggTmplTest.java @@ -77,6 +77,7 @@ public class RowAggTmplTest extends AutomatedTestBase private static final String TEST_NAME38 = TEST_NAME+"38"; //sigmoid(X/rowSums) private static final String TEST_NAME39 = TEST_NAME+"39"; //BitwAnd operation private static final String TEST_NAME40 = TEST_NAME+"40"; //relu operation -> (X>0)* dout + private static final String TEST_NAME41 = TEST_NAME+"41"; //X*rowSums(X/seq(1,N)+t(seq(M,1))) private static final String TEST_DIR = "functions/codegen/"; private static final String TEST_CLASS_DIR = TEST_DIR + RowAggTmplTest.class.getSimpleName() + "/"; @@ -88,7 +89,7 @@ public class RowAggTmplTest extends AutomatedTestBase @Override public void setUp() { TestUtils.clearAssertionInformation(); - for(int i=1; i<=40; i++) + for(int i=1; i<=41; i++) addTestConfiguration( TEST_NAME+i, new TestConfiguration(TEST_CLASS_DIR, TEST_NAME+i, new String[] { String.valueOf(i) }) ); } @@ -691,6 +692,21 @@ public class RowAggTmplTest extends AutomatedTestBase public void testCodegenRowAgg40SP() { testCodegenIntegration( TEST_NAME40, false, ExecType.SPARK ); } + + @Test + public void testCodegenRowAggRewrite41CP() { + testCodegenIntegration( TEST_NAME41, true, ExecType.CP ); + } + + @Test + public void testCodegenRowAgg41CP() { + testCodegenIntegration( TEST_NAME41, false, ExecType.CP ); + } + + @Test + public void testCodegenRowAgg41SP() { + testCodegenIntegration( TEST_NAME41, false, ExecType.SPARK ); + } private void testCodegenIntegration( String testname, boolean rewrites, ExecType instType ) { @@ -748,6 +764,8 @@ public class RowAggTmplTest extends AutomatedTestBase && !heavyHittersContainsSubString("cbind")); if( testname.equals(TEST_NAME36) ) Assert.assertTrue(!heavyHittersContainsSubString("xor")); + if( testname.equals(TEST_NAME41) ) + Assert.assertTrue(!heavyHittersContainsSubString("seq")); } finally { rtplatform = platformOld; http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/test/scripts/functions/codegen/cellwisetmpl22.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/cellwisetmpl22.R b/src/test/scripts/functions/codegen/cellwisetmpl22.R new file mode 100644 index 0000000..09c7fe5 --- /dev/null +++ b/src/test/scripts/functions/codegen/cellwisetmpl22.R @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = matrix(7, 500, 2); +t1 = as.matrix(seq(1,nrow(X))) %*% matrix(1,1,2); +t2 = as.matrix(seq(nrow(X),1)) %*% matrix(1,1,2); +R = X * t1 + t2; + +writeMM(as(R,"CsparseMatrix"), paste(args[2], "S", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/test/scripts/functions/codegen/cellwisetmpl22.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/cellwisetmpl22.dml b/src/test/scripts/functions/codegen/cellwisetmpl22.dml new file mode 100644 index 0000000..1132619 --- /dev/null +++ b/src/test/scripts/functions/codegen/cellwisetmpl22.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix(7, 500, 2); + +while(FALSE){} + +R = X * seq(1,nrow(X)) + seq(nrow(X),1); + +write(R, $1) http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/test/scripts/functions/codegen/rowAggPattern41.R ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/rowAggPattern41.R b/src/test/scripts/functions/codegen/rowAggPattern41.R new file mode 100644 index 0000000..947c446 --- /dev/null +++ b/src/test/scripts/functions/codegen/rowAggPattern41.R @@ -0,0 +1,31 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +args<-commandArgs(TRUE) +options(digits=22) +library("Matrix") + +X = matrix(7, 500, 2); +t1 = as.matrix(seq(1,nrow(X))) %*% matrix(1,1,2); +t2 = as.matrix(seq(nrow(X),1))%*% matrix(1,1,2); +R = X * (rowSums(X/t1 + t2)%*%matrix(1,1,2)); + +writeMM(as(R,"CsparseMatrix"), paste(args[2], "S", sep="")); http://git-wip-us.apache.org/repos/asf/systemml/blob/d0b4373f/src/test/scripts/functions/codegen/rowAggPattern41.dml ---------------------------------------------------------------------- diff --git a/src/test/scripts/functions/codegen/rowAggPattern41.dml b/src/test/scripts/functions/codegen/rowAggPattern41.dml new file mode 100644 index 0000000..60b8a47 --- /dev/null +++ b/src/test/scripts/functions/codegen/rowAggPattern41.dml @@ -0,0 +1,28 @@ +#------------------------------------------------------------- +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +#------------------------------------------------------------- + +X = matrix(7, 500, 2); + +while(FALSE){} + +R = X * rowSums(X/seq(1,nrow(X)) + seq(nrow(X),1)); + +write(R, $1)
