http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java index d6e3939..8839223 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/mapred/ReduceBase.java @@ -30,7 +30,6 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.OutputCollector; import org.apache.hadoop.mapred.Reporter; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.DMLUnsupportedOperationException; import org.apache.sysml.runtime.functionobjects.Plus; import org.apache.sysml.runtime.instructions.mr.AggregateInstruction; import org.apache.sysml.runtime.instructions.mr.MRInstruction; @@ -126,8 +125,6 @@ public class ReduceBase extends MRBaseForCommonInstructions } - } catch (DMLUnsupportedOperationException e) { - throw new RuntimeException(e); } catch (DMLRuntimeException e) { throw new RuntimeException(e); } @@ -271,8 +268,7 @@ public class ReduceBase extends MRBaseForCommonInstructions protected void outputResultsFromCachedValues(Reporter reporter) throws IOException { - for(int i=0; i<resultIndexes.length; i++) - { + for(int i=0; i<resultIndexes.length; i++) { byte output=resultIndexes[i]; ArrayList<IndexedMatrixValue> outValues=cachedValues.get(output); if(outValues==null) @@ -280,52 +276,14 @@ public class ReduceBase extends MRBaseForCommonInstructions for(IndexedMatrixValue outValue: outValues) collectOutput_N_Increase_Counter(outValue.getIndexes(), outValue.getValue(), i, reporter); - // LOG.info("output: "+outValue.getIndexes()+" -- "+outValue.getValue()+" ~~ tag: "+output); - // System.out.println("Reducer output: "+outValue.getIndexes()+" -- "+outValue.getValue()+" ~~ tag: "+output); } } //process one aggregate instruction -/* private void processAggregateHelp(MatrixIndexes indexes, TaggedMatrixValue tagged, - AggregateInstruction instruction) - throws DMLUnsupportedOperationException, DMLRuntimeException - { - AggregateOperator aggOp=(AggregateOperator)instruction.getOperator(); - - IndexedMatrixValue out=cachedValues.get(instruction.output); - IndexedMatrixValue correction=null; - if(aggOp.correctionExists) - correction=correctionCache.get(instruction.output); - if(out==null) - { - out=cachedValues.holdPlace(instruction.output, valueClass); - out.getIndexes().setIndexes(indexes); - if(aggOp.correctionExists ) - { - if(correction==null) - correction=correctionCache.holdPlace(instruction.output, valueClass); - OperationsOnMatrixValues.startAggregation(out.getValue(), correction.getValue(), aggOp, - tagged.getBaseObject().getNumRows(), tagged.getBaseObject().getNumColumns(), - tagged.getBaseObject().isInSparseFormat()); - }else - OperationsOnMatrixValues.startAggregation(out.getValue(), null, aggOp, - tagged.getBaseObject().getNumRows(), tagged.getBaseObject().getNumColumns(), - tagged.getBaseObject().isInSparseFormat()); - } - - if(aggOp.correctionExists) - OperationsOnMatrixValues.incrementalAggregation(out.getValue(), correction.getValue(), - tagged.getBaseObject(), (AggregateOperator)instruction.getOperator()); - else - OperationsOnMatrixValues.incrementalAggregation(out.getValue(), null, - tagged.getBaseObject(), (AggregateOperator)instruction.getOperator()); - } - */ - //process one aggregate instruction private void processAggregateHelp(long row, long col, MatrixValue value, AggregateInstruction instruction, boolean imbededCorrection) - throws DMLUnsupportedOperationException, DMLRuntimeException + throws DMLRuntimeException { AggregateOperator aggOp=(AggregateOperator)instruction.getOperator(); @@ -355,19 +313,14 @@ public class ReduceBase extends MRBaseForCommonInstructions OperationsOnMatrixValues.startAggregation(out.getValue(), null, aggOp, value.getNumRows(), value.getNumColumns(), value.isInSparseFormat(), imbededCorrection); - //System.out.println("after start: "+out); } - //System.out.println("value to add: "+value); + if(aggOp.correctionExists)// && !imbededCorrection) - { - //System.out.println("incremental aggregation maxindex/minindex"); OperationsOnMatrixValues.incrementalAggregation(out.getValue(), correction.getValue(), value, (AggregateOperator)instruction.getOperator(), imbededCorrection); - } else OperationsOnMatrixValues.incrementalAggregation(out.getValue(), null, value, (AggregateOperator)instruction.getOperator(), imbededCorrection); - //System.out.println("after increment: "+out); } //process all the aggregate instructions for one group of values
http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java b/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java index 4b3f78f..85f8738 100644 --- a/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java +++ b/src/main/java/org/apache/sysml/udf/ExternalFunctionInvocationInstruction.java @@ -20,7 +20,6 @@ package org.apache.sysml.udf; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.DMLUnsupportedOperationException; import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; import org.apache.sysml.runtime.instructions.Instruction; @@ -75,7 +74,7 @@ public class ExternalFunctionInvocationInstruction extends Instruction @Override public void processInstruction(ExecutionContext ec) - throws DMLRuntimeException, DMLUnsupportedOperationException + throws DMLRuntimeException { //do nothing (not applicable because this instruction is only used as //meta data container) http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/main/java/org/apache/sysml/utils/AppException.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/AppException.java b/src/main/java/org/apache/sysml/utils/AppException.java deleted file mode 100644 index a8a23f2..0000000 --- a/src/main/java/org/apache/sysml/utils/AppException.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.utils; - -import org.apache.sysml.api.DMLException; - -/** - * This exception should be thrown to flag errors by systemML apps - * including data transformation, post-processing etc. - */ -public class AppException extends DMLException -{ - - private static final long serialVersionUID = 1L; - - public AppException(String string) { - super(string); - } - - public AppException(String string, Exception ex){ - super(string,ex); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java index c4de6b7..0930475 100644 --- a/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java +++ b/src/main/java/org/apache/sysml/yarn/DMLAppMasterUtils.java @@ -35,7 +35,6 @@ import org.apache.sysml.hops.OptimizerUtils.OptimizationLevel; import org.apache.sysml.lops.Lop; import org.apache.sysml.lops.LopsException; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.DMLUnsupportedOperationException; import org.apache.sysml.runtime.controlprogram.Program; import org.apache.sysml.runtime.controlprogram.ProgramBlock; import org.apache.sysml.runtime.controlprogram.parfor.stat.InfrastructureAnalyzer; @@ -116,11 +115,10 @@ public class DMLAppMasterUtils * @throws DMLRuntimeException * @throws HopsException * @throws LopsException - * @throws DMLUnsupportedOperationException * @throws IOException */ public static void setupProgramMappingRemoteMaxMemory(Program prog) - throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException + throws DMLRuntimeException, HopsException, LopsException, IOException { if( DMLScript.isActiveAM() && isResourceOptimizerEnabled() ) { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java index 4b31af2..701ba76 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java @@ -41,7 +41,6 @@ import org.apache.sysml.parser.StatementBlock; import org.apache.sysml.parser.WhileStatementBlock; import org.apache.sysml.parser.Expression.DataType; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.DMLUnsupportedOperationException; import org.apache.sysml.runtime.controlprogram.ForProgramBlock; import org.apache.sysml.runtime.controlprogram.FunctionProgramBlock; import org.apache.sysml.runtime.controlprogram.IfProgramBlock; @@ -183,13 +182,12 @@ public class ResourceOptimizer * @param rc * @return * @throws IOException - * @throws DMLUnsupportedOperationException * @throws LopsException * @throws HopsException * @throws DMLRuntimeException */ public static ArrayList<ProgramBlock> compileProgram( ArrayList<ProgramBlock> prog, ResourceConfig rc ) - throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException + throws DMLRuntimeException, HopsException, LopsException, IOException { //recompile program block hierarchy to list of blocks and apply optimized resource configuration ArrayList<ProgramBlock> B = compileProgram(prog, null, rc.getCPResource(), rc.getMaxMRResource()); @@ -205,13 +203,12 @@ public class ResourceOptimizer * @param B * @return * @throws IOException - * @throws DMLUnsupportedOperationException * @throws LopsException * @throws HopsException * @throws DMLRuntimeException */ private static ArrayList<ProgramBlock> compileProgram( ArrayList<ProgramBlock> prog, ArrayList<ProgramBlock> B, double cp, double mr ) - throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException + throws DMLRuntimeException, HopsException, LopsException, IOException { if( B == null ) //init { @@ -234,14 +231,13 @@ public class ResourceOptimizer * @param pb * @param Bp * @return - * @throws IOException - * @throws DMLUnsupportedOperationException + * @throws IOException * @throws LopsException * @throws HopsException * @throws DMLRuntimeException */ private static ArrayList<ProgramBlock> compileProgram( ProgramBlock pb, ArrayList<ProgramBlock> B, double cp, double mr ) - throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException + throws DMLRuntimeException, HopsException, LopsException, IOException { if (pb instanceof FunctionProgramBlock) { @@ -317,11 +313,10 @@ public class ResourceOptimizer * @throws DMLRuntimeException * @throws HopsException * @throws LopsException - * @throws DMLUnsupportedOperationException * @throws IOException */ private static void recompileProgramBlocks( ArrayList<ProgramBlock> pbs, long cp, double[][] memo ) - throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException + throws DMLRuntimeException, HopsException, LopsException, IOException { for( int i=0; i<pbs.size(); i++ ) { @@ -339,11 +334,10 @@ public class ResourceOptimizer * @throws DMLRuntimeException * @throws HopsException * @throws LopsException - * @throws DMLUnsupportedOperationException * @throws IOException */ private static void recompileProgramBlock( ProgramBlock pb, long cp, long mr ) - throws DMLRuntimeException, HopsException, LopsException, DMLUnsupportedOperationException, IOException + throws DMLRuntimeException, HopsException, LopsException, IOException { //init compiler memory budget InfrastructureAnalyzer.setLocalMaxMemory( cp ); @@ -455,11 +449,10 @@ public class ResourceOptimizer * @param pb * @return * @throws DMLRuntimeException - * @throws DMLUnsupportedOperationException * @throws HopsException */ private static double getProgramCosts( ProgramBlock pb ) - throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException + throws DMLRuntimeException, HopsException { double val = 0; if( COST_INDIVIDUAL_BLOCKS ) { @@ -483,11 +476,10 @@ public class ResourceOptimizer /** * * @param prog - * @throws DMLUnsupportedOperationException * @throws DMLRuntimeException */ private static double getProgramCosts( Program prog ) - throws DMLRuntimeException, DMLUnsupportedOperationException + throws DMLRuntimeException { //we need to cost the entire program in order to take in-memory status into account ExecutionContext ec = ExecutionContextFactory.createContext(); @@ -708,10 +700,9 @@ public class ResourceOptimizer * @param min * @return * @throws DMLRuntimeException - * @throws DMLUnsupportedOperationException */ private static double[][] initLocalMemoTable( ArrayList<ProgramBlock> Bp, double min ) - throws DMLRuntimeException, DMLUnsupportedOperationException + throws DMLRuntimeException { //allocate memo structure int len = Bp.size(); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java index 0bb5cc3..416a615 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizerCPMigration.java @@ -168,7 +168,7 @@ public class ResourceOptimizerCPMigration public HashMap<Long, ProgramBlock> sbIdMap; // Pointer to all reachable program blocks - public CPMigrationOptimizer(Program rtprog) throws DMLRuntimeException, DMLUnsupportedOperationException, HopsException { + public CPMigrationOptimizer(Program rtprog) throws DMLRuntimeException, HopsException { prog = rtprog; copiedProgramBlocks = ProgramConverter.rcreateDeepCopyProgramBlocks(prog._programBlocks, 1, -1, new HashSet<String>(), false); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6c89c3a7/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java index 757a418..866a93d 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameReadWriteTest.java @@ -25,7 +25,6 @@ import java.util.List; import org.apache.sysml.parser.Expression.ValueType; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.DMLUnsupportedOperationException; import org.apache.sysml.runtime.instructions.cp.AppendCPInstruction.AppendType; import org.apache.sysml.runtime.io.FrameReader; import org.apache.sysml.runtime.io.FrameReaderFactory; @@ -34,6 +33,7 @@ import org.apache.sysml.runtime.io.FrameWriterFactory; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.OutputInfo; +import org.apache.sysml.runtime.util.MapReduceTool; import org.apache.sysml.runtime.util.UtilFunctions; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.utils.TestUtils; @@ -155,11 +155,11 @@ public class FrameReadWriteTest extends AutomatedTestBase * @param frame2 * @param fprop * @return - * @throws DMLUnsupportedOperationException, DMLRuntimeException, IOException + * @throws DMLRuntimeException, IOException */ void writeAndVerifyData(OutputInfo oinfo, FrameBlock frame1, FrameBlock frame2, CSVFileFormatProperties fprop) - throws DMLUnsupportedOperationException, DMLRuntimeException, IOException + throws DMLRuntimeException, IOException { String fname1 = TEST_DIR + "/frameData1"; String fname2 = TEST_DIR + "/frameData2"; @@ -179,6 +179,8 @@ public class FrameReadWriteTest extends AutomatedTestBase // Verify that data read with original frames verifyFrameData(frame1, frame1Read); verifyFrameData(frame2, frame2Read); + MapReduceTool.deleteFileIfExistOnHDFS(fname1); + MapReduceTool.deleteFileIfExistOnHDFS(fname2); } }
