Repository: systemml Updated Branches: refs/heads/master f76f2138a -> 820b53046
http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/GMR.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java index f0c214e..ae9cfce 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/GMR.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/GMR.java @@ -47,7 +47,6 @@ import org.apache.sysml.runtime.instructions.mr.MRInstruction; import org.apache.sysml.runtime.instructions.mr.PickByCountInstruction; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; -import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.data.TaggedMatrixBlock; import org.apache.sysml.runtime.matrix.data.TaggedMatrixPackedCell; @@ -136,13 +135,13 @@ public class GMR PickFromCompactInputFormat.setKeyValueClasses(job, (Class<? extends WritableComparable>) inputInfos[ins.input1].inputKeyClass, inputInfos[ins.input1].inputValueClass); job.setInputFormat(PickFromCompactInputFormat.class); - PickFromCompactInputFormat.setZeroValues(job, (NumItemsByEachReducerMetaData)inputInfos[ins.input1].metadata); + PickFromCompactInputFormat.setZeroValues(job, (MetaDataNumItemsByEachReducer)inputInfos[ins.input1].metadata); if(ins.isValuePick) { double[] probs=MapReduceTool.readColumnVectorFromHDFS(inputs[ins.input2], inputInfos[ins.input2], rlens[ins.input2], clens[ins.input2], brlens[ins.input2], bclens[ins.input2]); - PickFromCompactInputFormat.setPickRecordsInEachPartFile(job, (NumItemsByEachReducerMetaData) inputInfos[ins.input1].metadata, probs); + PickFromCompactInputFormat.setPickRecordsInEachPartFile(job, (MetaDataNumItemsByEachReducer) inputInfos[ins.input1].metadata, probs); realinputs=new String[inputs.length-1]; realinputInfos=new InputInfo[inputs.length-1]; @@ -179,8 +178,8 @@ public class GMR }else { //PickFromCompactInputFormat.setPickRecordsInEachPartFile(job, (NumItemsByEachReducerMetaData) inputInfos[ins.input1].metadata, ins.cst, 1-ins.cst); - PickFromCompactInputFormat.setRangePickPartFiles(job, (NumItemsByEachReducerMetaData) inputInfos[ins.input1].metadata, ins.cst, 1-ins.cst); - realrlens[ins.input1]=UtilFunctions.getLengthForInterQuantile((NumItemsByEachReducerMetaData)inputInfos[ins.input1].metadata, ins.cst); + PickFromCompactInputFormat.setRangePickPartFiles(job, (MetaDataNumItemsByEachReducer) inputInfos[ins.input1].metadata, ins.cst, 1-ins.cst); + realrlens[ins.input1]=UtilFunctions.getLengthForInterQuantile((MetaDataNumItemsByEachReducer)inputInfos[ins.input1].metadata, ins.cst); realclens[ins.input1]=clens[ins.input1]; realbrlens[ins.input1]=1; realbclens[ins.input1]=1; http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java index a2f3f8d..3d846a7 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/JobReturn.java @@ -21,7 +21,6 @@ package org.apache.sysml.runtime.matrix; import org.apache.sysml.runtime.DMLRuntimeException; -import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData; import org.apache.sysml.runtime.matrix.data.OutputInfo; @@ -39,31 +38,31 @@ public class JobReturn public JobReturn(MatrixCharacteristics[] sts, boolean success) { successful = success; - metadata = new MatrixDimensionsMetaData[sts.length]; + metadata = new MetaData[sts.length]; for (int i = 0; i < sts.length; i++) { - metadata[i] = new MatrixDimensionsMetaData(sts[i]); + metadata[i] = new MetaData(sts[i]); } } public JobReturn(MatrixCharacteristics[] sts, OutputInfo[] infos, boolean success) throws DMLRuntimeException { successful = success; - metadata = new MatrixFormatMetaData[sts.length]; + metadata = new MetaDataFormat[sts.length]; for (int i = 0; i < sts.length; i++) { - metadata[i] = new MatrixFormatMetaData(sts[i], infos[i], OutputInfo.getMatchingInputInfo(infos[i])); + metadata[i] = new MetaDataFormat(sts[i], infos[i], OutputInfo.getMatchingInputInfo(infos[i])); } } public JobReturn(MatrixCharacteristics sts, OutputInfo info, boolean success) throws DMLRuntimeException { successful = success; - metadata = new MatrixFormatMetaData[1]; - metadata[0] = new MatrixFormatMetaData(sts, info, OutputInfo.getMatchingInputInfo(info)); + metadata = new MetaDataFormat[1]; + metadata[0] = new MetaDataFormat(sts, info, OutputInfo.getMatchingInputInfo(info)); } public JobReturn(MatrixCharacteristics mc, long[] items, int partition0, long number0s, boolean success) { successful = success; - metadata = new NumItemsByEachReducerMetaData[1]; - metadata[0] = new NumItemsByEachReducerMetaData(mc, items, partition0, number0s); + metadata = new MetaDataNumItemsByEachReducer[1]; + metadata[0] = new MetaDataNumItemsByEachReducer(mc, items, partition0, number0s); } public boolean checkReturnStatus() throws DMLRuntimeException { @@ -79,14 +78,4 @@ public class JobReturn public MetaData getMetaData(int i) { return metadata[i]; } - - /* - * Since MatrixCharacteristics is the most common type of metadata, we - * define a method to extract this information for a given output index. - */ - public MatrixCharacteristics getMatrixCharacteristics(int i) { - return ((MatrixDimensionsMetaData) metadata[i]) - .getMatrixCharacteristics(); - } - } http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java index cb7ca57..a0e81ed 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/MatrixCharacteristics.java @@ -21,6 +21,7 @@ package org.apache.sysml.runtime.matrix; import java.io.Serializable; +import java.util.Arrays; import java.util.HashMap; import org.apache.sysml.lops.MMTSJ.MMTSJType; @@ -409,24 +410,20 @@ public class MatrixCharacteristics implements Serializable } @Override - public boolean equals (Object anObject) - { - if (anObject instanceof MatrixCharacteristics) - { - MatrixCharacteristics mc = (MatrixCharacteristics) anObject; - return ((numRows == mc.numRows) && - (numColumns == mc.numColumns) && - (numRowsPerBlock == mc.numRowsPerBlock) && - (numColumnsPerBlock == mc.numColumnsPerBlock) && - (nonZero == mc.nonZero)) ; - } - else + public boolean equals (Object anObject) { + if( !(anObject instanceof MatrixCharacteristics) ) return false; + MatrixCharacteristics mc = (MatrixCharacteristics) anObject; + return ((numRows == mc.numRows) + && (numColumns == mc.numColumns) + && (numRowsPerBlock == mc.numRowsPerBlock) + && (numColumnsPerBlock == mc.numColumnsPerBlock) + && (nonZero == mc.nonZero)); } @Override - public int hashCode() - { - return super.hashCode(); + public int hashCode() { + return Arrays.hashCode(new long[]{numRows,numColumns, + numRowsPerBlock,numColumnsPerBlock,nonZero}); } } http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java deleted file mode 100644 index a9bce8a..0000000 --- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixDimensionsMetaData.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.sysml.runtime.matrix; - -public class MatrixDimensionsMetaData extends MetaData -{ - protected MatrixCharacteristics matchar; - - public MatrixDimensionsMetaData() { - matchar = null; - } - - public MatrixDimensionsMetaData(MatrixCharacteristics mc) { - matchar = mc; - } - - public MatrixCharacteristics getMatrixCharacteristics() { - return matchar; - } - - public void setMatrixCharacteristics(MatrixCharacteristics mc) { - matchar = mc; - } - - @Override - public boolean equals (Object anObject) - { - if (anObject instanceof MatrixDimensionsMetaData) - { - MatrixDimensionsMetaData md = (MatrixDimensionsMetaData) anObject; - return (matchar.equals (md.matchar)); - } - else - return false; - } - - @Override - public int hashCode() - { - //use identity hash code - return super.hashCode(); - } - - @Override - public String toString() { - return "[rows = " + matchar.getRows() + - ", cols = " + matchar.getCols() + - ", rpb = " + matchar.getRowsPerBlock() + - ", cpb = " + matchar.getColsPerBlock() + "]"; - } - - @Override - public Object clone() - { - MatrixCharacteristics mc = new MatrixCharacteristics(matchar); - MatrixDimensionsMetaData ret = new MatrixDimensionsMetaData(mc); - - return ret; - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java deleted file mode 100644 index 557e3eb..0000000 --- a/src/main/java/org/apache/sysml/runtime/matrix/MatrixFormatMetaData.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.sysml.runtime.matrix; - -import org.apache.sysml.runtime.matrix.data.InputInfo; -import org.apache.sysml.runtime.matrix.data.OutputInfo; - -public class MatrixFormatMetaData extends MatrixDimensionsMetaData -{ - private InputInfo iinfo; - private OutputInfo oinfo; - - public MatrixFormatMetaData(MatrixCharacteristics mc, OutputInfo oinfo_, InputInfo iinfo_ ) { - super(mc); - oinfo = oinfo_; - iinfo = iinfo_; - } - - public InputInfo getInputInfo() { - return iinfo; - } - - public OutputInfo getOutputInfo() { - return oinfo; - } - - @Override - public Object clone() - { - MatrixCharacteristics mc = new MatrixCharacteristics(matchar); - MatrixFormatMetaData meta = new MatrixFormatMetaData(mc, oinfo, iinfo); - - return meta; - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java index 09b5c78..bb15c78 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaData.java @@ -17,30 +17,44 @@ * under the License. */ - package org.apache.sysml.runtime.matrix; /** * Class to store metadata associated with a file (e.g., a matrix) on disk. - * This class must be extended to associate specific information with the file. * */ - -public abstract class MetaData +public class MetaData { - @Override - public abstract boolean equals (Object anObject); + protected final MatrixCharacteristics _mc; + + public MetaData(MatrixCharacteristics mc) { + _mc = mc; + } + + public MatrixCharacteristics getMatrixCharacteristics() { + return _mc; + } @Override - public int hashCode() - { - //use identity hash code - return super.hashCode(); + public boolean equals (Object anObject) { + if( !(anObject instanceof MetaData) ) + return false; + MetaData that = (MetaData)anObject; + return _mc.equals(that._mc); } @Override - public abstract String toString(); + public int hashCode() { + return _mc.hashCode(); + } + + @Override + public String toString() { + return _mc.toString(); + } - @Override - public abstract Object clone(); + @Override + public Object clone() { + return new MetaData(new MatrixCharacteristics(_mc)); + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java new file mode 100644 index 0000000..ad033e1 --- /dev/null +++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataFormat.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.sysml.runtime.matrix; + +import org.apache.sysml.runtime.matrix.data.InputInfo; +import org.apache.sysml.runtime.matrix.data.OutputInfo; + +public class MetaDataFormat extends MetaData +{ + private final InputInfo iinfo; + private final OutputInfo oinfo; + + public MetaDataFormat(MatrixCharacteristics mc, OutputInfo oinfo, InputInfo iinfo ) { + super(mc); + this.oinfo = oinfo; + this.iinfo = iinfo; + } + + public InputInfo getInputInfo() { + return iinfo; + } + + public OutputInfo getOutputInfo() { + return oinfo; + } + + @Override + public Object clone() { + return new MetaDataFormat(new MatrixCharacteristics(_mc), oinfo, iinfo); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java new file mode 100644 index 0000000..d728c10 --- /dev/null +++ b/src/main/java/org/apache/sysml/runtime/matrix/MetaDataNumItemsByEachReducer.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + + +package org.apache.sysml.runtime.matrix; + +public class MetaDataNumItemsByEachReducer extends MetaData +{ + private long[] numItems=null; + private int partitionOfZero=-1; + private long numberOfZero=0; + + public MetaDataNumItemsByEachReducer(MatrixCharacteristics mc, long[] nums, int part0, long num0) { + super(mc); + numItems=nums; + partitionOfZero=part0; + numberOfZero=num0; + } + + public long[] getNumItemsArray() { + return numItems; + } + + public int getPartitionOfZero() { + return partitionOfZero; + } + + public long getNumberOfZero() { + return numberOfZero; + } + + @Override + public Object clone() { + return new MetaDataNumItemsByEachReducer( + new MatrixCharacteristics(_mc), numItems, partitionOfZero, numberOfZero); + } +} http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java b/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java deleted file mode 100644 index ac82403..0000000 --- a/src/main/java/org/apache/sysml/runtime/matrix/data/NumItemsByEachReducerMetaData.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - - -package org.apache.sysml.runtime.matrix.data; - -import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData; - -public class NumItemsByEachReducerMetaData extends MatrixDimensionsMetaData -{ - - private long[] numItems=null; - private int partitionOfZero=-1; - private long numberOfZero=0; - - public NumItemsByEachReducerMetaData(MatrixCharacteristics mc, long[] nums, int part0, long num0) - { - super(mc); - numItems=nums; - partitionOfZero=part0; - numberOfZero=num0; - } - - public long[] getNumItemsArray() - { - return numItems; - } - - public int getPartitionOfZero() - { - return partitionOfZero; - } - - public long getNumberOfZero() - { - return numberOfZero; - } - - @Override - public Object clone() - { - MatrixCharacteristics mc = new MatrixCharacteristics(matchar); - NumItemsByEachReducerMetaData ret = new NumItemsByEachReducerMetaData(mc, numItems, partitionOfZero, numberOfZero); - - return ret; - } -} http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java index a615741..903c3bd 100644 --- a/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java +++ b/src/main/java/org/apache/sysml/runtime/matrix/sort/PickFromCompactInputFormat.java @@ -41,9 +41,9 @@ import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.Reporter; import org.apache.sysml.runtime.io.IOUtilFunctions; +import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer; import org.apache.sysml.runtime.matrix.data.MatrixCell; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; -import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData; import org.apache.sysml.runtime.matrix.data.Pair; //key class to read has to be DoubleWritable @@ -62,7 +62,7 @@ public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, M return false; } - public static void setZeroValues(JobConf job, NumItemsByEachReducerMetaData metadata) + public static void setZeroValues(JobConf job, MetaDataNumItemsByEachReducer metadata) { job.setInt(PARTITION_OF_ZERO, metadata.getPartitionOfZero()); job.setLong(NUMBER_OF_ZERO, metadata.getNumberOfZero()); @@ -141,7 +141,7 @@ public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, M } } - public static Set<Integer> setPickRecordsInEachPartFile(JobConf job, NumItemsByEachReducerMetaData metadata, double[] probs) + public static Set<Integer> setPickRecordsInEachPartFile(JobConf job, MetaDataNumItemsByEachReducer metadata, double[] probs) { HashMap<Integer, ArrayList<Pair<Integer, Integer>>> posMap = new HashMap<>(); getPointsInEachPartFile(metadata.getNumItemsArray(), probs, posMap); @@ -155,7 +155,7 @@ public class PickFromCompactInputFormat extends FileInputFormat<MatrixIndexes, M return posMap.keySet(); } - public static void setRangePickPartFiles(JobConf job, NumItemsByEachReducerMetaData metadata, double lbound, double ubound) { + public static void setRangePickPartFiles(JobConf job, MetaDataNumItemsByEachReducer metadata, double lbound, double ubound) { if(lbound<0 || lbound > 1 || ubound <0 || ubound >1 || lbound >= ubound ) { throw new RuntimeException("Invalid ranges for range pick: [" + lbound + "," + ubound + "]"); http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java index e8f206a..eb2b141 100644 --- a/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java +++ b/src/main/java/org/apache/sysml/runtime/util/MapReduceTool.java @@ -51,11 +51,11 @@ import org.apache.sysml.runtime.io.IOUtilFunctions; import org.apache.sysml.runtime.io.MatrixReader; import org.apache.sysml.runtime.io.MatrixReaderFactory; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; +import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer; import org.apache.sysml.runtime.matrix.data.CSVFileFormatProperties; import org.apache.sysml.runtime.matrix.data.FileFormatProperties; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; -import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData; import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.mapred.MRConfigurationNames; import org.apache.sysml.runtime.matrix.sort.ReadWithZeros; @@ -526,7 +526,7 @@ public class MapReduceTool return DataConverter.convertToDoubleVector(mb, false); } - public static double median(String dir, NumItemsByEachReducerMetaData metadata) throws IOException { + public static double median(String dir, MetaDataNumItemsByEachReducer metadata) throws IOException { long[] counts=metadata.getNumItemsArray(); long[] ranges=new long[counts.length]; ranges[0]=counts[0]; @@ -539,11 +539,11 @@ public class MapReduceTool } - public static double pickValue(String dir, NumItemsByEachReducerMetaData metadata, double p) throws IOException { + public static double pickValue(String dir, MetaDataNumItemsByEachReducer metadata, double p) throws IOException { return pickValueWeight(dir, metadata, p, false)[0]; } - public static double[] pickValueWeight(String dir, NumItemsByEachReducerMetaData metadata, double p, boolean average) + public static double[] pickValueWeight(String dir, MetaDataNumItemsByEachReducer metadata, double p, boolean average) throws IOException { long[] counts=metadata.getNumItemsArray(); http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java index 5644ac6..199108b 100644 --- a/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/util/UtilFunctions.java @@ -27,9 +27,9 @@ import java.util.Set; import org.apache.commons.lang.ArrayUtils; import org.apache.sysml.parser.Expression.ValueType; +import org.apache.sysml.runtime.matrix.MetaDataNumItemsByEachReducer; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; -import org.apache.sysml.runtime.matrix.data.NumItemsByEachReducerMetaData; import org.apache.sysml.runtime.matrix.data.Pair; import org.apache.sysml.runtime.matrix.mapred.IndexedMatrixValue; @@ -242,7 +242,7 @@ public class UtilFunctions return new IndexRange(iRowStart, iRowEnd, iColStart, iColEnd); } - public static long getTotalLength(NumItemsByEachReducerMetaData metadata) { + public static long getTotalLength(MetaDataNumItemsByEachReducer metadata) { long[] counts=metadata.getNumItemsArray(); long total=0; for(long count: counts) @@ -250,7 +250,7 @@ public class UtilFunctions return total; } - public static long getLengthForInterQuantile(NumItemsByEachReducerMetaData metadata, double p) + public static long getLengthForInterQuantile(MetaDataNumItemsByEachReducer metadata, double p) { long total = UtilFunctions.getTotalLength(metadata); long lpos=(long)Math.ceil(total*p);//lower bound is inclusive http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/udf/Matrix.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/udf/Matrix.java b/src/main/java/org/apache/sysml/udf/Matrix.java index 1e6dfbd..5da92e3 100644 --- a/src/main/java/org/apache/sysml/udf/Matrix.java +++ b/src/main/java/org/apache/sysml/udf/Matrix.java @@ -29,7 +29,7 @@ import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; import org.apache.sysml.runtime.io.MatrixReader; import org.apache.sysml.runtime.io.MatrixReaderFactory; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; +import org.apache.sysml.runtime.matrix.MetaDataFormat; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.OutputInfo; @@ -216,7 +216,7 @@ public class Matrix extends FunctionParameter int cblen = ConfigurationManager.getBlocksize(); MatrixCharacteristics mc = new MatrixCharacteristics(_rows, _cols, rblen, cblen, nnz); - MatrixFormatMetaData mfmd = new MatrixFormatMetaData(mc, oinfo, iinfo); + MetaDataFormat mfmd = new MetaDataFormat(mc, oinfo, iinfo); try { _mo = new MatrixObject(Expression.ValueType.DOUBLE, _filePath, mfmd); http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java index 0378924..c5cabbe 100644 --- a/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java +++ b/src/main/java/org/apache/sysml/yarn/ropt/ResourceOptimizer.java @@ -57,7 +57,7 @@ import org.apache.sysml.runtime.controlprogram.parfor.stat.Timing; import org.apache.sysml.runtime.instructions.Instruction; import org.apache.sysml.runtime.instructions.MRJobInstruction; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.runtime.matrix.MatrixDimensionsMetaData; +import org.apache.sysml.runtime.matrix.MetaData; import org.apache.sysml.yarn.DMLYarnClient; import org.apache.sysml.yarn.ropt.YarnOptimizerUtils.GridEnumType; @@ -440,8 +440,7 @@ public class ResourceOptimizer String varname = hop.getName(); MatrixCharacteristics mc = new MatrixCharacteristics(hop.getDim1(), hop.getDim2(), (int)hop.getRowsInBlock(), (int)hop.getColsInBlock(), hop.getNnz()); - MatrixDimensionsMetaData md = new MatrixDimensionsMetaData(mc); - MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", md); + MatrixObject mo = new MatrixObject(ValueType.DOUBLE, "/tmp", new MetaData(mc)); vars.put(varname, mo); } http://git-wip-us.apache.org/repos/asf/systemml/blob/d7d312c3/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java index 7d0abfa..e3c8281 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/frame/FrameEvictionTest.java @@ -26,7 +26,7 @@ import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.controlprogram.caching.FrameObject; import org.apache.sysml.runtime.controlprogram.caching.LazyWriteBuffer; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.runtime.matrix.MatrixFormatMetaData; +import org.apache.sysml.runtime.matrix.MetaDataFormat; import org.apache.sysml.runtime.matrix.data.FrameBlock; import org.apache.sysml.runtime.matrix.data.InputInfo; import org.apache.sysml.runtime.matrix.data.MatrixBlock; @@ -202,7 +202,7 @@ public class FrameEvictionTest extends AutomatedTestBase //create frame object MatrixCharacteristics mc = new MatrixCharacteristics(rows, schema.length, -1, -1, -1); - MatrixFormatMetaData meta = new MatrixFormatMetaData (mc, + MetaDataFormat meta = new MetaDataFormat (mc, OutputInfo.BinaryBlockOutputInfo, InputInfo.BinaryBlockInputInfo); FrameObject fo = new FrameObject("fA", meta, schema); fo.acquireModify(fA);
