Repository: incubator-systemml Updated Branches: refs/heads/master 56e6b6e94 -> c8007f454
[MINOR] Remove unused tests and test utility functions Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/edd3491f Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/edd3491f Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/edd3491f Branch: refs/heads/master Commit: edd3491fa07e8897b2f119f438b88e8bf769f100 Parents: 56e6b6e Author: Matthias Boehm <mboe...@gmail.com> Authored: Sat Apr 22 22:29:40 2017 -0700 Committer: Matthias Boehm <mboe...@gmail.com> Committed: Sat Apr 22 22:29:40 2017 -0700 ---------------------------------------------------------------------- .../AutomatedScalabilityTestBase.java | 93 --------- .../test/integration/AutomatedTestBase.java | 14 -- .../BinaryMatrixCharacteristics.java | 86 --------- .../integration/TestMatrixCharacteristics.java | 102 ---------- .../integration/functions/data/WriteTest.java | 28 +-- .../scalability/LinearRegressionTest.java | 57 ------ .../integration/scalability/PageRankTest.java | 56 ------ .../org/apache/sysml/test/utils/TestUtils.java | 187 +------------------ .../integration/scalability/ZPackageSuite.java | 37 ---- 9 files changed, 23 insertions(+), 637 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java deleted file mode 100644 index 0624c55..0000000 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedScalabilityTestBase.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration; - -import java.util.ArrayList; - -import org.junit.After; -import org.junit.Before; - -import org.apache.sysml.test.utils.TestUtils; - - -public abstract class AutomatedScalabilityTestBase extends AutomatedTestBase -{ - - - private static final boolean RUN_SCALABILITY_TESTS = false; - - private long[] timeMeasurements; - protected int[][] matrixSizes; - protected ArrayList<TestMatrixCharacteristics> inputMatrices; - - - @Before - public void setUpScalabilityTest() { - inputMatrices = new ArrayList<TestMatrixCharacteristics>(); - } - - public abstract void setUp(); - - protected void runTest() { - if(!RUN_SCALABILITY_TESTS) - return; - - timeMeasurements = new long[matrixSizes.length]; - for(int i = 0; i < matrixSizes.length; i++) { - for(TestMatrixCharacteristics inputMatrix : inputMatrices) { - if(inputMatrix.getRows() == -1) - inputMatrix.setRows(matrixSizes[i][inputMatrix.getRowsIndexInMatrixSizes()]); - if(inputMatrix.getCols() == -1) - inputMatrix.setCols(matrixSizes[i][inputMatrix.getColsIndexInMatrixSizes()]); - createRandomMatrix(inputMatrix); - } - - for(int j = 0; j < matrixSizes[i].length; j++) { - testVariables.put(Integer.toString(j), Integer.toString(matrixSizes[i][j])); - } - - long startingTime = System.currentTimeMillis(); - super.runTest(); - long finishingTime = System.currentTimeMillis(); - timeMeasurements[i] = (finishingTime - startingTime); - - TestUtils.renameTempDMLScript(baseDirectory + selectedTest + ".dml"); - } - } - - protected TestMatrixCharacteristics addInputMatrix(String name, int rows, int cols, double min, double max, - double sparsity, long seed) { - TestMatrixCharacteristics inputMatrix = new TestMatrixCharacteristics(name, rows, cols, min, max, - sparsity, seed); - inputMatrices.add(inputMatrix); - return inputMatrix; - } - - @After - public void displayTimeMeasurements() { - if(!RUN_SCALABILITY_TESTS) - return; - - for(long timeMeasurement : timeMeasurements) { - System.out.println("measured time: " + timeMeasurement); - } - } - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java index d1cba1d..a822ae7 100644 --- a/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java +++ b/src/test/java/org/apache/sysml/test/integration/AutomatedTestBase.java @@ -479,20 +479,6 @@ public abstract class AutomatedTestBase return matrix; } - /** - * <p> - * Generates a random matrix with the specified characteristics and writes - * it to a file. - * </p> - * - * @param matrix - * matrix characteristics - */ - protected void createRandomMatrix(TestMatrixCharacteristics matrix) { - createRandomMatrix(matrix.getMatrixName(), matrix.getRows(), matrix.getCols(), matrix.getMinValue(), matrix - .getMaxValue(), matrix.getSparsity(), matrix.getSeed()); - } - private void cleanupExistingData(String fname, boolean cleanupRData) throws IOException { MapReduceTool.deleteFileIfExistOnHDFS(fname); MapReduceTool.deleteFileIfExistOnHDFS(fname + ".mtd"); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java b/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java deleted file mode 100644 index c36c737..0000000 --- a/src/test/java/org/apache/sysml/test/integration/BinaryMatrixCharacteristics.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration; - - -/** - * <p>Contains characteristics about a binary matrix.</p> - * - * - */ -public class BinaryMatrixCharacteristics -{ - - - private double[][] values; - private int rows; - private int cols; - private int rowsInBlock; - private int rowsInLastBlock; - private int colsInBlock; - private int colsInLastBlock; - private long nonZeros; - - - public BinaryMatrixCharacteristics(double[][] values, int rows, int cols, int rowsInBlock, int rowsInLastBlock, - int colsInBlock, int colsInLastBlock, long nonZeros) { - this.values = values; - this.rows = rows; - this.cols = cols; - this.rowsInBlock = rowsInBlock; - this.rowsInLastBlock = rowsInLastBlock; - this.colsInBlock = colsInBlock; - this.colsInLastBlock = colsInLastBlock; - this.nonZeros = nonZeros; - } - - public double[][] getValues() { - return values; - } - - public int getRows() { - return rows; - } - - public int getCols() { - return cols; - } - - public int getRowsInBlock() { - return rowsInBlock; - } - - public int getRowsInLastBlock() { - return rowsInLastBlock; - } - - public int getColsInBlock() { - return colsInBlock; - } - - public int getColsInLastBlock() { - return colsInLastBlock; - } - - public long getNonZeros() { - return nonZeros; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java b/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java deleted file mode 100644 index 38c51a7..0000000 --- a/src/test/java/org/apache/sysml/test/integration/TestMatrixCharacteristics.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration; - -public class TestMatrixCharacteristics -{ - - - private String matrixName; - private int rows; - private int rowsIndexInMatrixSizes; - private int cols; - private int colsIndexInMatrixSizes; - private double minValue; - private double maxValue; - private double sparsity; - private long seed; - - - public TestMatrixCharacteristics(String matrixName, int rows, int cols, double minValue, double maxValue, - double sparsity, long seed) { - this.matrixName = matrixName; - this.rows = rows; - this.cols = cols; - this.minValue = minValue; - this.maxValue = maxValue; - this.sparsity = sparsity; - this.seed = seed; - } - - public String getMatrixName() { - return matrixName; - } - - public int getRows() { - return rows; - } - - public void setRows(int rows) { - this.rows = rows; - } - - public int getRowsIndexInMatrixSizes() { - return rowsIndexInMatrixSizes; - } - - public TestMatrixCharacteristics setRowsIndexInMatrixSizes(int rowsIndexInMatrixSizes) { - this.rowsIndexInMatrixSizes = rowsIndexInMatrixSizes; - return this; - } - - public int getCols() { - return cols; - } - - public void setCols(int cols) { - this.cols = cols; - } - - public int getColsIndexInMatrixSizes() { - return colsIndexInMatrixSizes; - } - - public TestMatrixCharacteristics setColsIndexInMatrixSizes(int colsIndexInMatrixSizes) { - this.colsIndexInMatrixSizes = colsIndexInMatrixSizes; - return this; - } - - public double getMinValue() { - return minValue; - } - - public double getMaxValue() { - return maxValue; - } - - public double getSparsity() { - return sparsity; - } - - public long getSeed() { - return seed; - } - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java index 448893c..441c9c7 100644 --- a/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java +++ b/src/test/java/org/apache/sysml/test/integration/functions/data/WriteTest.java @@ -19,12 +19,13 @@ package org.apache.sysml.test.integration.functions.data; -import static org.junit.Assert.*; - import org.junit.Test; - +import org.apache.sysml.runtime.io.MatrixReader; +import org.apache.sysml.runtime.io.MatrixReaderFactory; import org.apache.sysml.runtime.matrix.MatrixCharacteristics; -import org.apache.sysml.test.integration.BinaryMatrixCharacteristics; +import org.apache.sysml.runtime.matrix.data.InputInfo; +import org.apache.sysml.runtime.matrix.data.MatrixBlock; +import org.apache.sysml.runtime.util.DataConverter; import org.apache.sysml.test.integration.AutomatedTestBase; import org.apache.sysml.test.integration.TestConfiguration; import org.apache.sysml.test.utils.TestUtils; @@ -96,16 +97,15 @@ public class WriteTest extends AutomatedTestBase runTest(); - //compareResults(); - - BinaryMatrixCharacteristics matrix = TestUtils.readBlocksFromSequenceFile(output("a"),1000,1000); - assertEquals(rows, matrix.getRows()); - assertEquals(cols, matrix.getCols()); - double[][] matrixValues = matrix.getValues(); - for(int i = 0; i < rows; i++) { - for(int j = 0; j < cols; j++) { - assertEquals(i + "," + j, a[i][j], matrixValues[i][j], 0); - } + //read and compare output matrix + try { + MatrixReader reader = MatrixReaderFactory.createMatrixReader(InputInfo.BinaryBlockInputInfo); + MatrixBlock mb = reader.readMatrixFromHDFS(output("a"), rows, cols, 1000, 1000, -1); + checkDMLMetaDataFile("a", new MatrixCharacteristics(rows,cols,1000,1000)); + TestUtils.compareMatrices(a, DataConverter.convertToDoubleMatrix(mb), rows, cols, 0); + } + catch(Exception ex) { + throw new RuntimeException(ex); } } http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java b/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java deleted file mode 100644 index 9795245..0000000 --- a/src/test/java/org/apache/sysml/test/integration/scalability/LinearRegressionTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.scalability; - -import org.junit.Test; - -import org.apache.sysml.test.integration.AutomatedScalabilityTestBase; -import org.apache.sysml.test.integration.TestConfiguration; -import org.apache.sysml.test.utils.TestUtils; - - - -public class LinearRegressionTest extends AutomatedScalabilityTestBase -{ - - private static final String TEST_DIR = "test/scripts/scalability/linear_regression/"; - private static final String TEST_CLASS_DIR = TEST_DIR + LinearRegressionTest.class.getSimpleName() + "/"; - - - @Override - public void setUp() { - TestUtils.clearAssertionInformation(); - addTestConfiguration("LinearRegressionTest", new TestConfiguration(TEST_CLASS_DIR, "LinearRegressionTest", new String[] { "w" })); - matrixSizes = new int[][] { - { 19004, 15436 } - }; - } - - @Test - public void testLinearRegression() { - TestConfiguration config = getTestConfiguration("LinearRegressionTest"); - loadTestConfiguration(config); - - addInputMatrix("g", -1, -1, 0, 1, 0.00594116, -1).setRowsIndexInMatrixSizes(0).setColsIndexInMatrixSizes(1); - addInputMatrix("b", -1, 1, 1, 10, 1, -1).setRowsIndexInMatrixSizes(0); - - runTest(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java b/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java deleted file mode 100644 index d8159ee..0000000 --- a/src/test/java/org/apache/sysml/test/integration/scalability/PageRankTest.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.scalability; - -import org.junit.Test; - -import org.apache.sysml.test.integration.AutomatedScalabilityTestBase; -import org.apache.sysml.test.integration.TestConfiguration; - - - -public class PageRankTest extends AutomatedScalabilityTestBase -{ - private static final String TEST_DIR = "test/scripts/scalability/page_rank/"; - private static final String TEST_CLASS_DIR = TEST_DIR + PageRankTest.class.getSimpleName() + "/"; - - - @Override - public void setUp() { - addTestConfiguration("PageRankTest", new TestConfiguration(TEST_CLASS_DIR, "PageRankTest", new String[] { "p" })); - matrixSizes = new int[][] { - { 9914 } - }; - } - - @Test - public void testPageRank() { - TestConfiguration config = getTestConfiguration("PageRankTest"); - loadTestConfiguration(config); - - addInputMatrix("g", -1, -1, 1, 1, 0.000374962, -1).setRowsIndexInMatrixSizes(0).setColsIndexInMatrixSizes(0); - addInputMatrix("p", -1, 1, 1, 1, 1, -1).setRowsIndexInMatrixSizes(0); - addInputMatrix("e", -1, 1, 1, 1, 1, -1).setRowsIndexInMatrixSizes(0); - addInputMatrix("u", 1, -1, 1, 1, 1, -1).setColsIndexInMatrixSizes(0); - - runTest(); - } - -} http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test/java/org/apache/sysml/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java index 2d1a4c1..19b741d 100644 --- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java +++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java @@ -40,7 +40,6 @@ import java.text.NumberFormat; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.Iterator; import java.util.Locale; import java.util.Random; import java.util.StringTokenizer; @@ -58,8 +57,8 @@ import org.apache.sysml.runtime.DMLRuntimeException; import org.apache.sysml.runtime.io.FrameWriter; import org.apache.sysml.runtime.io.FrameWriterFactory; import org.apache.sysml.runtime.io.IOUtilFunctions; +import org.apache.sysml.runtime.matrix.MatrixCharacteristics; import org.apache.sysml.runtime.matrix.data.FrameBlock; -import org.apache.sysml.runtime.matrix.data.IJV; import org.apache.sysml.runtime.matrix.data.MatrixBlock; import org.apache.sysml.runtime.matrix.data.MatrixCell; import org.apache.sysml.runtime.matrix.data.MatrixIndexes; @@ -67,7 +66,6 @@ import org.apache.sysml.runtime.matrix.data.OutputInfo; import org.apache.sysml.runtime.matrix.data.MatrixValue.CellIndex; import org.apache.sysml.runtime.util.UtilFunctions; import org.apache.sysml.test.integration.AutomatedTestBase; -import org.apache.sysml.test.integration.BinaryMatrixCharacteristics; /** @@ -1048,8 +1046,10 @@ public class TestUtils * Checks a matrix against a number of specifications. * </p> * - * @param matrix - * matrix + * @param data + * matrix data + * @param mc + * matrix characteristics * @param rows * number of rows * @param cols @@ -1059,14 +1059,13 @@ public class TestUtils * @param max * maximum value */ - public static void checkMatrix(BinaryMatrixCharacteristics matrix, long rows, long cols, double min, double max) { - assertEquals(rows, matrix.getRows()); - assertEquals(cols, matrix.getCols()); - double[][] matrixValues = matrix.getValues(); + public static void checkMatrix(double[][] data, MatrixCharacteristics mc, long rows, long cols, double min, double max) { + assertEquals(rows, mc.getRows()); + assertEquals(cols, mc.getCols()); for (int i = 0; i < rows; i++) { for (int j = 0; j < cols; j++) { assertTrue("invalid value", - ((matrixValues[i][j] >= min && matrixValues[i][j] <= max) || matrixValues[i][j] == 0)); + ((data[i][j] >= min && data[i][j] <= max) || data[i][j] == 0)); } } } @@ -1819,174 +1818,6 @@ public class TestUtils /** * <p> - * Reads binary cells from a file. A matrix characteristic is created which - * contains the characteristics of the matrix read from the file and the - * values. - * </p> - * - * @param directory - * directory containing the matrix - * @return matrix characteristics - */ - @SuppressWarnings("deprecation") - public static BinaryMatrixCharacteristics readCellsFromSequenceFile(String directory) { - try { - FileSystem fs = FileSystem.get(conf); - FileStatus[] files = fs.listStatus(new Path(directory)); - - HashMap<MatrixIndexes, Double> valueMap = new HashMap<MatrixIndexes, Double>(); - int rows = 0; - int cols = 0; - MatrixIndexes indexes = new MatrixIndexes(); - MatrixCell value = new MatrixCell(); - for (FileStatus file : files) { - SequenceFile.Reader reader = null; - try { - reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf); - while (reader.next(indexes, value)) { - if (rows < indexes.getRowIndex()) - rows = (int) indexes.getRowIndex(); - if (cols < indexes.getColumnIndex()) - cols = (int) indexes.getColumnIndex(); - valueMap.put(new MatrixIndexes(indexes), value.getValue()); - } - } - finally { - IOUtilFunctions.closeSilently(reader); - } - } - - double[][] values = new double[rows][cols]; - long nonZeros = 0; - for (MatrixIndexes index : valueMap.keySet()) { - values[(int)index.getRowIndex() - 1][(int)index.getColumnIndex() - 1] = valueMap.get(index); - if (valueMap.get(index) != 0) - nonZeros++; - } - - return new BinaryMatrixCharacteristics(values, rows, cols, 0, 0, 0, 0, nonZeros); - } catch (IOException e) { - e.printStackTrace(); - fail("unable to read sequence file in " + directory); - } - - return null; - } - - /** - * <p> - * Reads binary blocks from a file. A matrix characteristic is created which - * contains the characteristics of the matrix read from the file and the - * values. - * </p> - * - * @param directory - * directory containing the matrix - * @param rowsInBlock - * rows in block - * @param colsInBlock - * columns in block - * @return matrix characteristics - */ - @SuppressWarnings("deprecation") - public static BinaryMatrixCharacteristics readBlocksFromSequenceFile(String directory, int rowsInBlock, - int colsInBlock) { - try { - FileSystem fs = FileSystem.get(conf); - FileStatus[] files = fs.listStatus(new Path(directory)); - - HashMap<MatrixIndexes, Double> valueMap = new HashMap<MatrixIndexes, Double>(); - int rowsInLastBlock = -1; - int colsInLastBlock = -1; - int rows = 0; - int cols = 0; - MatrixIndexes indexes = new MatrixIndexes(); - MatrixBlock value = new MatrixBlock(); - for (FileStatus file : files) { - SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf); - - try { - while (reader.next(indexes, value)) { - if (value.getNumRows() < rowsInBlock) { - if (rowsInLastBlock == -1) - rowsInLastBlock = value.getNumRows(); - else if (rowsInLastBlock != value.getNumRows()) - fail("invalid block sizes"); - rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows()); - } else if (value.getNumRows() == rowsInBlock) { - if (rows <= (indexes.getRowIndex() * rowsInBlock + value.getNumRows())) { - if (rowsInLastBlock == -1) - rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows()); - else - fail("invalid block sizes"); - } - } else { - fail("invalid block sizes"); - } - - if (value.getNumColumns() < colsInBlock) { - if (colsInLastBlock == -1) - colsInLastBlock = value.getNumColumns(); - else if (colsInLastBlock != value.getNumColumns()) - fail("invalid block sizes"); - cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns()); - } else if (value.getNumColumns() == colsInBlock) { - if (cols <= (indexes.getColumnIndex() * colsInBlock + value.getNumColumns())) { - if (colsInLastBlock == -1) - cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns()); - else - fail("invalid block sizes"); - } - } else { - fail("invalid block sizes"); - } - - if (value.isInSparseFormat()) { - Iterator<IJV> iter = value.getSparseBlockIterator(); - while( iter.hasNext() ) - { - IJV cell = iter.next(); - valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + cell.getI()), - (int) ((indexes.getColumnIndex() - 1) * colsInBlock + cell.getJ())), cell.getV()); - } - - } else { - double[] valuesInBlock = value.getDenseBlock(); - for (int i = 0; i < value.getNumRows(); i++) { - for (int j = 0; j < value.getNumColumns(); j++) { - valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + i), - (int) ((indexes.getColumnIndex() - 1) * colsInBlock + j)), valuesInBlock[i - * value.getNumColumns() + j]); - } - } - } - } - } - finally { - IOUtilFunctions.closeSilently(reader); - } - } - - long nonZeros = 0; - double[][] values = new double[rows][cols]; - for (MatrixIndexes index : valueMap.keySet()) { - values[(int)index.getRowIndex()][(int)index.getColumnIndex()] = valueMap.get(index); - if (valueMap.get(index) != 0) - nonZeros++; - } - - return new BinaryMatrixCharacteristics(values, rows, cols, rowsInBlock, rowsInLastBlock, colsInBlock, - colsInLastBlock, nonZeros); - } catch (IOException e) { - e.printStackTrace(); - fail("unable to read sequence file in " + directory); - } - - return null; - } - - /** - * <p> * Returns the path to a file in a directory if it is the only file in the * directory. * </p> http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/edd3491f/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java ---------------------------------------------------------------------- diff --git a/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java b/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java deleted file mode 100644 index f0e3ae8..0000000 --- a/src/test_suites/java/org/apache/sysml/test/integration/scalability/ZPackageSuite.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.sysml.test.integration.scalability; - -import org.junit.runner.RunWith; -import org.junit.runners.Suite; - -/** Group together the tests in this package into a single suite so that the Maven build - * won't run two of them at once. */ -@RunWith(Suite.class) -@Suite.SuiteClasses({ - LinearRegressionTest.class, - PageRankTest.class -}) - - -/** This class is just a holder for the above JUnit annotations. */ -public class ZPackageSuite { - -}
