Repository: incubator-systemml Updated Branches: refs/heads/master 3757995b5 -> d65c7b5bb
[SYSTEMML-1358] Fix minor memory leaks for tests (cla, test utils) Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/6946ff04 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/6946ff04 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/6946ff04 Branch: refs/heads/master Commit: 6946ff047f3f67f1b252261980473fd51b456280 Parents: 3757995 Author: Matthias Boehm <[email protected]> Authored: Fri Mar 3 19:04:11 2017 -0800 Committer: Matthias Boehm <[email protected]> Committed: Fri Mar 3 19:54:02 2017 -0800 ---------------------------------------------------------------------- .../estim/CompressedSizeEstimatorSample.java | 10 +- .../org/apache/sysml/test/utils/TestUtils.java | 239 +++++-------------- 2 files changed, 65 insertions(+), 184 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6946ff04/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java b/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java index a59893d..92461d4 100644 --- a/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java +++ b/src/main/java/org/apache/sysml/runtime/compress/estim/CompressedSizeEstimatorSample.java @@ -45,20 +45,12 @@ public class CompressedSizeEstimatorSample extends CompressedSizeEstimator public static final boolean HAAS_AND_STOKES_UJ2A_CUT2 = true; //cut frequency in half public static final boolean HAAS_AND_STOKES_UJ2A_SOLVE = true; //true recommended public static final int MAX_SOLVE_CACHE_SIZE = 64*1024; //global 2MB cache - //note: we use a relatively high ALPHA2 and the cut-in-half approach because it - //leads to moderate overestimation (compared to systematic underestimation) in - //order to follow a conservative approach private static final Log LOG = LogFactory.getLog(CompressedSizeEstimatorSample.class.getName()); - - private static ThreadLocal<RandomDataGenerator> _rng = new ThreadLocal<RandomDataGenerator>() { - protected RandomDataGenerator initialValue() { return new RandomDataGenerator(); } - }; private int[] _sampleRows = null; private HashMap<Integer, Double> _solveCache = null; - public CompressedSizeEstimatorSample(MatrixBlock data, int sampleSize) throws DMLRuntimeException { @@ -315,7 +307,7 @@ public class CompressedSizeEstimatorSample extends CompressedSizeEstimator private static int[] getSortedUniformSample(int range, int smplSize) { if (smplSize == 0) return new int[] {}; - RandomDataGenerator rng = _rng.get(); + RandomDataGenerator rng = new RandomDataGenerator(); int[] sample = rng.nextPermutation(range, smplSize); Arrays.sort(sample); return sample; http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/6946ff04/src/test/java/org/apache/sysml/test/utils/TestUtils.java ---------------------------------------------------------------------- diff --git a/src/test/java/org/apache/sysml/test/utils/TestUtils.java b/src/test/java/org/apache/sysml/test/utils/TestUtils.java index a383a48..4cb7df0 100644 --- a/src/test/java/org/apache/sysml/test/utils/TestUtils.java +++ b/src/test/java/org/apache/sysml/test/utils/TestUtils.java @@ -95,81 +95,6 @@ public class TestUtils private static ArrayList<String> _AssertInfos = new ArrayList<String>(); private static boolean _AssertOccured = false; - /** - * <p> - * Compares to arrays for equality. The elements in the array can be in - * different order. - * </p> - * - * @param expecteds - * expected values - * @param actuals - * actual values - */ - public static void assertInterchangedArraysEquals(String[] expecteds, String[] actuals) { - assertEquals("different number of elements in arrays", expecteds.length, actuals.length); - ArrayList<Integer> foundIndexes = new ArrayList<Integer>(); - expactation: for (int i = 0; i < expecteds.length; i++) { - for (int j = 0; j < actuals.length; j++) { - if (expecteds[i] == actuals[j] && !foundIndexes.contains(Integer.valueOf(j))) { - foundIndexes.add(Integer.valueOf(j)); - continue expactation; - } - } - fail("Missing element " + expecteds[i]); - } - } - - /** - * <p> - * Compares to arrays for equality. The elements in the array can be in - * different order. - * </p> - * - * @param expecteds - * expected values - * @param actuals - * actual values - */ - public static void assertInterchangedArraysEquals(int[] expecteds, int[] actuals) { - assertEquals("different number of elements in arrays", expecteds.length, actuals.length); - ArrayList<Integer> foundIndexes = new ArrayList<Integer>(); - expactation: for (int i = 0; i < expecteds.length; i++) { - for (int j = 0; j < actuals.length; j++) { - if (expecteds[i] == actuals[j] && !foundIndexes.contains(Integer.valueOf(j))) { - foundIndexes.add(Integer.valueOf(j)); - continue expactation; - } - } - fail("Missing element " + expecteds[i]); - } - } - - /** - * <p> - * Compares to arrays for equality. The elements in the array can be in - * different order. - * </p> - * - * @param expecteds - * expected values - * @param actuals - * actual values - */ - public static void assertInterchangedArraysEquals(double[] expecteds, double[] actuals) { - assertEquals("different number of elements in arrays", expecteds.length, actuals.length); - ArrayList<Integer> foundIndexes = new ArrayList<Integer>(); - expactation: for (int i = 0; i < expecteds.length; i++) { - for (int j = 0; j < actuals.length; j++) { - if (expecteds[i] == actuals[j] && !foundIndexes.contains(Integer.valueOf(j))) { - foundIndexes.add(Integer.valueOf(j)); - continue expactation; - } - } - fail("Missing element " + expecteds[i]); - } - } - /* Compare expected scalar generated by Java with actual scalar generated by DML */ public static void compareDMLScalarWithJavaScalar(String expectedFile, String actualFile, double epsilon) { try { @@ -337,7 +262,7 @@ public class TestUtils double v = Double.parseDouble(st.nextToken()); actualValues.put(new CellIndex(i, j), v); } - + outIn.close(); int countErrors = 0; @@ -348,9 +273,7 @@ public class TestUtils expectedValue = 0.0; if (actualValue == null) actualValue = 0.0; - - // System.out.println("actual value: "+actualValue+", expected value: "+expectedValue); - + if (!compareCellValue(expectedValue, actualValue, epsilon, false)) { System.out.println(expectedFile+": "+index+" mismatch: expected " + expectedValue + ", actual " + actualValue); countErrors++; @@ -361,6 +284,7 @@ public class TestUtils fail("unable to read file: " + e.getMessage()); } } + /** * <p> * Compares the expected values calculated in Java by testcase and which are @@ -420,9 +344,7 @@ public class TestUtils expectedValue = 0.0; if (actualValue == null) actualValue = 0.0; - - // System.out.println("actual value: "+actualValue+", expected value: "+expectedValue); - + if (!compareCellValue(expectedValue, actualValue, epsilon, false)) { System.out.println(expectedFile+": "+index+" mismatch: expected " + expectedValue + ", actual " + actualValue); countErrors++; @@ -482,10 +404,6 @@ public class TestUtils * @param filePath * @return */ - - // TODO: we must use http://www.inf.uni-konstanz.de/algo/lehre/ws05/pp/mtj/mvio/MatrixVectorReader.html - // to read matrices from R - public static HashMap<CellIndex, Double> readRMatrixFromFS(String filePath) { HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>(); @@ -600,7 +518,9 @@ public class TestUtils FileStatus[] outFiles = fs.listStatus(outDirectory); for (FileStatus file : outFiles) { FSDataInputStream fsout = fs.open(file.getPath()); - sb.append(IOUtils.toString(new InputStreamReader(fsout))); + InputStreamReader is = new InputStreamReader(fsout); + sb.append(IOUtils.toString(is)); + is.close(); } return sb.toString(); } catch (IOException e) { @@ -681,8 +601,7 @@ public class TestUtils out.append(fileContents); } } finally { - if (null != out) - out.close(); + IOUtilFunctions.closeSilently(out); } csvFile = tmp.getCanonicalPath(); @@ -1078,7 +997,7 @@ public class TestUtils HashMap<CellIndex, Double> expectedValues = new HashMap<CellIndex, Double>(); HashMap<CellIndex, Double> actualValues = new HashMap<CellIndex, Double>(); String line; - /** skip both R header lines */ + // skip both R header lines compareIn.readLine(); compareIn.readLine(); while ((line = compareIn.readLine()) != null) { @@ -1376,20 +1295,13 @@ public class TestUtils */ public static double[][] generateTestMatrix(int rows, int cols, double min, double max, double sparsity, long seed) { double[][] matrix = new double[rows][cols]; - Random random; - if (seed == -1) - random = TestUtils.random; - else - random = new Random(seed); - + Random random = (seed == -1) ? TestUtils.random : new Random(seed); for (int i = 0; i < rows; i++) { for (int j = 0; j < cols; j++) { if (random.nextDouble() > sparsity) continue; matrix[i][j] = (random.nextDouble() * (max - min) + min); - // System.out.print(matrix[i][j] + "(" + i + "," + j + ")"); } - // System.out.println(); } return matrix; @@ -1418,12 +1330,7 @@ public class TestUtils */ public static double[][] generateNonZeroTestMatrix(int rows, int cols, double min, double max, long seed) { double[][] matrix = new double[rows][cols]; - Random random; - if (seed == -1) - random = TestUtils.random; - else - random = new Random(seed); - + Random random = (seed == -1) ? TestUtils.random : new Random(seed); for (int i = 0; i < rows; i++) { for (int j = 0; j < cols; j++) { double randValue; @@ -1468,12 +1375,8 @@ public class TestUtils Path inFile = new Path(file); DataOutputStream out = fs.create(inFile); PrintWriter pw = new PrintWriter(out); - Random random; - if (seed == -1) - random = TestUtils.random; - else - random = new Random(seed); - + Random random = (seed == -1) ? TestUtils.random : new Random(seed); + for (int i = 1; i <= rows; i++) { for (int j = 1; j <= cols; j++) { if (random.nextDouble() > sparsity) @@ -1484,7 +1387,6 @@ public class TestUtils } } pw.close(); - out.close(); } catch (IOException e) { fail("unable to write test matrix: " + e.getMessage()); } @@ -1512,10 +1414,8 @@ public class TestUtils try { //create outputstream to HDFS / FS and writer - DataOutputStream out = null; FileSystem fs = FileSystem.get(conf); - out = fs.create(new Path(file), true); - + DataOutputStream out = fs.create(new Path(file), true); BufferedWriter pw = new BufferedWriter(new OutputStreamWriter(out)); //writer actual matrix @@ -1536,7 +1436,6 @@ public class TestUtils //close writer and streams pw.close(); - out.close(); } catch (IOException e) { @@ -1605,7 +1504,6 @@ public class TestUtils //close writer and streams pw.close(); - out.close(); } catch (IOException e) { @@ -1695,7 +1593,6 @@ public class TestUtils PrintWriter pw = new PrintWriter(out); pw.println(value); pw.close(); - out.close(); } catch (IOException e) { fail("unable to write test scalar (" + file + "): " + e.getMessage()); } @@ -1867,20 +1764,9 @@ public class TestUtils * temporary script file */ public static void renameTempDMLScript(String dmlScriptFile) { - // try { - // FileSystem fs = FileSystem.get(conf); - // Path oldPath = new Path(dmlScriptFile + "t"); - // Path newPath = new Path(dmlScriptFile); - // if (fs.exists(oldPath)) - // fs.rename(oldPath, newPath); File oldPath = new File(dmlScriptFile + "t"); File newPath = new File(dmlScriptFile); oldPath.renameTo(newPath); - - /* - * } catch (IOException e) { e.printStackTrace(); - * fail("unable to write dml script back: " + e.getMessage()); } - */ } /** @@ -2019,63 +1905,66 @@ public class TestUtils for (FileStatus file : files) { SequenceFile.Reader reader = new SequenceFile.Reader(FileSystem.get(conf), file.getPath(), conf); - while (reader.next(indexes, value)) { - if (value.getNumRows() < rowsInBlock) { - if (rowsInLastBlock == -1) - rowsInLastBlock = value.getNumRows(); - else if (rowsInLastBlock != value.getNumRows()) - fail("invalid block sizes"); - rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows()); - } else if (value.getNumRows() == rowsInBlock) { - if (rows <= (indexes.getRowIndex() * rowsInBlock + value.getNumRows())) { + try { + while (reader.next(indexes, value)) { + if (value.getNumRows() < rowsInBlock) { if (rowsInLastBlock == -1) - rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows()); - else + rowsInLastBlock = value.getNumRows(); + else if (rowsInLastBlock != value.getNumRows()) fail("invalid block sizes"); - } - } else { - fail("invalid block sizes"); - } - - if (value.getNumColumns() < colsInBlock) { - if (colsInLastBlock == -1) - colsInLastBlock = value.getNumColumns(); - else if (colsInLastBlock != value.getNumColumns()) + rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows()); + } else if (value.getNumRows() == rowsInBlock) { + if (rows <= (indexes.getRowIndex() * rowsInBlock + value.getNumRows())) { + if (rowsInLastBlock == -1) + rows = (int) ((indexes.getRowIndex() - 1) * rowsInBlock + value.getNumRows()); + else + fail("invalid block sizes"); + } + } else { fail("invalid block sizes"); - cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns()); - } else if (value.getNumColumns() == colsInBlock) { - if (cols <= (indexes.getColumnIndex() * colsInBlock + value.getNumColumns())) { + } + + if (value.getNumColumns() < colsInBlock) { if (colsInLastBlock == -1) - cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns()); - else + colsInLastBlock = value.getNumColumns(); + else if (colsInLastBlock != value.getNumColumns()) fail("invalid block sizes"); + cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns()); + } else if (value.getNumColumns() == colsInBlock) { + if (cols <= (indexes.getColumnIndex() * colsInBlock + value.getNumColumns())) { + if (colsInLastBlock == -1) + cols = (int) ((indexes.getColumnIndex() - 1) * colsInBlock + value.getNumColumns()); + else + fail("invalid block sizes"); + } + } else { + fail("invalid block sizes"); } - } else { - fail("invalid block sizes"); - } - - if (value.isInSparseFormat()) { - Iterator<IJV> iter = value.getSparseBlockIterator(); - while( iter.hasNext() ) - { - IJV cell = iter.next(); - valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + cell.getI()), - (int) ((indexes.getColumnIndex() - 1) * colsInBlock + cell.getJ())), cell.getV()); - } - - } else { - double[] valuesInBlock = value.getDenseBlock(); - for (int i = 0; i < value.getNumRows(); i++) { - for (int j = 0; j < value.getNumColumns(); j++) { - valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + i), - (int) ((indexes.getColumnIndex() - 1) * colsInBlock + j)), valuesInBlock[i - * value.getNumColumns() + j]); + + if (value.isInSparseFormat()) { + Iterator<IJV> iter = value.getSparseBlockIterator(); + while( iter.hasNext() ) + { + IJV cell = iter.next(); + valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + cell.getI()), + (int) ((indexes.getColumnIndex() - 1) * colsInBlock + cell.getJ())), cell.getV()); + } + + } else { + double[] valuesInBlock = value.getDenseBlock(); + for (int i = 0; i < value.getNumRows(); i++) { + for (int j = 0; j < value.getNumColumns(); j++) { + valueMap.put(new MatrixIndexes(((indexes.getRowIndex() - 1) * rowsInBlock + i), + (int) ((indexes.getColumnIndex() - 1) * colsInBlock + j)), valuesInBlock[i + * value.getNumColumns() + j]); + } } } } } - - reader.close(); + finally { + IOUtilFunctions.closeSilently(reader); + } } long nonZeros = 0;
