Repository: systemml Updated Branches: refs/heads/master 0abeb60b3 -> 607a402bc
[SYSTEMML-1738] Fix various stream resource leaks via auto closing Closes #749. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/607a402b Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/607a402b Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/607a402b Branch: refs/heads/master Commit: 607a402bcf809d951e9634479d0db635e19c28ce Parents: 0abeb60 Author: Janardhan Pulivarthi <[email protected]> Authored: Sun Apr 1 16:18:19 2018 -0700 Committer: Matthias Boehm <[email protected]> Committed: Sun Apr 1 16:18:19 2018 -0700 ---------------------------------------------------------------------- .../org/apache/sysml/api/jmlc/Connection.java | 24 ++++++++++---- .../sysml/api/mlcontext/ScriptFactory.java | 10 +++--- .../org/apache/sysml/parser/ParserWrapper.java | 34 ++++++++++++-------- .../instructions/gpu/context/JCudaKernels.java | 5 +-- .../runtime/transform/meta/TfMetaUtils.java | 16 ++++++--- .../org/apache/sysml/utils/NativeHelper.java | 25 ++++++-------- 6 files changed, 65 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/api/jmlc/Connection.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java index 8e8bb29..8684e72 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java @@ -421,7 +421,9 @@ public class Connection implements Closeable * @throws IOException if IOException occurs */ public double[][] convertToDoubleMatrix(String input, int rows, int cols) throws IOException { - return convertToDoubleMatrix(IOUtilFunctions.toInputStream(input), rows, cols); + try( InputStream is = IOUtilFunctions.toInputStream(input) ) { + return convertToDoubleMatrix(is, rows, cols); + } } /** @@ -467,7 +469,9 @@ public class Connection implements Closeable * @throws IOException if IOException occurs */ public MatrixBlock convertToMatrix(String input, String meta) throws IOException { - return convertToMatrix(IOUtilFunctions.toInputStream(input), meta); + try( InputStream is = IOUtilFunctions.toInputStream(input) ) { + return convertToMatrix(is, meta); + } } /** @@ -509,7 +513,9 @@ public class Connection implements Closeable * @throws IOException if IOException occurs */ public MatrixBlock convertToMatrix(String input, int rows, int cols) throws IOException { - return convertToMatrix(IOUtilFunctions.toInputStream(input), rows, cols); + try( InputStream is = IOUtilFunctions.toInputStream(input) ) { + return convertToMatrix(is, rows, cols); + } } /** @@ -655,7 +661,9 @@ public class Connection implements Closeable * @throws IOException if IOException occurs */ public String[][] convertToStringFrame(String input, int rows, int cols) throws IOException { - return convertToStringFrame(IOUtilFunctions.toInputStream(input), rows, cols); + try( InputStream is = IOUtilFunctions.toInputStream(input) ) { + return convertToStringFrame(is, rows, cols); + } } /** @@ -701,7 +709,9 @@ public class Connection implements Closeable * @throws IOException if IOException occurs */ public FrameBlock convertToFrame(String input, String meta) throws IOException { - return convertToFrame(IOUtilFunctions.toInputStream(input), meta); + try( InputStream is = IOUtilFunctions.toInputStream(input) ) { + return convertToFrame(is, meta); + } } /** @@ -743,7 +753,9 @@ public class Connection implements Closeable * @throws IOException if IOException occurs */ public FrameBlock convertToFrame(String input, int rows, int cols) throws IOException { - return convertToFrame(IOUtilFunctions.toInputStream(input), rows, cols); + try( InputStream is = IOUtilFunctions.toInputStream(input) ) { + return convertToFrame(is, rows, cols); + } } /** http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java index 10cd947..567a2d7 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptFactory.java @@ -298,8 +298,11 @@ public class ScriptFactory { if (!resourcePath.startsWith("/")) { resourcePath = "/" + resourcePath; } - InputStream inputStream = ScriptFactory.class.getResourceAsStream(resourcePath); - return scriptFromInputStream(inputStream, scriptType).setName(resourcePath); + try( InputStream inputStream = ScriptFactory.class.getResourceAsStream(resourcePath) ) { + return scriptFromInputStream(inputStream, scriptType).setName(resourcePath); + } catch (Exception e){ + throw new MLContextException("Error trying to read script from resource: "+ resourcePath, e); + } } /** @@ -425,8 +428,7 @@ public class ScriptFactory { if ((!urlString.toLowerCase().startsWith("http:")) && (!urlString.toLowerCase().startsWith("https:"))) { throw new MLContextException("Currently only reading from http and https URLs is supported"); } - try { - InputStream is = url.openStream(); + try( InputStream is = url.openStream() ) { return IOUtils.toString(is); } catch (IOException e) { throw new MLContextException("Error trying to read script string from URL: " + url, e); http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/parser/ParserWrapper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/parser/ParserWrapper.java b/src/main/java/org/apache/sysml/parser/ParserWrapper.java index 0f43971..8dc9712 100644 --- a/src/main/java/org/apache/sysml/parser/ParserWrapper.java +++ b/src/main/java/org/apache/sysml/parser/ParserWrapper.java @@ -124,24 +124,30 @@ public abstract class ParserWrapper { { String resPath = scriptPathToResourcePath(script); LOG.debug("Looking for the following resource from the SystemML jar file: " + resPath); - InputStream is = ParserWrapper.class.getResourceAsStream(resPath); - if (is == null) { - if (resPath.startsWith("/scripts")) { - LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "')"); - throw ex; - } else { - // for accessing script packages in the scripts directory - String scriptsResPath = "/scripts" + resPath; - LOG.debug("Looking for the following resource from the SystemML jar file: " + scriptsResPath); - is = ParserWrapper.class.getResourceAsStream(scriptsResPath); - if (is == null) { - LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "' or '" + scriptsResPath + "')"); + InputStream is = null; + try { + is = ParserWrapper.class.getResourceAsStream(resPath); + if (is == null) { + if (resPath.startsWith("/scripts")) { + LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "')"); throw ex; + } else { + // for accessing script packages in the scripts directory + String scriptsResPath = "/scripts" + resPath; + LOG.debug("Looking for the following resource from the SystemML jar file: " + scriptsResPath); + is = ParserWrapper.class.getResourceAsStream(scriptsResPath); + if (is == null) { + LOG.error("Failed to read from the file system ('" + script + "') or SystemML jar file ('" + resPath + "' or '" + scriptsResPath + "')"); + throw ex; + } } } + String s = IOUtils.toString(is); + return s; + } + finally { + IOUtilFunctions.closeSilently(is); } - String s = IOUtils.toString(is); - return s; } finally { IOUtilFunctions.closeSilently(in); http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java index 743c7e6..9a0b4c5 100644 --- a/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java +++ b/src/main/java/org/apache/sysml/runtime/instructions/gpu/context/JCudaKernels.java @@ -125,10 +125,8 @@ public class JCudaKernels { * @return */ private static Pointer initKernels(String ptxFileName) { - InputStream in = null; ByteArrayOutputStream out = null; - try { - in = JCudaKernels.class.getResourceAsStream(ptxFileName); + try( InputStream in = JCudaKernels.class.getResourceAsStream(ptxFileName) ) { if (in != null) { out = new ByteArrayOutputStream(); byte buffer[] = new byte[8192]; @@ -149,7 +147,6 @@ public class JCudaKernels { throw new DMLRuntimeException("Could not initialize the kernels", e); } finally { IOUtilFunctions.closeSilently(out); - IOUtilFunctions.closeSilently(in); } } } http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java index 9242d4a..2d89502 100644 --- a/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java +++ b/src/main/java/org/apache/sysml/runtime/transform/meta/TfMetaUtils.java @@ -216,7 +216,7 @@ public class TfMetaUtils throws IOException { //read column names - String colnamesStr = IOUtilFunctions.toString(Connection.class.getResourceAsStream(metapath+"/"+TfUtils.TXMTD_COLNAMES)); + String colnamesStr = getStringFromResource(metapath+"/"+TfUtils.TXMTD_COLNAMES); String[] colnames = IOUtilFunctions.split(colnamesStr.trim(), colDelim); //read meta data (currently supported: recode, dummycode, bin, omit) @@ -228,22 +228,22 @@ public class TfMetaUtils String colName = colnames[j]; //read recode maps for recoded or dummycoded columns String name = metapath+"/"+"Recode"+"/"+colName; - String map = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_MAP_SUFFIX)); + String map = getStringFromResource(name+TfUtils.TXMTD_RCD_MAP_SUFFIX); if( map != null ) { meta.put(colName, map); - String ndistinct = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name+TfUtils.TXMTD_RCD_DISTINCT_SUFFIX)); + String ndistinct = getStringFromResource(name+TfUtils.TXMTD_RCD_DISTINCT_SUFFIX); rows = Math.max(rows, Integer.parseInt(ndistinct)); } //read binning map for binned columns String name2 = metapath+"/"+"Bin"+"/"+colName; - String map2 = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name2+TfUtils.TXMTD_BIN_FILE_SUFFIX)); + String map2 = getStringFromResource(name2+TfUtils.TXMTD_BIN_FILE_SUFFIX); if( map2 != null ) { meta.put(colName, map2); rows = Math.max(rows, Integer.parseInt(map2.split(TfUtils.TXMTD_SEP)[4])); } //read impute value for mv columns String name3 = metapath+File.separator+"Impute"+File.separator+colName; - String map3 = IOUtilFunctions.toString(Connection.class.getResourceAsStream(name3+TfUtils.TXMTD_MV_FILE_SUFFIX)); + String map3 = getStringFromResource(name3+TfUtils.TXMTD_MV_FILE_SUFFIX); if( map3 != null ) { mvmeta.put(colName, map3); } @@ -389,4 +389,10 @@ public class TfMetaUtils throw new IOException(ex); } } + + private static String getStringFromResource(String path) throws IOException { + try(InputStream is = Connection.class.getResourceAsStream(path) ) { + return IOUtilFunctions.toString(is); + } + } } http://git-wip-us.apache.org/repos/asf/systemml/blob/607a402b/src/main/java/org/apache/sysml/utils/NativeHelper.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/utils/NativeHelper.java b/src/main/java/org/apache/sysml/utils/NativeHelper.java index 246c5bd..86d849f 100644 --- a/src/main/java/org/apache/sysml/utils/NativeHelper.java +++ b/src/main/java/org/apache/sysml/utils/NativeHelper.java @@ -37,6 +37,7 @@ import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.hops.OptimizerUtils; import org.apache.sysml.runtime.DMLRuntimeException; +import org.apache.sysml.runtime.io.IOUtilFunctions; /** * This class helps in loading native library. @@ -291,33 +292,25 @@ public class NativeHelper { private static boolean loadLibraryHelper(String path) { - InputStream in = null; OutputStream out = null; - try { - // This logic is added because Java doesnot allow to load library from a resource file. - in = NativeHelper.class.getResourceAsStream("/lib/"+path); + OutputStream out = null; + try( InputStream in = NativeHelper.class.getResourceAsStream("/lib/"+path) ) { + // This logic is added because Java does not allow to load library from a resource file. if(in != null) { File temp = File.createTempFile(path, ""); temp.deleteOnExit(); out = FileUtils.openOutputStream(temp); IOUtils.copy(in, out); - in.close(); in = null; - out.close(); out = null; System.load(temp.getAbsolutePath()); return true; } else LOG.warn("No lib available in the jar:" + path); - } catch(IOException e) { + } + catch(IOException e) { LOG.warn("Unable to load library " + path + " from resource:" + e.getMessage()); - } finally { - if(out != null) - try { - out.close(); - } catch (IOException e) {} - if(in != null) - try { - in.close(); - } catch (IOException e) {} + } + finally { + IOUtilFunctions.closeSilently(out); } return false; }
