[SYSTEMML-653] Performance bufferpool eviction serialized blocks (nio) This patch changes the local file system primitive of writing serialized byte arrays (as used for bufferpool evictions of serialized sparse and ultra-sparse blocks) from java.io to java.nio. The performance gain for both small and medium matrices was roughly 10-15%. Furthermore, this also includes a cleanup of IO close utilities by generalizing them to any closeable implementation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-systemml/commit/3de43d62 Tree: http://git-wip-us.apache.org/repos/asf/incubator-systemml/tree/3de43d62 Diff: http://git-wip-us.apache.org/repos/asf/incubator-systemml/diff/3de43d62 Branch: refs/heads/master Commit: 3de43d62fdd40436308ea070f83817beafd43a6b Parents: a054639 Author: Matthias Boehm <[email protected]> Authored: Fri Apr 29 00:25:49 2016 -0700 Committer: Matthias Boehm <[email protected]> Committed: Fri Apr 29 11:57:23 2016 -0700 ---------------------------------------------------------------------- .../org/apache/sysml/api/jmlc/Connection.java | 4 +- .../sysml/runtime/io/IOUtilFunctions.java | 93 ++------------------ .../sysml/runtime/util/LocalFileUtils.java | 39 +++----- 3 files changed, 24 insertions(+), 112 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3de43d62/src/main/java/org/apache/sysml/api/jmlc/Connection.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/api/jmlc/Connection.java b/src/main/java/org/apache/sysml/api/jmlc/Connection.java index 54f7e44..50e1aa0 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/Connection.java +++ b/src/main/java/org/apache/sysml/api/jmlc/Connection.java @@ -20,6 +20,7 @@ package org.apache.sysml.api.jmlc; import java.io.BufferedReader; +import java.io.Closeable; import java.io.FileReader; import java.io.IOException; import java.io.InputStream; @@ -88,7 +89,7 @@ import org.apache.wink.json4j.JSONObject; * of SystemML online documentation</li> * </ul> */ -public class Connection +public class Connection implements Closeable { private DMLConfig _dmlconf = null; @@ -201,6 +202,7 @@ public class Connection * Close connection to SystemML, which clears the * thread-local DML and compiler configurations. */ + @Override public void close() { //clear thread-local dml / compiler configs ConfigurationManager.clearLocalConfigs(); http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3de43d62/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java index 3b618d0..69efec0 100644 --- a/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java +++ b/src/main/java/org/apache/sysml/runtime/io/IOUtilFunctions.java @@ -19,18 +19,15 @@ package org.apache.sysml.runtime.io; -import java.io.BufferedReader; -import java.io.BufferedWriter; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.Closeable; import java.io.IOException; import java.io.InputStream; -import java.io.OutputStream; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.mapred.RecordReader; import org.apache.sysml.runtime.util.LocalFileUtils; import org.apache.sysml.runtime.util.UtilFunctions; @@ -39,98 +36,24 @@ public class IOUtilFunctions { private static final Log LOG = LogFactory.getLog(UtilFunctions.class.getName()); - - - /** - * - * @param is - */ - public static void closeSilently( InputStream is ) - { - try { - if( is != null ) - is.close(); - } - catch (Exception ex) { - LOG.error("Failed to close input stream.", ex); - } - } - - /** - * - * @param is - */ - public static void closeSilently( OutputStream os ) - { - try { - if( os != null ) - os.close(); - } - catch (Exception ex) { - LOG.error("Failed to close output stream.", ex); - } - } - - /** - * - * @param br - */ - public static void closeSilently( BufferedReader br ) - { - try { - if( br != null ) - br.close(); - } - catch (Exception ex) { - LOG.error("Failed to close buffered reader.", ex); - } - } - - /** - * - * @param br - */ - public static void closeSilently( BufferedWriter bw ) - { - try { - if( bw != null ) - bw.close(); - } - catch (Exception ex) { - LOG.error("Failed to buffered writer.", ex); - } - } - /** * - * @param br + * @param io */ - public static void closeSilently( SequenceFile.Reader br ) - { + public static void closeSilently( Closeable io ) { try { - if( br != null ) - br.close(); + if( io != null ) + io.close(); } catch (Exception ex) { - LOG.error("Failed to close reader.", ex); + LOG.error("Failed to close IO resource.", ex); } } - + /** * - * @param br + * @param rr */ - public static void closeSilently( SequenceFile.Writer bw ) - { - try { - if( bw != null ) - bw.close(); - } - catch (Exception ex) { - LOG.error("Failed to writer.", ex); - } - } - public static void closeSilently( RecordReader<?,?> rr ) { try { http://git-wip-us.apache.org/repos/asf/incubator-systemml/blob/3de43d62/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java index 7825013..c121844 100644 --- a/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java +++ b/src/main/java/org/apache/sysml/runtime/util/LocalFileUtils.java @@ -23,6 +23,11 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.util.HashMap; import org.apache.hadoop.io.Writable; @@ -154,35 +159,17 @@ public class LocalFileUtils */ public static void writeByteArrayToLocal( String filePathAndName, byte[] data ) throws IOException - { - FileOutputStream fos = new FileOutputStream( filePathAndName ); - - try { - fos.write( data ); - } - finally { - IOUtilFunctions.closeSilently(fos); - } - } - - /** - * - * @param filePathAndName - * @param data - * @throws IOException - */ - public static void writeByteArrayToLocal( String filePathAndName, byte[][] data ) - throws IOException - { - FileOutputStream fos = new FileOutputStream( filePathAndName ); - + { + //byte array write via java.nio file channel ~10-15% faster than java.io + FileChannel channel = null; try { - for( int i=0; i<data.length; i++ ) - if( data[i]!=null ) - fos.write( data[i] ); + Path path = Paths.get(filePathAndName); + channel = FileChannel.open(path, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + channel.write(ByteBuffer.wrap(data)); } finally { - IOUtilFunctions.closeSilently(fos); + IOUtilFunctions.closeSilently(channel); } }
