Repository: systemml Updated Branches: refs/heads/master 8c11e8c07 -> 2c9694dec
[SYSTEMML-1823] Fix parallel csv frame reader w/ unknowns (shutdown) This patch fixes an issue of missing process exit after successful script execution. The root cause was a missing thread pool shutdown in the parallel csv frame reader w/ unknown input sizes. Project: http://git-wip-us.apache.org/repos/asf/systemml/repo Commit: http://git-wip-us.apache.org/repos/asf/systemml/commit/61a0931d Tree: http://git-wip-us.apache.org/repos/asf/systemml/tree/61a0931d Diff: http://git-wip-us.apache.org/repos/asf/systemml/diff/61a0931d Branch: refs/heads/master Commit: 61a0931d8ea5174ba19e3ed107925c5d42e07fb5 Parents: 8c11e8c Author: Matthias Boehm <[email protected]> Authored: Tue Aug 1 14:10:53 2017 -0700 Committer: Matthias Boehm <[email protected]> Committed: Tue Aug 1 19:30:55 2017 -0700 ---------------------------------------------------------------------- src/main/java/org/apache/sysml/conf/DMLConfig.java | 1 - .../apache/sysml/runtime/io/FrameReaderTextCSVParallel.java | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/systemml/blob/61a0931d/src/main/java/org/apache/sysml/conf/DMLConfig.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/conf/DMLConfig.java b/src/main/java/org/apache/sysml/conf/DMLConfig.java index 37db080..563573c 100644 --- a/src/main/java/org/apache/sysml/conf/DMLConfig.java +++ b/src/main/java/org/apache/sysml/conf/DMLConfig.java @@ -22,7 +22,6 @@ package org.apache.sysml.conf; import java.io.ByteArrayInputStream; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.InputStream; import java.io.StringWriter; import java.util.HashMap; import java.util.Map; http://git-wip-us.apache.org/repos/asf/systemml/blob/61a0931d/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java ---------------------------------------------------------------------- diff --git a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java index 64a5d73..0f8cb3a 100644 --- a/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java +++ b/src/main/java/org/apache/sysml/runtime/io/FrameReaderTextCSVParallel.java @@ -67,7 +67,8 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV try { - ExecutorService pool = Executors.newFixedThreadPool(numThreads); + ExecutorService pool = Executors.newFixedThreadPool( + Math.min(numThreads, splits.length)); //compute num rows per split ArrayList<CountRowsTask> tasks = new ArrayList<CountRowsTask>(); @@ -126,6 +127,9 @@ public class FrameReaderTextCSVParallel extends FrameReaderTextCSV catch (Exception e) { throw new IOException("Failed parallel read of text csv input.", e); } + finally { + pool.shutdown(); + } return new Pair<Integer,Integer>(nrow, ncol); }
