Updated Branches: refs/heads/master 2fead510f -> f20093c3a
TimeTrackingOutputStream should pass on calls to close() and flush(). Without this fix you get a huge number of open shuffles after running shuffles. Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/53b94ef2 Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/53b94ef2 Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/53b94ef2 Branch: refs/heads/master Commit: 53b94ef2f5179bdbebe70883b2593b569518e77e Parents: 4ba3267 Author: Patrick Wendell <[email protected]> Authored: Thu Nov 21 17:17:06 2013 -0800 Committer: Patrick Wendell <[email protected]> Committed: Thu Nov 21 17:20:15 2013 -0800 ---------------------------------------------------------------------- .../main/scala/org/apache/spark/storage/BlockObjectWriter.scala | 2 ++ 1 file changed, 2 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/53b94ef2/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala b/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala index 32d2dd0..0a32df7 100644 --- a/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala +++ b/core/src/main/scala/org/apache/spark/storage/BlockObjectWriter.scala @@ -101,6 +101,8 @@ class DiskBlockObjectWriter( def write(i: Int): Unit = callWithTiming(out.write(i)) override def write(b: Array[Byte]) = callWithTiming(out.write(b)) override def write(b: Array[Byte], off: Int, len: Int) = callWithTiming(out.write(b, off, len)) + override def close() = out.close() + override def flush() = out.flush() } private val syncWrites = System.getProperty("spark.shuffle.sync", "false").toBoolean
