Repository: spark Updated Branches: refs/heads/master 066f31a6b -> 81b9d5b62
SPARK-3052. Misleading and spurious FileSystem closed errors whenever a ... ...job fails while reading from Hadoop Author: Sandy Ryza <sa...@cloudera.com> Closes #1956 from sryza/sandy-spark-3052 and squashes the following commits: 815813a [Sandy Ryza] SPARK-3052. Misleading and spurious FileSystem closed errors whenever a job fails while reading from Hadoop Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/81b9d5b6 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/81b9d5b6 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/81b9d5b6 Branch: refs/heads/master Commit: 81b9d5b628229ed69aa9dae45ec4c94068dcd71e Parents: 066f31a Author: Sandy Ryza <sa...@cloudera.com> Authored: Tue Sep 2 11:34:55 2014 -0700 Committer: Matei Zaharia <ma...@databricks.com> Committed: Tue Sep 2 11:34:55 2014 -0700 ---------------------------------------------------------------------- core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala | 9 +++++++-- core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/81b9d5b6/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala index c862331..036dcc4 100644 --- a/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/HadoopRDD.scala @@ -42,7 +42,8 @@ import org.apache.spark.broadcast.Broadcast import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.executor.{DataReadMethod, InputMetrics} import org.apache.spark.rdd.HadoopRDD.HadoopMapPartitionsWithSplitRDD -import org.apache.spark.util.NextIterator +import org.apache.spark.util.{NextIterator, Utils} + /** * A Spark split class that wraps around a Hadoop InputSplit. @@ -228,7 +229,11 @@ class HadoopRDD[K, V]( try { reader.close() } catch { - case e: Exception => logWarning("Exception in RecordReader.close()", e) + case e: Exception => { + if (!Utils.inShutdown()) { + logWarning("Exception in RecordReader.close()", e) + } + } } } } http://git-wip-us.apache.org/repos/asf/spark/blob/81b9d5b6/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala index 58f707b..4c84b3f 100644 --- a/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala +++ b/core/src/main/scala/org/apache/spark/rdd/NewHadoopRDD.scala @@ -35,6 +35,7 @@ import org.apache.spark.SerializableWritable import org.apache.spark.{SparkContext, TaskContext} import org.apache.spark.executor.{DataReadMethod, InputMetrics} import org.apache.spark.rdd.NewHadoopRDD.NewHadoopMapPartitionsWithSplitRDD +import org.apache.spark.util.Utils private[spark] class NewHadoopPartition( rddId: Int, @@ -153,7 +154,11 @@ class NewHadoopRDD[K, V]( try { reader.close() } catch { - case e: Exception => logWarning("Exception in RecordReader.close()", e) + case e: Exception => { + if (!Utils.inShutdown()) { + logWarning("Exception in RecordReader.close()", e) + } + } } } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org