[
https://issues.apache.org/jira/browse/SPARK-4091?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Cheng Lian updated SPARK-4091:
------------------------------
Description:
By persisting an arbitrary RDD with storage level {{MEMORY_AND_DISK}}, Spark
may occasionally throw the following exception when shutting down:
{code}
java.io.IOException: Failed to list files for dir:
/var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027005012-5bcd/0b
at org.apache.spark.util.Utils$.listFilesSafely(Utils.scala:664)
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:680)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:678)
at
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:177)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:175)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply$mcV$sp(Utils.scala:175)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:173)
{code}
By adding log output to {{Utils.deleteRecursively}}, setting breakpoints at
{{File.delete}} in IntelliJ, and asking IntelliJ to evaluate and log
{{Thread.currentThread().getStackTrace()}} when the breakpoint is hit rather
than suspend execution, we can get the following result, which shows
{{spark.local.dir}} is deleted twice from both {{DiskBlockManager.stop}} and
the shutdown hook installed in {{Utils}}:
{code}
+++ Deleting file:
/var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027003412-7fae/1d
Breakpoint reached at java.io.File.delete(File.java:1028)
[java.lang.Thread.getStackTrace(Thread.java:1589)
java.io.File.delete(File.java:1028)
org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:695)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:177)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:175)
scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply$mcV$sp(Utils.scala:175)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
org.apache.spark.util.Utils$$anon$4.run(Utils.scala:173)]
+++ Deleting file:
/var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027003412-7fae/1d
Breakpoint reached at java.io.File.delete(File.java:1028)
[java.lang.Thread.getStackTrace(Thread.java:1589)
java.io.File.delete(File.java:1028)
org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:695)
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:680)
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:678)
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
org.apache.spark.storage.DiskBlockManager$$anonfun$stop$1.apply(DiskBlockManager.scala:157)
org.apache.spark.storage.DiskBlockManager$$anonfun$stop$1.apply(DiskBlockManager.scala:154)
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
org.apache.spark.storage.DiskBlockManager.stop(DiskBlockManager.scala:154)
org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply$mcV$sp(DiskBlockManager.scala:147)
org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply(DiskBlockManager.scala:145)
org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply(DiskBlockManager.scala:145)
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
org.apache.spark.storage.DiskBlockManager$$anon$1.run(DiskBlockManager.scala:145)]
{code}
When this bug happens during Jenkins build, it fails {{CliSuite}}.
was:
By persisting an arbitrary RDD with storage level {{MEMORY_AND_DISK}}, Spark
may occasionally throw the following exception when shutting down:
{code}
java.io.IOException: Failed to list files for dir:
/var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027005012-5bcd/0b
at org.apache.spark.util.Utils$.listFilesSafely(Utils.scala:664)
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:680)
at
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:678)
at
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:177)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:175)
at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply$mcV$sp(Utils.scala:175)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
at
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:173)
{code}
By adding log output to {{Utils.deleteRecursively}}, setting breakpoints at
{{File.delete}} in IntelliJ, and asking IntelliJ to evaluate and log
{{Thread.currentThread().getStackTrace()}} when the breakpoint is hit rather
than suspend execution, we can get the following result, which shows
{{spark.local.dir}} is deleted twice from both {{DiskBlockManager.stop}} and
the shutdown hook installed in {{Utils}}:
{code}
+++ Deleting file:
/var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027003412-7fae/1d
Breakpoint reached at java.io.File.delete(File.java:1028)
[java.lang.Thread.getStackTrace(Thread.java:1589)
java.io.File.delete(File.java:1028)
org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:695)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:177)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:175)
scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply$mcV$sp(Utils.scala:175)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
org.apache.spark.util.Utils$$anon$4.run(Utils.scala:173)]
+++ Deleting file:
/var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027003412-7fae/1d
Breakpoint reached at java.io.File.delete(File.java:1028)
[java.lang.Thread.getStackTrace(Thread.java:1589)
java.io.File.delete(File.java:1028)
org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:695)
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:680)
org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:678)
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
org.apache.spark.storage.DiskBlockManager$$anonfun$stop$1.apply(DiskBlockManager.scala:157)
org.apache.spark.storage.DiskBlockManager$$anonfun$stop$1.apply(DiskBlockManager.scala:154)
scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
org.apache.spark.storage.DiskBlockManager.stop(DiskBlockManager.scala:154)
org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply$mcV$sp(DiskBlockManager.scala:147)
org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply(DiskBlockManager.scala:145)
org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply(DiskBlockManager.scala:145)
org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
org.apache.spark.storage.DiskBlockManager$$anon$1.run(DiskBlockManager.scala:145)]
{code}
When this bug happens during Jenkins build, it fails {{CliSuite}}.
> Occasionally spark.local.dir can be deleted twice and causes test failure
> -------------------------------------------------------------------------
>
> Key: SPARK-4091
> URL: https://issues.apache.org/jira/browse/SPARK-4091
> Project: Spark
> Issue Type: Bug
> Components: Spark Core
> Affects Versions: 1.1.0
> Reporter: Cheng Lian
>
> By persisting an arbitrary RDD with storage level {{MEMORY_AND_DISK}}, Spark
> may occasionally throw the following exception when shutting down:
> {code}
> java.io.IOException: Failed to list files for dir:
> /var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027005012-5bcd/0b
> at org.apache.spark.util.Utils$.listFilesSafely(Utils.scala:664)
> at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
> at
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:680)
> at
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:678)
> at
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> at scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
> at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
> at
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:177)
> at
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:175)
> at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
> at
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply$mcV$sp(Utils.scala:175)
> at
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
> at
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
> at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
> at org.apache.spark.util.Utils$$anon$4.run(Utils.scala:173)
> {code}
> By adding log output to {{Utils.deleteRecursively}}, setting breakpoints at
> {{File.delete}} in IntelliJ, and asking IntelliJ to evaluate and log
> {{Thread.currentThread().getStackTrace()}} when the breakpoint is hit rather
> than suspend execution, we can get the following result, which shows
> {{spark.local.dir}} is deleted twice from both {{DiskBlockManager.stop}} and
> the shutdown hook installed in {{Utils}}:
> {code}
> +++ Deleting file:
> /var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027003412-7fae/1d
> Breakpoint reached at java.io.File.delete(File.java:1028)
> [java.lang.Thread.getStackTrace(Thread.java:1589)
> java.io.File.delete(File.java:1028)
> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:695)
>
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:177)
>
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1$$anonfun$apply$mcV$sp$2.apply(Utils.scala:175)
> scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
>
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply$mcV$sp(Utils.scala:175)
>
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
>
> org.apache.spark.util.Utils$$anon$4$$anonfun$run$1.apply(Utils.scala:173)
> org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
> org.apache.spark.util.Utils$$anon$4.run(Utils.scala:173)]
> +++ Deleting file:
> /var/folders/kh/r9ylmzln40n9nrlchnsry2qw0000gn/T/spark-local-20141027003412-7fae/1d
> Breakpoint reached at java.io.File.delete(File.java:1028)
> [java.lang.Thread.getStackTrace(Thread.java:1589)
> java.io.File.delete(File.java:1028)
> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:695)
>
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:680)
>
> org.apache.spark.util.Utils$$anonfun$deleteRecursively$1.apply(Utils.scala:678)
>
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> scala.collection.mutable.WrappedArray.foreach(WrappedArray.scala:34)
> org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:678)
>
> org.apache.spark.storage.DiskBlockManager$$anonfun$stop$1.apply(DiskBlockManager.scala:157)
>
> org.apache.spark.storage.DiskBlockManager$$anonfun$stop$1.apply(DiskBlockManager.scala:154)
>
> scala.collection.IndexedSeqOptimized$class.foreach(IndexedSeqOptimized.scala:33)
> scala.collection.mutable.ArrayOps$ofRef.foreach(ArrayOps.scala:108)
>
> org.apache.spark.storage.DiskBlockManager.stop(DiskBlockManager.scala:154)
>
> org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply$mcV$sp(DiskBlockManager.scala:147)
>
> org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply(DiskBlockManager.scala:145)
>
> org.apache.spark.storage.DiskBlockManager$$anon$1$$anonfun$run$1.apply(DiskBlockManager.scala:145)
> org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1323)
>
> org.apache.spark.storage.DiskBlockManager$$anon$1.run(DiskBlockManager.scala:145)]
> {code}
> When this bug happens during Jenkins build, it fails {{CliSuite}}.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
