[
https://issues.apache.org/jira/browse/SPARK-8333?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14652531#comment-14652531
]
Sudhakar Thota commented on SPARK-8333:
---------------------------------------
Hi Sheng,
I have verified in spark application, it works for me. I have opened a
SparkContext then SQLContext then issued sc.stop(). To test if it is still
existing, I tried to SparkContext to open a file which should throw error and
it did. It is working on my mac.
------------------------------------------------
Sudhakars-MacBook-Pro-2:test sudhakarthota$ cat src/main/scala/SimpleApp.scala
/* SimpleApp.scala */
import org.apache.spark.SparkContext
import org.apache.spark.SparkContext._
import org.apache.spark.SparkConf
//import org.apache.spark.sql.hive.SQLContext
//import sqlContext.implicits._
object SimpleApp {
def main(args: Array[String]) {
val conf = new SparkConf().setAppName("Simple Application")
val sc = new SparkContext(conf)
println("************************************************************")
println("We are in SPARK CONTEXT")
println("************************************************************")
val sqlContext = new org.apache.spark.sql.SQLContext(sc)
println("************************************************************")
println("We are in HIVE CONTEXT")
println("************************************************************")
sc.stop()
println("************************************************************")
println("SparkContext stopped using sc.stop()")
println("************************************************************")
println("************************************************************")
println("Trying to reuse SparkContext that was stopped ")
println("************************************************************")
sc.textFile("/Users/sudhakarthota/spark/README.md")
}
}
Sudhakars-MacBook-Pro-2:spark-1.4.0 sudhakarthota$ bin/spark-submit --class
"SimpleApp" --master local[4]
/Users/sudhakarthota/test/target/scala-2.10/simple-project_2.10-1.0.jar
************************************************************
We are in SPARK CONTEXT
************************************************************
************************************************************
We are in HIVE CONTEXT
************************************************************
************************************************************
SparkContext stopped using sc.stop()
************************************************************
************************************************************
Trying to reuse SparkContext that was stopped
************************************************************
Exception in thread "main" java.lang.IllegalStateException: Cannot call methods
on a stopped SparkContext
at
org.apache.spark.SparkContext.org$apache$spark$SparkContext$$assertNotStopped(SparkContext.scala:103)
at
org.apache.spark.SparkContext.defaultParallelism(SparkContext.scala:1912)
at
org.apache.spark.SparkContext.defaultMinPartitions(SparkContext.scala:1925)
at
org.apache.spark.SparkContext.textFile$default$2(SparkContext.scala:797)
at SimpleApp$.main(SimpleApp.scala:26)
at SimpleApp.main(SimpleApp.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:664)
at
org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:169)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:192)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:111)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
Sudhakars-MacBook-Pro-2:spark-1.4.0 sudhakarthota$
Thanks
Sudhakar Thota
> Spark failed to delete temp directory created by HiveContext
> ------------------------------------------------------------
>
> Key: SPARK-8333
> URL: https://issues.apache.org/jira/browse/SPARK-8333
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 1.4.0
> Environment: Windows7 64bit
> Reporter: sheng
> Priority: Minor
> Labels: Hive, metastore, sparksql
>
> Spark 1.4.0 failed to stop SparkContext.
> {code:title=LocalHiveTest.scala|borderStyle=solid}
> val sc = new SparkContext("local", "local-hive-test", new SparkConf())
> val hc = Utils.createHiveContext(sc)
> ... // execute some HiveQL statements
> sc.stop()
> {code}
> sc.stop() failed to execute, it threw the following exception:
> {quote}
> 15/06/13 03:19:06 INFO Utils: Shutdown hook called
> 15/06/13 03:19:06 INFO Utils: Deleting directory
> C:\Users\moshangcheng\AppData\Local\Temp\spark-d6d3c30e-512e-4693-a436-485e2af4baea
> 15/06/13 03:19:06 ERROR Utils: Exception while deleting Spark temp dir:
> C:\Users\moshangcheng\AppData\Local\Temp\spark-d6d3c30e-512e-4693-a436-485e2af4baea
> java.io.IOException: Failed to delete:
> C:\Users\moshangcheng\AppData\Local\Temp\spark-d6d3c30e-512e-4693-a436-485e2af4baea
> at org.apache.spark.util.Utils$.deleteRecursively(Utils.scala:963)
> at
> org.apache.spark.util.Utils$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(Utils.scala:204)
> at
> org.apache.spark.util.Utils$$anonfun$1$$anonfun$apply$mcV$sp$5.apply(Utils.scala:201)
> at scala.collection.mutable.HashSet.foreach(HashSet.scala:79)
> at org.apache.spark.util.Utils$$anonfun$1.apply$mcV$sp(Utils.scala:201)
> at org.apache.spark.util.SparkShutdownHook.run(Utils.scala:2292)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply$mcV$sp(Utils.scala:2262)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(Utils.scala:2262)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1$$anonfun$apply$mcV$sp$1.apply(Utils.scala:2262)
> at org.apache.spark.util.Utils$.logUncaughtExceptions(Utils.scala:1772)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply$mcV$sp(Utils.scala:2262)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(Utils.scala:2262)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anonfun$runAll$1.apply(Utils.scala:2262)
> at scala.util.Try$.apply(Try.scala:161)
> at
> org.apache.spark.util.SparkShutdownHookManager.runAll(Utils.scala:2262)
> at
> org.apache.spark.util.SparkShutdownHookManager$$anon$6.run(Utils.scala:2244)
> at
> org.apache.hadoop.util.ShutdownHookManager$1.run(ShutdownHookManager.java:54)
> {quote}
> It seems this bug is introduced by this SPARK-6907. In SPARK-6907, a local
> hive metastore is created in a temp directory. The problem is the local hive
> metastore is not shut down correctly. At the end of application, if
> SparkContext.stop() is called, it tries to delete the temp directory which is
> still used by the local hive metastore, and throws an exception.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
