Anh Nguyen created SPARK-16762:
----------------------------------
Summary: spark hanging when action method print
Key: SPARK-16762
URL: https://issues.apache.org/jira/browse/SPARK-16762
Project: Spark
Issue Type: Bug
Components: Deploy
Reporter: Anh Nguyen
I write code spark Streaming (consumer) intergate kafaka and deploy on mesos FW:
import org.apache.spark.SparkConf
import org.apache.spark.streaming._
import org.apache.spark.streaming.kafka._
import org.apache.spark.streaming.kafka._
object consumer {
def main(args: Array[String]) {
if (args.length < 4) {
System.err.println("Usage: KafkaWordCount <zkQuorum><group> <topics>
<numThreads>")
System.exit(1)
}
val Array(zkQuorum, group, topics, numThreads) = args
val sparkConf = new
SparkConf().setAppName("KafkaWordCount").set("spark.rpc.netty.dispatcher.numThreads","4")
val ssc = new StreamingContext(sparkConf, Seconds(2))
ssc.checkpoint("checkpoint")
val topicMap = topics.split(",").map((_, numThreads.toInt)).toMap
val lines = KafkaUtils.createStream(ssc, zkQuorum, group,
topicMap).map(_._2)
lines.print()
val words = lines.flatMap(_.split(" "))
val wordCounts = words.map(x => (x, 1L)).reduceByKeyAndWindow(_ + _, _ -
_, Minutes(10), Seconds(2), 2)
ssc.start()
ssc.awaitTermination()
}
}
when deploy, spark hanging at method action lines.print()
Please help me, explain this issue
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
