I am trying to run this code on spark 0.9.0 and I am getting this error:
"java.lang.ClassNotFoundException: org.apache.spark.streaming.StreamingContext"
My CLASSPATH variable contains the following
path"....spark-0.9.0/assembly/target/scala-2.10/spark-assembly-0.9.0-incubating-hadoop1.0.4.jar"
what else do I need to add here ?
The code works fine in the spark-shell.
"import org.apache.spark.streaming.{Time, Seconds, StreamingContext}
import org.apache.spark.streaming.StreamingContext._
object HdfsWordCount {
def main(args: Array[String]) {
if (args.length < 2) {
System.err.println("Usage: HdfsWordCount <master> <directory>")
System.exit(1)
}
// StreamingExamples.setStreamingLogLevels()
// Create the context
// val ssc = new StreamingContext(args(0), "HdfsWordCount", Seconds(2),
System.getenv("SPARK_HOME"), Seq(System.getenv("SPARK_HOME")))
val ssc = new StreamingContext(args(0), "HdfsWordCount", Seconds(2))
// Create the FileInputDStream on the directory and use the
// stream to count words in new files created
val lines = ssc.textFileStream(args(1))
val words = lines.flatMap(_.split(" "))
val wordCounts = words.map(x => (x, 1)).reduceByKey(_ + _)
wordCounts.print()
ssc.start()
ssc.awaitTermination()
}
}"
Thanks
