.
The solution for which did not work for me.
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/Using-data-in-RDD-to-specify-HDFS-directory-to-write-to-tp18789p20526.html
Sent from the Apache Spark User List mailing list archive at Nabbl
ew this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/Using-data-in-RDD-to-specify-HDFS-directory-to-write-to-tp18789p19114.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
--
; val eventStream = pipe.map(data => {
> parse(data)
> }).map(json => {
>
>
> implicit val formats = DefaultFormats
> val eventName = (json \ "event").extractOpt[String]
> Event(eventName.getOrElse("*** NO EVENT NAME ***"), js
toArray
x.foreachPartition(y => {
y.foreach(z => {print(z)})
})
})
ssc.start()
ssc.awaitTermination()
}
}
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/Using-data-in-RDD-to-specify-HDFS
(event._2)
eventRDD.saveAsTextFile("hdfs://cloudera01.local.company.com:8020/user/hdfs/"
+ event._1 + "/rdd=" + eventRDD.id + "_ID_" + uuid)
})
})
})
ssc.start()
ssc.awaitTermination()
}
}
--
View this message in context:
http://apache-spark
rElse("*** NO EVENT NAME ***"), json)
>
> })
>
> eventStream.foreachRDD(event => {
> //val eventName = event.EventName //CAN'T ACCESS eventName!
> event.saveAsTextFile("hdfs://ip-here/user/hdfs/" + eventName +
> "/rdd="
saveAsTextFile("hdfs://ip-here/user/hdfs/" + eventName + "/rdd="
+ pageHit.id) //what I would like to do if I could access eventName
})
ssc.start()
ssc.awaitTermination()
}
}
--
View this message in context:
http://apache-spark-user-list.10015
