I solved it :) I moved the lookupObject into the function where I create the
broadcast and now all works very well!
object lookupObject
{
private var treeFile : org.apache.spark.broadcast.Broadcast[String] = _
def main(args: Array[String]): Unit = {
…
val treeFile = sc.broadcast(args(0))
object treeContainer
{
val tree : S2Lookup = loadTree
def dolookup(id : Long) : Boolean =
{
return tree.lookupSimple(new S2CellId(id))
}
def loadTree() : S2Lookup =
{
val path = new Path(treeFile.value); // treeFile is everytime
null
val fileSystem = FileSystem.get(new Configuration())
new S2Lookup(ConversionUtils.deserializeCovering(new
InputStreamReader(fileSystem.open(path))))
}
}
…
}
}
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/Setup-an-huge-Unserializable-Object-in-a-mapper-tp14817p14916.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
