Dear folks,
Is Spark an ideal workload on zlinux and is it supported on zlinux? We tried
to complie spart on SUSE linux on z and encouter the following problem.
15/03/10 09:30:07 INFO DAGScheduler: Job 0 failed: reduce at
JavaSparkPi.java:48, took 0.087641 s
Exception in thread "main" org.apache.spark.SparkException: Job aborted due to
stage failure: Task serialization failed: org.xerial.snappy.SnappyError:
[FAILED_TO_LOAD_NATIVE_LIBRARY] no native library is found for os.name=Linux
and os.arch=s390x
org.xerial.snappy.SnappyLoader.findNativeLibrary(SnappyLoader.java:299)
org.xerial.snappy.SnappyLoader.loadNativeLibrary(SnappyLoader.java:163)
org.xerial.snappy.SnappyLoader.load(SnappyLoader.java:145)
org.xerial.snappy.Snappy.<clinit>(Snappy.java:47)
java.lang.J9VMInternals.initializeImpl(Native Method)
java.lang.J9VMInternals.initialize(J9VMInternals.java:200)
org.xerial.snappy.SnappyOutputStream.<init>(SnappyOutputStream.java:90)
org.xerial.snappy.SnappyOutputStream.<init>(SnappyOutputStream.java:83)
org.apache.spark.io.SnappyCompressionCodec.compressedOutputStream(CompressionCodec.scala:125)
org.apache.spark.broadcast.TorrentBroadcast$$anonfun$4.apply(TorrentBroadcast.scala:199)
org.apache.spark.broadcast.TorrentBroadcast$$anonfun$4.apply(TorrentBroadcast.scala:199)
scala.Option.map(Option.scala:145)
org.apache.spark.broadcast.TorrentBroadcast$.blockifyObject(TorrentBroadcast.scala:199)
org.apache.spark.broadcast.TorrentBroadcast.writeBlocks(TorrentBroadcast.scala:101)
org.apache.spark.broadcast.TorrentBroadcast.<init>(TorrentBroadcast.scala:84)
org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:34)
org.apache.spark.broadcast.TorrentBroadcastFactory.newBroadcast(TorrentBroadcastFactory.scala:29)
org.apache.spark.broadcast.BroadcastManager.newBroadcast(BroadcastManager.scala:62)
org.apache.spark.SparkContext.broadcast(SparkContext.scala:980)
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$submitMissingTasks(DAGScheduler.scala:838)
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$submitStage(DAGScheduler.scala:778)
org.apache.spark.scheduler.DAGScheduler.handleJobSubmitted(DAGScheduler.scala:762)
org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1389)
akka.actor.Actor$class.aroundReceive(Actor.scala:465)
org.apache.spark.scheduler.DAGSchedulerEventProcessActor.aroundReceive(DAGScheduler.scala:1375)
akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
akka.actor.ActorCell.invoke(ActorCell.scala:487)
akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
akka.dispatch.Mailbox.run(Mailbox.scala:220)
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1214)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1203)
at
org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1202)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
at
org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1202)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$submitMissingTasks(DAGScheduler.scala:846)
at
org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$submitStage(DAGScheduler.scala:778)
at
org.apache.spark.scheduler.DAGScheduler.handleJobSubmitted(DAGScheduler.scala:762)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessActor$$anonfun$receive$2.applyOrElse(DAGScheduler.scala:1389)
at akka.actor.Actor$class.aroundReceive(Actor.scala:465)
at
org.apache.spark.scheduler.DAGSchedulerEventProcessActor.aroundReceive(DAGScheduler.scala:1375)
at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516)
at akka.actor.ActorCell.invoke(ActorCell.scala:487)
at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238)
at akka.dispatch.Mailbox.run(Mailbox.scala:220)
at
akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393)
at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
at
scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
at
scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
at
scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
When tried compliing snapy-java, the following error showed,
g++ -I/usr/lib64/jvm/jre/include -Ilib/inc_mac -O2 -fPIC -fvisibility=hidden
-Ilib/include -Itarget/snappy-1.1.1 -o
target/snappy-1.1.1-Default/libsnappyjava.so
target/snappy-1.1.1-Default/snappy-sinksource.o
target/snappy-1.1.1-Default/snappy-stubs-internal.o
target/snappy-1.1.1-Default/snappy.o target/snappy-1.1.1-Default/SnappyNative.o
-shared -static
/usr/lib64/gcc/s390x-suse-linux/4.3/../../../../s390x-suse-linux/bin/ld:
/usr/lib64/gcc/s390x-suse-linux/4.3/crtbeginT.o: relocation R_390_PC32DBL
against `_Jv_RegisterClasses' can not be used when making a shared object;
recompile with -fPIC
/usr/lib64/gcc/s390x-suse-linux/4.3/../../../../s390x-suse-linux/bin/ld: final
link failed: Bad value
collect2: ld returned 1 exit status
make: *** [target/snappy-1.1.1-Default/libsnappyjava.so] Error 1
Appreciate your help. Thanks.
Brant Zhang
