[
https://issues.apache.org/jira/browse/SPARK-12413?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Michael Gummelt updated SPARK-12413:
------------------------------------
Description:
https://github.com/apache/spark/pull/10359 breaks ZK persistence due to
https://issues.scala-lang.org/browse/SI-6654
This line throws a NotSerializable exception:
https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/scheduler/cluster
The MesosClusterDispatcher attempts to serialize MesosDriverDescription objects
to ZK, but https://github.com/apache/spark/pull/10359 makes it so the
{{command}} property is unserializable
Offer id: 72f4d1ce-67f7-41b0-95a3-aa6fb208df32-O189, cpu: 3.0, mem: 12995.0
15/12/17 21:52:44 DEBUG ClientCnxn: Got ping response for sessionid:
0x151b1d1567e0002 after 0ms
15/12/17 21:52:44 DEBUG nio: created
SCEP@2e746d70{l(/10.0.6.166:41456)<->r(/10.0.0.240:17386),s=0,open=true,ishut=false,oshut=false,rb=false,wb=false,w=true,i=0}-{AsyncHttpConnection@5dbcebe3,g=HttpGenerator{s=0,h=-1,b=-1,c=-1},p=HttpParser{s=-14,l=0,c=0},r=0}
15/12/17 21:52:44 DEBUG HttpParser: filled 1591/1591
15/12/17 21:52:44 DEBUG Server: REQUEST /v1/submissions/create on
AsyncHttpConnection@5dbcebe3,g=HttpGenerator{s=0,h=-1,b=-1,c=-1},p=HttpParser{s=2,l=2,c=1174},r=1
15/12/17 21:52:44 DEBUG ContextHandler: scope null||/v1/submissions/create @
o.s.j.s.ServletContextHandler{/,null}
15/12/17 21:52:44 DEBUG ContextHandler: context=||/v1/submissions/create @
o.s.j.s.ServletContextHandler{/,null}
15/12/17 21:52:44 DEBUG ServletHandler: servlet |/v1/submissions/create|null ->
org.apache.spark.deploy.rest.mesos.MesosSubmitRequestServlet-368e091
15/12/17 21:52:44 DEBUG ServletHandler: chain=null
15/12/17 21:52:44 WARN ServletHandler: /v1/submissions/create
java.io.NotSerializableException: scala.collection.immutable.MapLike$$anon$1
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1183)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1547)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1547)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347)
at org.apache.spark.util.Utils$.serialize(Utils.scala:83)
at
org.apache.spark.scheduler.cluster.mesos.ZookeeperMesosClusterPersistenceEngine.persist(MesosClusterPersistenceEngine.scala:110)
at
org.apache.spark.scheduler.cluster.mesos.MesosClusterScheduler.submitDriver(MesosClusterScheduler.scala:166)
at
org.apache.spark.deploy.rest.mesos.MesosSubmitRequestServlet.handleSubmit(MesosRestServer.scala:132)
at
org.apache.spark.deploy.rest.SubmitRequestServlet.doPost(RestSubmissionServer.scala:258)
was:
https://github.com/apache/spark/pull/10359 breaks ZK persistence due to
https://issues.scala-lang.org/browse/SI-6654
This line throws a NotSerializable exception:
https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/scheduler/cluster
Offer id: 72f4d1ce-67f7-41b0-95a3-aa6fb208df32-O189, cpu: 3.0, mem: 12995.0
15/12/17 21:52:44 DEBUG ClientCnxn: Got ping response for sessionid:
0x151b1d1567e0002 after 0ms
15/12/17 21:52:44 DEBUG nio: created
SCEP@2e746d70{l(/10.0.6.166:41456)<->r(/10.0.0.240:17386),s=0,open=true,ishut=false,oshut=false,rb=false,wb=false,w=true,i=0}-{AsyncHttpConnection@5dbcebe3,g=HttpGenerator{s=0,h=-1,b=-1,c=-1},p=HttpParser{s=-14,l=0,c=0},r=0}
15/12/17 21:52:44 DEBUG HttpParser: filled 1591/1591
15/12/17 21:52:44 DEBUG Server: REQUEST /v1/submissions/create on
AsyncHttpConnection@5dbcebe3,g=HttpGenerator{s=0,h=-1,b=-1,c=-1},p=HttpParser{s=2,l=2,c=1174},r=1
15/12/17 21:52:44 DEBUG ContextHandler: scope null||/v1/submissions/create @
o.s.j.s.ServletContextHandler{/,null}
15/12/17 21:52:44 DEBUG ContextHandler: context=||/v1/submissions/create @
o.s.j.s.ServletContextHandler{/,null}
15/12/17 21:52:44 DEBUG ServletHandler: servlet |/v1/submissions/create|null ->
org.apache.spark.deploy.rest.mesos.MesosSubmitRequestServlet-368e091
15/12/17 21:52:44 DEBUG ServletHandler: chain=null
15/12/17 21:52:44 WARN ServletHandler: /v1/submissions/create
java.io.NotSerializableException: scala.collection.immutable.MapLike$$anon$1
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1183)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1547)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at
java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1547)
at
java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
at
java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431)
at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347)
at org.apache.spark.util.Utils$.serialize(Utils.scala:83)
at
org.apache.spark.scheduler.cluster.mesos.ZookeeperMesosClusterPersistenceEngine.persist(MesosClusterPersistenceEngine.scala:110)
at
org.apache.spark.scheduler.cluster.mesos.MesosClusterScheduler.submitDriver(MesosClusterScheduler.scala:166)
at
org.apache.spark.deploy.rest.mesos.MesosSubmitRequestServlet.handleSubmit(MesosRestServer.scala:132)
at
org.apache.spark.deploy.rest.SubmitRequestServlet.doPost(RestSubmissionServer.scala:258)
> Mesos ZK persistence throws a NotSerializableException
> ------------------------------------------------------
>
> Key: SPARK-12413
> URL: https://issues.apache.org/jira/browse/SPARK-12413
> Project: Spark
> Issue Type: Bug
> Components: Mesos
> Affects Versions: 1.6.0
> Reporter: Michael Gummelt
>
> https://github.com/apache/spark/pull/10359 breaks ZK persistence due to
> https://issues.scala-lang.org/browse/SI-6654
> This line throws a NotSerializable exception:
> https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/scheduler/cluster
> The MesosClusterDispatcher attempts to serialize MesosDriverDescription
> objects to ZK, but https://github.com/apache/spark/pull/10359 makes it so the
> {{command}} property is unserializable
> Offer id: 72f4d1ce-67f7-41b0-95a3-aa6fb208df32-O189, cpu: 3.0, mem: 12995.0
> 15/12/17 21:52:44 DEBUG ClientCnxn: Got ping response for sessionid:
> 0x151b1d1567e0002 after 0ms
> 15/12/17 21:52:44 DEBUG nio: created
> SCEP@2e746d70{l(/10.0.6.166:41456)<->r(/10.0.0.240:17386),s=0,open=true,ishut=false,oshut=false,rb=false,wb=false,w=true,i=0}-{AsyncHttpConnection@5dbcebe3,g=HttpGenerator{s=0,h=-1,b=-1,c=-1},p=HttpParser{s=-14,l=0,c=0},r=0}
> 15/12/17 21:52:44 DEBUG HttpParser: filled 1591/1591
> 15/12/17 21:52:44 DEBUG Server: REQUEST /v1/submissions/create on
> AsyncHttpConnection@5dbcebe3,g=HttpGenerator{s=0,h=-1,b=-1,c=-1},p=HttpParser{s=2,l=2,c=1174},r=1
> 15/12/17 21:52:44 DEBUG ContextHandler: scope null||/v1/submissions/create @
> o.s.j.s.ServletContextHandler{/,null}
> 15/12/17 21:52:44 DEBUG ContextHandler: context=||/v1/submissions/create @
> o.s.j.s.ServletContextHandler{/,null}
> 15/12/17 21:52:44 DEBUG ServletHandler: servlet |/v1/submissions/create|null
> -> org.apache.spark.deploy.rest.mesos.MesosSubmitRequestServlet-368e091
> 15/12/17 21:52:44 DEBUG ServletHandler: chain=null
> 15/12/17 21:52:44 WARN ServletHandler: /v1/submissions/create
> java.io.NotSerializableException: scala.collection.immutable.MapLike$$anon$1
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1183)
> at
> java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1547)
> at
> java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
> at
> java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431)
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
> at
> java.io.ObjectOutputStream.defaultWriteFields(ObjectOutputStream.java:1547)
> at
> java.io.ObjectOutputStream.writeSerialData(ObjectOutputStream.java:1508)
> at
> java.io.ObjectOutputStream.writeOrdinaryObject(ObjectOutputStream.java:1431)
> at java.io.ObjectOutputStream.writeObject0(ObjectOutputStream.java:1177)
> at java.io.ObjectOutputStream.writeObject(ObjectOutputStream.java:347)
> at org.apache.spark.util.Utils$.serialize(Utils.scala:83)
> at
> org.apache.spark.scheduler.cluster.mesos.ZookeeperMesosClusterPersistenceEngine.persist(MesosClusterPersistenceEngine.scala:110)
> at
> org.apache.spark.scheduler.cluster.mesos.MesosClusterScheduler.submitDriver(MesosClusterScheduler.scala:166)
> at
> org.apache.spark.deploy.rest.mesos.MesosSubmitRequestServlet.handleSubmit(MesosRestServer.scala:132)
> at
> org.apache.spark.deploy.rest.SubmitRequestServlet.doPost(RestSubmissionServer.scala:258)
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
