Is there a way to resolve this, in HDP2.6.4 & Livy 0.4.0,
Steps:
1. Livy session was created as kind=spark.
2. Added scala jar(my function)
3. livyClient.run(livyJob()) gives error:
18/02/20 14:53:44 INFO InteractiveSession: Interactive session 12 created
[appid: application_1518698685392_0040, owner: null, proxyUser: Some(hive),
state: idle, kind: spark, info:
{driverLogUrl=http://hdp04d03.fuzzyl.com:8042/node/containerlogs/container_e04_1518698685392_0040_01_000001/hive,
sparkUiUrl=http://ambari04.fuzzyl.com:8088/proxy/application_1518698685392_0040/}]
18/02/20 14:53:44 INFO RSCClient: Received result for
51f4a85f-a499-4dc4-ad9f-9a0e7a42ea64
18/02/20 14:53:44 ERROR SessionServlet$: internal error
java.util.concurrent.ExecutionException: java.lang.RuntimeException:
py4j.Py4JException: Error while obtaining a new communication channel
py4j.CallbackClient.getConnectionLock(CallbackClient.java:218)
py4j.CallbackClient.sendCommand(CallbackClient.java:337)
py4j.CallbackClient.sendCommand(CallbackClient.java:316)
py4j.reflection.PythonProxyHandler.invoke(PythonProxyHandler.java:103)
com.sun.proxy.$Proxy24.getLocalTmpDirPath(Unknown Source)
org.apache.livy.repl.PythonInterpreter.addPyFile(PythonInterpreter.scala:264)
org.apache.livy.repl.ReplDriver$$anonfun$addJarOrPyFile$1.apply(ReplDriver.scala:110)
org.apache.livy.repl.ReplDriver$$anonfun$addJarOrPyFile$1.apply(ReplDriver.scala:110)
scala.Option.foreach(Option.scala:257)
org.apache.livy.repl.ReplDriver.addJarOrPyFile(ReplDriver.scala:110)
org.apache.livy.rsc.driver.JobContextImpl.addJarOrPyFile(JobContextImpl.java:100)
org.apache.livy.rsc.driver.AddJarJob.call(AddJarJob.java:39)
org.apache.livy.rsc.driver.JobWrapper.call(JobWrapper.java:57)
org.apache.livy.rsc.driver.JobWrapper.call(JobWrapper.java:34)
java.util.concurrent.FutureTask.run(FutureTask.java:266)
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
java.lang.Thread.run(Thread.java:745)
at io.netty.util.concurrent.AbstractFuture.get(AbstractFuture.java:37)
at org.apache.livy.rsc.JobHandleImpl.get(JobHandleImpl.java:60)
at
org.apache.livy.server.interactive.InteractiveSession.addJar(InteractiveSession.scala:542)
at
org.apache.livy.server.interactive.InteractiveSessionServlet.org$apache$livy$server$interactive$InteractiveSessionServlet$$addJarOrPyFile(InteractiveSessionServlet.scala:241)
at
org.apache.livy.server.interactive.InteractiveSessionServlet$$anonfun$19$$anonfun$apply$16.apply(InteractiveSessionServlet.scala:208)
at
org.apache.livy.server.interactive.InteractiveSessionServlet$$anonfun$19$$anonfun$apply$16.apply(InteractiveSessionServlet.scala:207)
at
org.apache.livy.server.interactive.SessionHeartbeatNotifier$$anonfun$withModifyAccessSession$1.apply(SessionHeartbeat.scala:76)
at
org.apache.livy.server.interactive.SessionHeartbeatNotifier$$anonfun$withModifyAccessSession$1.apply(SessionHeartbeat.scala:74)
at
org.apache.livy.server.SessionServlet.doWithSession(SessionServlet.scala:221)
at
org.apache.livy.server.SessionServlet.withModifyAccessSession(SessionServlet.scala:212)
at
org.apache.livy.server.interactive.InteractiveSessionServlet.org$apache$livy$server$interactive$SessionHeartbeatNotifier$$super$withModifyAccessSession(InteractiveSessionServlet.scala:40)
at
org.apache.livy.server.interactive.SessionHeartbeatNotifier$class.withModifyAccessSession(SessionHeartbeat.scala:74)
at
org.apache.livy.server.interactive.InteractiveSessionServlet.withModifyAccessSession(InteractiveSessionServlet.scala:40)
at
org.apache.livy.server.interactive.InteractiveSessionServlet$$anonfun$19.apply(InteractiveSessionServlet.scala:207)
at
org.apache.livy.server.interactive.InteractiveSessionServlet$$anonfun$19.apply(InteractiveSessionServlet.scala:206)
at
org.apache.livy.server.JsonServlet.org$apache$livy$server$JsonServlet$$doAction(JsonServlet.scala:113)
at
org.apache.livy.server.JsonServlet$$anonfun$jpost$1.apply(JsonServlet.scala:75)
at
org.scalatra.ScalatraBase$class.org$scalatra$ScalatraBase$$liftAction(ScalatraBase.scala:270)
at
org.scalatra.ScalatraBase$$anonfun$invoke$1.apply(ScalatraBase.scala:265)
at
org.scalatra.ScalatraBase$$anonfun$invoke$1.apply(ScalatraBase.scala:265)
at
org.scalatra.ApiFormats$class.withRouteMultiParams(ApiFormats.scala:178)
at
org.apache.livy.server.JsonServlet.withRouteMultiParams(JsonServlet.scala:39)
at org.scalatra.ScalatraBase$class.invoke(ScalatraBase.scala:264)
at org.scalatra.ScalatraServlet.invoke(ScalatraServlet.scala:49)
at
org.scalatra.ScalatraBase$$anonfun$runRoutes$1$$anonfun$apply$8.apply(ScalatraBase.scala:240)
at
org.scalatra.ScalatraBase$$anonfun$runRoutes$1$$anonfun$apply$8.apply(ScalatraBase.scala:238)
at scala.Option.flatMap(Option.scala:170)
at
org.scalatra.ScalatraBase$$anonfun$runRoutes$1.apply(ScalatraBase.scala:238)
at
org.scalatra.ScalatraBase$$anonfun$runRoutes$1.apply(ScalatraBase.scala:237)
at scala.collection.immutable.Stream.flatMap(Stream.scala:446)
