[
https://issues.apache.org/jira/browse/LIVY-514?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Ajay Yadav updated LIVY-514:
----------------------------
Description:
I am trying to submit jobs programmatically through Livy Scala API. I am able
to do it through REST call but not through programmatic API. Here is the code:
\{{ }}
<code>
import org.apache.livy._
import org.apache.livy.scalaapi._
import java.net.URI
import java.io.
{File, FileNotFoundException}
import scala.concurrent.duration._
import scala.concurrent.Await
object LivyClient {
def main(args: Array[String]): Unit =
{ val livyurl = "http://localhost:8998"; val javaClient = new
LivyClientBuilder(false) .setURI(new URI(livyurl)) .build() val
livyScalaClient = javaClient.asScalaClient val testJarPath =
"target/scala-2.11/SparkApp-assembly-0.1.jar" livyScalaClient.uploadJar(new
File(testJarPath)) var jobHandle = livyScalaClient.submit(JobContext =>
"hello") println(Await.result(jobHandle, 2000 millis)) }
}
</code>
I am running this code on EMR cluster where livy server is running at port
8998. I am running this code through spark-submit.
The code is giving error while creating javaClient. Error:
Exception in thread "main" java.lang.IllegalStateException: No
LivyClientFactory implementation was found.
at org.apache.livy.LivyClientBuilder.build(LivyClientBuilder.java:124)
at LivyClient$.main(LivyClient.scala:16)
at LivyClient.main(LivyClient.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
was:
I am trying to submit jobs programmatically through Livy Scala API. I am able
to do it through REST call but not through programmatic API. Here is the code:
{{ }}
{{{\{code}}}}
import org.apache.livy._
import org.apache.livy.scalaapi._
import java.net.URI
import java.io.{File, FileNotFoundException}
import scala.concurrent.duration._
import scala.concurrent.Await
object LivyClient {
def main(args: Array[String]): Unit = {
val livyurl = "http://localhost:8998";
val javaClient = new LivyClientBuilder(false)
.setURI(new URI(livyurl))
.build()
val livyScalaClient = javaClient.asScalaClient
val testJarPath = "target/scala-2.11/SparkApp-assembly-0.1.jar"
livyScalaClient.uploadJar(new File(testJarPath))
var jobHandle = livyScalaClient.submit(JobContext => "hello")
println(Await.result(jobHandle, 2000 millis))
}
}
{{/code}}
I am running this code on EMR cluster where livy server is running at port
8998. I am running this code through spark-submit.
The code is giving error while creating javaClient. Error:
Exception in thread "main" java.lang.IllegalStateException: No
LivyClientFactory implementation was found.
at org.apache.livy.LivyClientBuilder.build(LivyClientBuilder.java:124)
at LivyClient$.main(LivyClient.scala:16)
at LivyClient.main(LivyClient.scala)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
at
org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198)
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228)
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
> No LivyClientFactory implementation was found
> ---------------------------------------------
>
> Key: LIVY-514
> URL: https://issues.apache.org/jira/browse/LIVY-514
> Project: Livy
> Issue Type: Bug
> Environment: on linux using spark 2.3.0 and scala 2.11
> Reporter: Ajay Yadav
> Priority: Major
>
>
> I am trying to submit jobs programmatically through Livy Scala API. I am able
> to do it through REST call but not through programmatic API. Here is the
> code: \{{ }}
> <code>
> import org.apache.livy._
> import org.apache.livy.scalaapi._
>
> import java.net.URI
> import java.io.
> {File, FileNotFoundException}
> import scala.concurrent.duration._
> import scala.concurrent.Await
> object LivyClient {
> def main(args: Array[String]): Unit =
> { val livyurl = "http://localhost:8998"; val javaClient = new
> LivyClientBuilder(false) .setURI(new URI(livyurl)) .build()
> val livyScalaClient = javaClient.asScalaClient val testJarPath =
> "target/scala-2.11/SparkApp-assembly-0.1.jar"
> livyScalaClient.uploadJar(new File(testJarPath)) var jobHandle =
> livyScalaClient.submit(JobContext => "hello")
> println(Await.result(jobHandle, 2000 millis)) }
> }
> </code>
> I am running this code on EMR cluster where livy server is running at port
> 8998. I am running this code through spark-submit.
> The code is giving error while creating javaClient. Error:
> Exception in thread "main" java.lang.IllegalStateException: No
> LivyClientFactory implementation was found.
> at org.apache.livy.LivyClientBuilder.build(LivyClientBuilder.java:124)
> at LivyClient$.main(LivyClient.scala:16)
> at LivyClient.main(LivyClient.scala)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at
> org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
> at
> org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894)
> at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198)
> at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228)
> at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137)
> at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
>
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
