Hi Arkadiusz,
Try 'rooting' your import. It looks like the import is being interpreted as
being relative to the previous.
'rooting; is done by adding the '_root_' prefix to your import:
import org.apache.spark.streaming.kafka.KafkaUtils
import _root_.io.confluent.kafka.serializers.KafkaAvroDecoder
kr, Gerard.
On Wed, Dec 13, 2017 at 6:05 PM, Arkadiusz Bicz <arkadiusz.b...@gmail.com>
wrote:
> Hi,
>
> I try to test spark streaming 2.2.0 version with confluent 3.3.0
>
> I have got lot of error during compilation this is my sbt:
>
> lazy val sparkstreaming = (project in file("."))
> .settings(
> name := "sparkstreaming",
> organization := "org.arek",
> version := "0.1-SNAPSHOT",
> scalaVersion := "2.11.8",
> libraryDependencies ++= Seq(
> "org.apache.spark" %% "spark-streaming" % "2.2.0",
> "org.apache.spark" %% "spark-streaming-kafka-0-8" % "2.2.0",
> "io.confluent" % "kafka-avro-serializer" % "3.3.0"
> )
> )
>
>
> import org.apache.spark._
> import org.apache.spark.streaming._
> import org.apache.spark.streaming.kafka.KafkaUtils
> import io.confluent.kafka.serializers.KafkaAvroDecoder
>
> object Transformation extends Serializable {
>
> def main(args: Array[String]) = {
> val conf = new SparkConf().setAppName("StreamingTranformation").
> setMaster("local[*]")
> val streamingContext = new StreamingContext(conf, Seconds(1))
>
> val kafkaParams = Map[String, String]("metadata.broker.list" ->
> "local:2181",
> "schema.registry.url" -> "http://local:8081";,
> "auto.offset.reset" -> "smallest")
>
> val topicSet = Set("GEXPPROD_ROUTE")
> val messages = KafkaUtils.createDirectStream[Object, Object,
> KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams,
> topicSet).map(_._2)
>
> val lines = messages.foreachRDD(rdd => {
> rdd.foreach({ avroRecord =>
> println(avroRecord)
> })
> })
> }
>
>
> [warn] Found version conflict(s) in library dependencies; some are
> suspected to be binary incompatible:
> [warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final,
> 3.7.0.Final}
> [warn] +- org.apache.spark:spark-core_2.11:2.2.0
> (depends on 3.7.0.Final)
> [warn] +- org.apache.zookeeper:zookeeper:3.4.8
> (depends on 3.7.0.Final)
> [warn] +- org.apache.zookeeper:zookeeper:3.4.6
> (depends on 3.6.2.Final)
> [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5
> (depends on 3.6.2.Final)
> [warn] * commons-net:commons-net:2.2 is selected over 3.1
> [warn] +- org.apache.spark:spark-core_2.11:2.2.0
> (depends on 3.1)
> [warn] +- org.apache.hadoop:hadoop-common:2.6.5
> (depends on 3.1)
> [warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
> [warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-common:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.curator:curator-framework:2.6.0
> (depends on 16.0.1)
> [warn] +- org.apache.curator:curator-client:2.6.0
> (depends on 16.0.1)
> [warn] +- org.apache.curator:curator-recipes:2.6.0
> (depends on 16.0.1)
> [warn] +- org.htrace:htrace-core:3.0.4 (depends
> on 12.0.1)
> [warn] Run 'evicted' to see detailed eviction warnings
> [info] Compiling 1 Scala source to /home/adminuser/data-
> streaming-platform/sparkstreaming/target/scala-2.11/classes ...
> [error] /home/adminuser/data-streaming-platform/
> sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:6:11:
> object confluent is not a member of package org.apache.spark.io
> [error] import io.confluent.kafka.serializers.KafkaAvroDecoder
> [error] ^
> [error] /home/adminuser/data-streaming-platform/
> sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:66:
> not found: type KafkaAvroDecoder
> [error] val messages = KafkaUtils.createDirectStream[Object, Object,
> KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams,
> topicSet).map(_._2)
> [error] ^
> [error] /home/adminuser/data-streaming-platform/
> sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:84:
> not found: type KafkaAvroDecoder
> [error] val messages = KafkaUtils.createDirectStream[Object, Object,
> KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams,
> topicSet).map(_._2)
> [error]
>
>
> When changing to library "org.apache.spark" %%
> "spark-streaming-kafka-0-10" % "2.2.0" :
>
>
> [warn] Found version conflict(s) in library dependencies; some are
> suspected to be binary incompatible:
> [warn] * com.101tec:zkclient:0.10 is selected over 0.8
> [warn] +- io.confluent:common-utils:3.3.0
> (depends on 0.10)
> [warn] +- org.apache.kafka:kafka_2.11:0.10.0.1
> (depends on 0.8)
> [warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final,
> 3.7.0.Final}
> [warn] +- org.apache.spark:spark-core_2.11:2.2.0
> (depends on 3.7.0.Final)
> [warn] +- org.apache.zookeeper:zookeeper:3.4.8
> (depends on 3.7.0.Final)
> [warn] +- org.apache.zookeeper:zookeeper:3.4.6
> (depends on 3.6.2.Final)
> [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5
> (depends on 3.6.2.Final)
> [warn] * commons-net:commons-net:2.2 is selected over 3.1
> [warn] +- org.apache.spark:spark-core_2.11:2.2.0
> (depends on 3.1)
> [warn] +- org.apache.hadoop:hadoop-common:2.6.5
> (depends on 3.1)
> [warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1}
> [warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-common:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5
> (depends on 11.0.2)
> [warn] +- org.apache.curator:curator-framework:2.6.0
> (depends on 16.0.1)
> [warn] +- org.apache.curator:curator-client:2.6.0
> (depends on 16.0.1)
> [warn] +- org.apache.curator:curator-recipes:2.6.0
> (depends on 16.0.1)
> [warn] +- org.htrace:htrace-core:3.0.4 (depends
> on 12.0.1)
> [warn] Run 'evicted' to see detailed eviction warnings
> [info] Compiling 1 Scala source to /home/adminuser/data-
> streaming-platform/sparkstreaming/target/scala-2.11/classes ...
> [error] /home/adminuser/data-streaming-platform/
> sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:18:11:
> object confluent is not a member of package org.apache.spark.io
> [error] import io.confluent.kafka.serializers.KafkaAvroDecoder
>
> Regards,
>
> Arek
> ^
>
