Hi, I try to test spark streaming 2.2.0 version with confluent 3.3.0
I have got lot of error during compilation this is my sbt: lazy val sparkstreaming = (project in file(".")) .settings( name := "sparkstreaming", organization := "org.arek", version := "0.1-SNAPSHOT", scalaVersion := "2.11.8", libraryDependencies ++= Seq( "org.apache.spark" %% "spark-streaming" % "2.2.0", "org.apache.spark" %% "spark-streaming-kafka-0-8" % "2.2.0", "io.confluent" % "kafka-avro-serializer" % "3.3.0" ) ) import org.apache.spark._ import org.apache.spark.streaming._ import org.apache.spark.streaming.kafka.KafkaUtils import io.confluent.kafka.serializers.KafkaAvroDecoder object Transformation extends Serializable { def main(args: Array[String]) = { val conf = new SparkConf().setAppName("StreamingTranformation").setMaster("local[*]") val streamingContext = new StreamingContext(conf, Seconds(1)) val kafkaParams = Map[String, String]("metadata.broker.list" -> "local:2181", "schema.registry.url" -> "http://local:8081", "auto.offset.reset" -> "smallest") val topicSet = Set("GEXPPROD_ROUTE") val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2) val lines = messages.foreachRDD(rdd => { rdd.foreach({ avroRecord => println(avroRecord) }) }) } [warn] Found version conflict(s) in library dependencies; some are suspected to be binary incompatible: [warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, 3.7.0.Final} [warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends on 3.7.0.Final) [warn] +- org.apache.zookeeper:zookeeper:3.4.8 (depends on 3.7.0.Final) [warn] +- org.apache.zookeeper:zookeeper:3.4.6 (depends on 3.6.2.Final) [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends on 3.6.2.Final) [warn] * commons-net:commons-net:2.2 is selected over 3.1 [warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends on 3.1) [warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends on 3.1) [warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1} [warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends on 11.0.2) [warn] +- org.apache.curator:curator-framework:2.6.0 (depends on 16.0.1) [warn] +- org.apache.curator:curator-client:2.6.0 (depends on 16.0.1) [warn] +- org.apache.curator:curator-recipes:2.6.0 (depends on 16.0.1) [warn] +- org.htrace:htrace-core:3.0.4 (depends on 12.0.1) [warn] Run 'evicted' to see detailed eviction warnings [info] Compiling 1 Scala source to /home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes ... [error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:6:11: object confluent is not a member of package org.apache.spark.io [error] import io.confluent.kafka.serializers.KafkaAvroDecoder [error] ^ [error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:66: not found: type KafkaAvroDecoder [error] val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2) [error] ^ [error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:19:84: not found: type KafkaAvroDecoder [error] val messages = KafkaUtils.createDirectStream[Object, Object, KafkaAvroDecoder, KafkaAvroDecoder](streamingContext, kafkaParams, topicSet).map(_._2) [error] When changing to library "org.apache.spark" %% "spark-streaming-kafka-0-10" % "2.2.0" : [warn] Found version conflict(s) in library dependencies; some are suspected to be binary incompatible: [warn] * com.101tec:zkclient:0.10 is selected over 0.8 [warn] +- io.confluent:common-utils:3.3.0 (depends on 0.10) [warn] +- org.apache.kafka:kafka_2.11:0.10.0.1 (depends on 0.8) [warn] * io.netty:netty:3.9.9.Final is selected over {3.6.2.Final, 3.7.0.Final} [warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends on 3.7.0.Final) [warn] +- org.apache.zookeeper:zookeeper:3.4.8 (depends on 3.7.0.Final) [warn] +- org.apache.zookeeper:zookeeper:3.4.6 (depends on 3.6.2.Final) [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends on 3.6.2.Final) [warn] * commons-net:commons-net:2.2 is selected over 3.1 [warn] +- org.apache.spark:spark-core_2.11:2.2.0 (depends on 3.1) [warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends on 3.1) [warn] * com.google.guava:guava:11.0.2 is selected over {12.0.1, 16.0.1} [warn] +- org.apache.hadoop:hadoop-yarn-client:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-api:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-common:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-server-nodemanager:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-common:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-yarn-server-common:2.6.5 (depends on 11.0.2) [warn] +- org.apache.hadoop:hadoop-hdfs:2.6.5 (depends on 11.0.2) [warn] +- org.apache.curator:curator-framework:2.6.0 (depends on 16.0.1) [warn] +- org.apache.curator:curator-client:2.6.0 (depends on 16.0.1) [warn] +- org.apache.curator:curator-recipes:2.6.0 (depends on 16.0.1) [warn] +- org.htrace:htrace-core:3.0.4 (depends on 12.0.1) [warn] Run 'evicted' to see detailed eviction warnings [info] Compiling 1 Scala source to /home/adminuser/data-streaming-platform/sparkstreaming/target/scala-2.11/classes ... [error] /home/adminuser/data-streaming-platform/sparkstreaming/src/main/scala/com/arek/streaming/spark/Transformation.scala:18:11: object confluent is not a member of package org.apache.spark.io [error] import io.confluent.kafka.serializers.KafkaAvroDecoder Regards, Arek ^