Hi Madhukar, could you check whether your Flink installation contains the flink-dist-0.10.1.jar in the lib folder? This file contains the necessary scala-library.jar which you are missing. You can also remove the line <exclude>org.scala-lang:scala-library</exclude> which excludes the scala-library dependency to be included in the fat jar of your job.
Cheers, Till On Wed, Jan 6, 2016 at 5:54 AM, Madhukar Thota <madhukar.th...@gmail.com> wrote: > Hi > > I am seeing the following error when i am trying to run the jar in Flink > Cluster. I am not sure what dependency is missing. > > /opt/DataHUB/flink-0.10.1/bin/flink run datahub-heka-1.0-SNAPSHOT.jar > flink.properties > java.lang.NoClassDefFoundError: scala/collection/GenTraversableOnce$class > at kafka.utils.Pool.<init>(Pool.scala:28) > at > kafka.consumer.FetchRequestAndResponseStatsRegistry$.<init>(FetchRequestAndResponseStats.scala:60) > at > kafka.consumer.FetchRequestAndResponseStatsRegistry$.<clinit>(FetchRequestAndResponseStats.scala) > at kafka.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:39) > at > kafka.javaapi.consumer.SimpleConsumer.<init>(SimpleConsumer.scala:34) > at > org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.getPartitionsForTopic(FlinkKafkaConsumer.java:691) > at > org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.<init>(FlinkKafkaConsumer.java:281) > at > org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer082.<init>(FlinkKafkaConsumer082.java:49) > at com.lmig.datahub.heka.Main.main(Main.java:39) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:497) > at > org.apache.flink.client.program.PackagedProgram.callMainMethod(PackagedProgram.java:497) > at > org.apache.flink.client.program.PackagedProgram.invokeInteractiveModeForExecution(PackagedProgram.java:395) > at > org.apache.flink.client.program.Client.runBlocking(Client.java:252) > at > org.apache.flink.client.CliFrontend.executeProgramBlocking(CliFrontend.java:676) > at org.apache.flink.client.CliFrontend.run(CliFrontend.java:326) > at > org.apache.flink.client.CliFrontend.parseParameters(CliFrontend.java:978) > at org.apache.flink.client.CliFrontend.main(CliFrontend.java:1028) > Caused by: java.lang.ClassNotFoundException: > scala.collection.GenTraversableOnce$class > at java.net.URLClassLoader.findClass(URLClassLoader.java:381) > at java.lang.ClassLoader.loadClass(ClassLoader.java:424) > at java.lang.ClassLoader.loadClass(ClassLoader.java:357) > ... 20 more > > The exception above occurred while trying to run your command. > > > Here is my pom.xml: > > <?xml version="1.0" encoding="UTF-8"?> > <project xmlns="http://maven.apache.org/POM/4.0.0"; > xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; > xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 > http://maven.apache.org/xsd/maven-4.0.0.xsd";> > <modelVersion>4.0.0</modelVersion> > > <groupId>com.datahub</groupId> > <artifactId>datahub-heka</artifactId> > <version>1.0-SNAPSHOT</version> > <dependencies> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-java</artifactId> > <version>0.10.1</version> > </dependency> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-streaming-java</artifactId> > <version>0.10.1</version> > </dependency> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-clients</artifactId> > <version>0.10.1</version> > </dependency> > <dependency> > <groupId>org.apache.kafka</groupId> > <artifactId>kafka_2.10</artifactId> > <version>0.8.2.2</version> > </dependency> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-connector-kafka</artifactId> > <version>0.10.1</version> > </dependency> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-connector-elasticsearch</artifactId> > <version>0.10.1</version> > </dependency> > <dependency> > <groupId>org.elasticsearch</groupId> > <artifactId>elasticsearch</artifactId> > <version>1.7.2</version> > </dependency> > <dependency> > <groupId>org.elasticsearch</groupId> > <artifactId>elasticsearch-shield</artifactId> > <version>1.3.3</version> > </dependency> > <dependency> > <groupId>org.elasticsearch</groupId> > <artifactId>elasticsearch-license-plugin</artifactId> > <version>1.0.0</version> > </dependency> > <dependency> > <groupId>com.fasterxml.jackson.core</groupId> > <artifactId>jackson-core</artifactId> > <version>2.6.4</version> > </dependency> > <dependency> > <groupId>com.fasterxml.jackson.core</groupId> > <artifactId>jackson-databind</artifactId> > <version>2.6.4</version> > </dependency> > </dependencies> > <repositories> > <repository> > <id>elasticsearch-releases</id> > <url>http://maven.elasticsearch.org/releases</url> > <releases> > <enabled>true</enabled> > </releases> > <snapshots> > <enabled>false</enabled> > </snapshots> > </repository> > </repositories> > <build> > <plugins> > <!-- We use the maven-shade plugin to create a fat jar that > contains all dependencies > except flink and it's transitive dependencies. The resulting > fat-jar can be executed > on a cluster. Change the value of Program-Class if your program > entry point changes. --> > <plugin> > <groupId>org.apache.maven.plugins</groupId> > <artifactId>maven-shade-plugin</artifactId> > <version>2.4.1</version> > <executions> > <!-- Run shade goal on package phase --> > <execution> > <phase>package</phase> > <goals> > <goal>shade</goal> > </goals> > <configuration> > <artifactSet> > <excludes> > <!-- This list contains all dependencies > of flink-dist > Everything else will be packaged into the > fat-jar > --> > > <exclude>org.apache.flink:flink-shaded-*</exclude> > > <exclude>org.apache.flink:flink-core</exclude> > > <exclude>org.apache.flink:flink-java</exclude> > > <exclude>org.apache.flink:flink-scala</exclude> > > <exclude>org.apache.flink:flink-runtime</exclude> > > <exclude>org.apache.flink:flink-optimizer</exclude> > > <exclude>org.apache.flink:flink-clients</exclude> > > <exclude>org.apache.flink:flink-avro</exclude> > > <exclude>org.apache.flink:flink-java-examples</exclude> > > <exclude>org.apache.flink:flink-scala-examples</exclude> > > <exclude>org.apache.flink:flink-streaming-examples</exclude> > > <exclude>org.apache.flink:flink-streaming-java</exclude> > > <!-- Also exclude very big transitive > dependencies of Flink > WARNING: You have to remove these > excludes if your code relies on other > versions of these dependencies. > --> > > <exclude>org.scala-lang:scala-library</exclude> > > <exclude>org.scala-lang:scala-compiler</exclude> > > <exclude>org.scala-lang:scala-reflect</exclude> > > <exclude>com.amazonaws:aws-java-sdk</exclude> > > <exclude>com.typesafe.akka:akka-actor_*</exclude> > > <exclude>com.typesafe.akka:akka-remote_*</exclude> > > <exclude>com.typesafe.akka:akka-slf4j_*</exclude> > <exclude>io.netty:netty-all</exclude> > <exclude>io.netty:netty</exclude> > > <exclude>org.eclipse.jetty:jetty-server</exclude> > > <exclude>org.eclipse.jetty:jetty-continuation</exclude> > > <exclude>org.eclipse.jetty:jetty-http</exclude> > > <exclude>org.eclipse.jetty:jetty-io</exclude> > > <exclude>org.eclipse.jetty:jetty-util</exclude> > > <exclude>org.eclipse.jetty:jetty-security</exclude> > > <exclude>org.eclipse.jetty:jetty-servlet</exclude> > > <exclude>commons-fileupload:commons-fileupload</exclude> > <exclude>org.apache.avro:avro</exclude> > > <exclude>commons-collections:commons-collections</exclude> > > <exclude>org.codehaus.jackson:jackson-core-asl</exclude> > > <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude> > > <exclude>com.thoughtworks.paranamer:paranamer</exclude> > > <exclude>org.xerial.snappy:snappy-java</exclude> > > <exclude>org.apache.commons:commons-compress</exclude> > <exclude>org.tukaani:xz</exclude> > > <exclude>com.esotericsoftware.kryo:kryo</exclude> > > <exclude>com.esotericsoftware.minlog:minlog</exclude> > <exclude>org.objenesis:objenesis</exclude> > <exclude>com.twitter:chill_*</exclude> > <exclude>com.twitter:chill-java</exclude> > > <exclude>com.twitter:chill-avro_*</exclude> > > <exclude>com.twitter:chill-bijection_*</exclude> > > <exclude>com.twitter:bijection-core_*</exclude> > > <exclude>com.twitter:bijection-avro_*</exclude> > > <exclude>commons-lang:commons-lang</exclude> > <exclude>junit:junit</exclude> > > <exclude>de.javakaffee:kryo-serializers</exclude> > <exclude>joda-time:joda-time</exclude> > > <exclude>org.apache.commons:commons-lang3</exclude> > <exclude>org.slf4j:slf4j-api</exclude> > <exclude>org.slf4j:slf4j-log4j12</exclude> > <exclude>log4j:log4j</exclude> > > <exclude>org.apache.commons:commons-math</exclude> > > <exclude>org.apache.sling:org.apache.sling.commons.json</exclude> > > <exclude>commons-logging:commons-logging</exclude> > > <exclude>org.apache.httpcomponents:httpclient</exclude> > > <exclude>org.apache.httpcomponents:httpcore</exclude> > > <exclude>commons-codec:commons-codec</exclude> > > <exclude>com.fasterxml.jackson.core:jackson-core</exclude> > > <exclude>com.fasterxml.jackson.core:jackson-databind</exclude> > > <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude> > > <exclude>org.codehaus.jettison:jettison</exclude> > <exclude>stax:stax-api</exclude> > <exclude>com.typesafe:config</exclude> > > <exclude>org.uncommons.maths:uncommons-maths</exclude> > > <exclude>com.github.scopt:scopt_*</exclude> > > <exclude>org.mortbay.jetty:servlet-api</exclude> > <exclude>commons-io:commons-io</exclude> > <exclude>commons-cli:commons-cli</exclude> > </excludes> > </artifactSet> > <filters> > <filter> > <artifact>org.apache.flink:*</artifact> > <excludes> > > <exclude>org/apache/flink/shaded/**</exclude> > <exclude>web-docs/**</exclude> > </excludes> > </filter> > <filter> > <!-- Do not copy the signatures in the > META-INF folder. > Otherwise, this might cause > SecurityExceptions when using the JAR. --> > <artifact>*:*</artifact> > <excludes> > <exclude>META-INF/*.SF</exclude> > <exclude>META-INF/*.DSA</exclude> > <exclude>META-INF/*.RSA</exclude> > </excludes> > </filter> > </filters> > <transformers> > <!-- add Main-Class to manifest file --> > <transformer > implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> > > <mainClass>com.datahub.heka.Main</mainClass> > </transformer> > </transformers> > > <createDependencyReducedPom>false</createDependencyReducedPom> > </configuration> > </execution> > </executions> > </plugin> > > <!-- Configure the jar plugin to add the main class as a manifest > entry --> > <plugin> > <groupId>org.apache.maven.plugins</groupId> > <artifactId>maven-jar-plugin</artifactId> > <version>2.5</version> > <configuration> > <archive> > <manifestEntries> > <Main-Class>com.datahub.heka.Main</Main-Class> > </manifestEntries> > </archive> > </configuration> > </plugin> > > <plugin> > <groupId>org.apache.maven.plugins</groupId> > <artifactId>maven-compiler-plugin</artifactId> > <version>3.1</version> > <configuration> > <source>1.8</source> <!-- If you want to use Java 8, > change this to "1.8" --> > <target>1.8</target> <!-- If you want to use Java 8, > change this to "1.8" --> > </configuration> > </plugin> > </plugins> > > > </build> > <profiles> > <profile> > <!-- A profile that does everyting correctly: > We set the Flink dependencies to provided --> > <id>build-jar</id> > <activation> > <activeByDefault>false</activeByDefault> > </activation> > <dependencies> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-java</artifactId> > <version>0.10.1</version> > <scope>provided</scope> > </dependency> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-streaming-java</artifactId> > <version>0.10.1</version> > <scope>provided</scope> > </dependency> > <dependency> > <groupId>org.apache.flink</groupId> > <artifactId>flink-clients</artifactId> > <version>0.10.1</version> > <scope>provided</scope> > </dependency> > </dependencies> > </profile> > </profiles> > </project> > > > Any help is appreciated. > > > Thanks > >
