It's a client and server version conflict. The default hadoop version support with spark 0.8 is 1.2.x. Please check the version of the hadoop cluster that you are running, it should be the same as 1.x. If version issues are not there, do check if you've some old stray jars in classpath.
On Thu, Nov 21, 2013 at 7:27 PM, Prashant Sharma <[email protected]>wrote: > What is the version of hadoop running HDFS ? > > > On Thu, Nov 21, 2013 at 5:34 PM, <[email protected]> wrote: > >> Hi, >> >> >> >> I am running pre-built spark 0.8 CDH4 binary downloaded from spark site >> with my application and it fails upon saving RDD as text file into HDFS >> >> >> >> This code works as expected without any exception in another env that >> uses pre-built spark 0.8 Hadoop 1 but against native apache hadoop 1.2.1. >> >> >> >> I followed the steps in >> http://spark.incubator.apache.org/docs/latest/hadoop-third-party-distributions.html >> >> >> >> In my spark-env.sh I have these settings copied to all nodes. >> >> >> >> export SPARK_JAVA_OPTS+=" -Dspark.local.dir=/tmp/spark" >> >> export HADOOP_CONF_DIR=/etc/hadoop/conf >> >> >> >> cluster logs not showing any errors. >> >> >> >> Any help in this exception? >> >> >> >> org.apache.hadoop.ipc.RemoteException: Server IPC version 7 cannot >> communicate with client version 4 >> >> at org.apache.hadoop.ipc.Client.call(Client.java:1113) >> >> at org.apache.hadoop.ipc.RPC$Invoker.invoke(RPC.java:229) >> >> at $Proxy74.getProtocolVersion(Unknown Source) >> >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> >> at >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) >> >> at >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) >> >> at java.lang.reflect.Method.invoke(Method.java:597) >> >> at >> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:85) >> >> at >> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:62) >> >> at $Proxy74.getProtocolVersion(Unknown Source) >> >> at org.apache.hadoop.ipc.RPC.checkVersion(RPC.java:422) >> >> at >> org.apache.hadoop.hdfs.DFSClient.createNamenode(DFSClient.java:183) >> >> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:281) >> >> at org.apache.hadoop.hdfs.DFSClient.<init>(DFSClient.java:245) >> >> at >> org.apache.hadoop.hdfs.DistributedFileSystem.initialize(DistributedFileSystem.java:100) >> >> at >> org.apache.hadoop.fs.FileSystem.createFileSystem(FileSystem.java:1446) >> >> at org.apache.hadoop.fs.FileSystem.access$200(FileSystem.java:67) >> >> at org.apache.hadoop.fs.FileSystem$Cache.get(FileSystem.java:1464) >> >> at org.apache.hadoop.fs.FileSystem.get(FileSystem.java:263) >> >> at org.apache.hadoop.fs.Path.getFileSystem(Path.java:187) >> >> at >> org.apache.hadoop.mapred.SparkHadoopWriter$.createPathFromString(SparkHadoopWriter.scala:194) >> >> at >> org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopFile(PairRDDFunctions.scala:632) >> >> at >> org.apache.spark.rdd.PairRDDFunctions.saveAsHadoopFile(PairRDDFunctions.scala:523) >> >> at org.apache.spark.rdd.RDD.saveAsTextFile(RDD.scala:818) >> >> at >> org.apache.spark.api.java.JavaRDDLike$class.saveAsTextFile(JavaRDDLike.scala:331) >> >> * at >> org.apache.spark.api.java.JavaRDD.saveAsTextFile(JavaRDD.scala:25)* >> >> at >> com.dell.omdb.service.DatasetServiceImpl.addDataSet(DatasetServiceImpl.java:239) >> >> at >> com.dell.omdb.rest.DatasetResource.createDataset(DatasetResource.java:59) >> >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> >> at >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) >> >> at >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) >> >> at java.lang.reflect.Method.invoke(Method.java:597) >> >> at >> com.sun.jersey.spi.container.JavaMethodInvokerFactory$1.invoke(JavaMethodInvokerFactory.java:60) >> >> at >> com.sun.jersey.server.impl.model.method.dispatch.AbstractResourceMethodDispatchProvider$ResponseOutInvoker._dispatch(AbstractResourceMethodDispatchProvider.java:205) >> >> at >> com.sun.jersey.server.impl.model.method.dispatch.ResourceJavaMethodDispatcher.dispatch(ResourceJavaMethodDispatcher.java:75) >> >> at >> com.sun.jersey.server.impl.uri.rules.HttpMethodRule.accept(HttpMethodRule.java:288) >> >> at >> com.sun.jersey.server.impl.uri.rules.ResourceClassRule.accept(ResourceClassRule.java:108) >> >> at >> com.sun.jersey.server.impl.uri.rules.RightHandPathRule.accept(RightHandPathRule.java:147) >> >> at >> com.sun.jersey.server.impl.uri.rules.RootResourceClassesRule.accept(RootResourceClassesRule.java:84) >> >> at >> com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1469) >> >> at >> com.sun.jersey.server.impl.application.WebApplicationImpl._handleRequest(WebApplicationImpl.java:1400) >> >> at >> com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1349) >> >> at >> com.sun.jersey.server.impl.application.WebApplicationImpl.handleRequest(WebApplicationImpl.java:1339) >> >> at >> com.sun.jersey.spi.container.servlet.WebComponent.service(WebComponent.java:416) >> >> at >> com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:537) >> >> at >> com.sun.jersey.spi.container.servlet.ServletContainer.service(ServletContainer.java:699) >> >> at javax.servlet.http.HttpServlet.service(HttpServlet.java:820) >> >> at >> org.eclipse.jetty.servlet.ServletHolder.handle(ServletHolder.java:652) >> >> at >> org.eclipse.jetty.servlet.ServletHandler.doHandle(ServletHandler.java:447) >> >> at >> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:137) >> >> at >> org.eclipse.jetty.security.SecurityHandler.handle(SecurityHandler.java:556) >> >> at >> org.eclipse.jetty.server.session.SessionHandler.doHandle(SessionHandler.java:227) >> >> at >> org.eclipse.jetty.server.handler.ContextHandler.doHandle(ContextHandler.java:1038) >> >> at >> org.eclipse.jetty.servlet.ServletHandler.doScope(ServletHandler.java:374) >> >> at >> org.eclipse.jetty.server.session.SessionHandler.doScope(SessionHandler.java:189) >> >> at >> org.eclipse.jetty.server.handler.ContextHandler.doScope(ContextHandler.java:972) >> >> at >> org.eclipse.jetty.server.handler.ScopedHandler.handle(ScopedHandler.java:135) >> >> at >> org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:116) >> >> at org.eclipse.jetty.server.Server.handle(Server.java:363) >> >> at >> org.eclipse.jetty.server.AbstractHttpConnection.handleRequest(AbstractHttpConnection.java:483) >> >> at >> org.eclipse.jetty.server.AbstractHttpConnection.content(AbstractHttpConnection.java:931) >> >> at >> org.eclipse.jetty.server.AbstractHttpConnection$RequestHandler.content(AbstractHttpConnection.java:992) >> >> at >> org.eclipse.jetty.http.HttpParser.parseNext(HttpParser.java:856) >> >> at >> org.eclipse.jetty.http.HttpParser.parseAvailable(HttpParser.java:240) >> >> at >> org.eclipse.jetty.server.AsyncHttpConnection.handle(AsyncHttpConnection.java:82) >> >> at >> org.eclipse.jetty.io.nio.SelectChannelEndPoint.handle(SelectChannelEndPoint.java:628) >> >> at >> org.eclipse.jetty.io.nio.SelectChannelEndPoint$1.run(SelectChannelEndPoint.java:52) >> >> at >> org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:608) >> >> at >> org.eclipse.jetty.util.thread.QueuedThreadPool$3.run(QueuedThreadPool.java:543) >> >> at java.lang.Thread.run(Thread.java:662) >> >> >> >> Thanks, >> >> Hussam >> > > > > -- > s > -- It's just about how deep your longing is!
