Alright. Thanks guys. So, I what version of Hadoop is currently supported by Spark. Also, I am not a Hadoop person, it is possible to access HDFS in Spark without YARN?
On Thu, Dec 12, 2013 at 5:19 PM, Prashant Sharma <[email protected]>wrote: > > Hey, > > On Thu, Dec 12, 2013 at 5:10 PM, Pinak Pani < > [email protected]> wrote: > >> Do you mean it has been decided not to support YARN 2.2 in any future >> release of version 0.8? >> >> > Well AFAIK. But it might get in 0.9. > > >> http://mail-archives.apache.org has big usability issue. You do not get >> URL at the thread level instead month level. Can you please tell me the >> subject of the mail you are referring. I will search in the threads. >> >> > Scala 2.10 Merge. > > >> Thanks. >> >> >> On Thu, Dec 12, 2013 at 4:45 PM, Prashant Sharma <[email protected]>wrote: >> >>> I don't think yarn 2.2 is supported in 0.8 and very soon it will not be >>> supported in master either. Read this thread >>> http://mail-archives.apache.org/mod_mbox/spark-dev/201312.mbox/browser. >>> >>> >>> On Thu, Dec 12, 2013 at 4:24 PM, Pinak Pani < >>> [email protected]> wrote: >>> >>>> I am trying to setup Spark with YARN 2.2.0. My Hadoop is plain Hadoop >>>> from Apache Hadoop website. When I SBT build against 2.2.0 it fails. While >>>> it compiles with a lot of warnings when I try against Hadoop 2.0.5-alpha. >>>> >>>> How can I compile Spark against YARN 2.2.0. >>>> >>>> There is a related thread here: >>>> https://groups.google.com/forum/#!topic/spark-users/8Gm6ByvdNME >>>> It did help. >>>> >>>> Also, I am a novice to SBT. >>>> >>>> Here is the error log: >>>> >>>> [root@ip-10-110-241-90 spark-0.8.0-incubating]# >>>> SPARK_HADOOP_VERSION=2.2.0 SPARK_YARN=true sbt/sbt clean assembly >>>> [info] Loading project definition from >>>> /tmp/spark/spark-0.8.0-incubating/project/project >>>> >>>> [-- snip --] >>>> >>>> [warn] jobCommitter.cleanupJob(jobTaskContext) >>>> [warn] ^ >>>> [warn] >>>> /tmp/spark/spark-0.8.0-incubating/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala:98: >>>> constructor Job in class Job is deprecated: see corresponding Javadoc for >>>> more information. >>>> [warn] val job = new Job(conf) >>>> [warn] ^ >>>> [warn] 9 warnings found >>>> [warn] Note: Some input files use unchecked or unsafe operations. >>>> [warn] Note: Recompile with -Xlint:unchecked for details. >>>> [info] Compiling 8 Scala sources to >>>> /tmp/spark/spark-0.8.0-incubating/yarn/target/scala-2.9.3/classes... >>>> [info] Compiling 50 Scala sources to >>>> /tmp/spark/spark-0.8.0-incubating/streaming/target/scala-2.9.3/classes... >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:42: >>>> not found: type AMRMProtocol >>>> [error] private var resourceManager: AMRMProtocol = null >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:126: >>>> not found: type AMRMProtocol >>>> [error] private def registerWithResourceManager(): AMRMProtocol = { >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:119: >>>> value AM_CONTAINER_ID_ENV is not a member of object >>>> org.apache.hadoop.yarn.api.ApplicationConstants >>>> [error] val containerIdString = >>>> envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:131: >>>> not found: type AMRMProtocol >>>> [error] return rpc.getProxy(classOf[AMRMProtocol], rmAddress, >>>> conf).asInstanceOf[AMRMProtocol] >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:131: >>>> not found: type AMRMProtocol >>>> [error] return rpc.getProxy(classOf[AMRMProtocol], rmAddress, >>>> conf).asInstanceOf[AMRMProtocol] >>>> [error] >>>> ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:138: >>>> value setApplicationAttemptId is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest >>>> [error] appMasterRequest.setApplicationAttemptId(appAttemptId) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:30: >>>> AMRMProtocol is not a member of org.apache.hadoop.yarn.api >>>> [error] import org.apache.hadoop.yarn.api.AMRMProtocol >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:504: >>>> not found: type AMRMProtocol >>>> [error] resourceManager: AMRMProtocol, appAttemptId: >>>> ApplicationAttemptId, >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:24: >>>> AMResponse is not a member of org.apache.hadoop.yarn.api.records >>>> [error] import org.apache.hadoop.yarn.api.records.{AMResponse, >>>> ApplicationAttemptId, ContainerId, Priority, Resource, ResourceRequest, >>>> ContainerStatus, Container} >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:493: >>>> not found: type AMRMProtocol >>>> [error] resourceManager: AMRMProtocol, appAttemptId: >>>> ApplicationAttemptId, >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:485: >>>> not found: type AMRMProtocol >>>> [error] resourceManager: AMRMProtocol, appAttemptId: >>>> ApplicationAttemptId, >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:325: >>>> value setAppAttemptId is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest >>>> [error] finishReq.setAppAttemptId(appAttemptId) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:326: >>>> value setFinishApplicationStatus is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest >>>> [error] finishReq.setFinishApplicationStatus(status) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:31: >>>> YarnClientImpl is not a member of org.apache.hadoop.yarn.client >>>> [error] import org.apache.hadoop.yarn.client.YarnClientImpl >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:42: >>>> not found: type YarnClientImpl >>>> [error] class Client(conf: Configuration, args: ClientArguments) >>>> extends YarnClientImpl with Logging { >>>> [error] >>>> ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:51: >>>> not found: value init >>>> [error] init(yarnConf) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:52: >>>> not found: value start >>>> [error] start() >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:55: >>>> value getNewApplication is not a member of AnyRef with >>>> org.apache.spark.Logging with ScalaObject >>>> [error] val newApp = super.getNewApplication() >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:66: >>>> value setUser is not a member of >>>> org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext >>>> [error] >>>> appContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName()) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:286: >>>> value submitApplication is not a member of AnyRef with >>>> org.apache.spark.Logging with ScalaObject >>>> [error] super.submitApplication(appContext) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:76: >>>> value getYarnClusterMetrics is not a member of AnyRef with >>>> org.apache.spark.Logging with ScalaObject >>>> [error] val clusterMetrics: YarnClusterMetrics = >>>> super.getYarnClusterMetrics >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:79: >>>> value getQueueInfo is not a member of AnyRef with org.apache.spark.Logging >>>> with ScalaObject >>>> [error] val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:216: >>>> value getMinimumResourceCapability is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse >>>> [error] val minResMemory: Int = >>>> newApp.getMinimumResourceCapability().getMemory() >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:273: >>>> value setResource is not a member of >>>> org.apache.hadoop.yarn.api.records.ContainerLaunchContext >>>> [error] amContainer.setResource(capability) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:278: >>>> value setContainerTokens is not a member of >>>> org.apache.hadoop.yarn.api.records.ContainerLaunchContext >>>> [error] >>>> amContainer.setContainerTokens(ByteBuffer.wrap(dob.getData())) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:292: >>>> value getApplicationReport is not a member of AnyRef with >>>> org.apache.spark.Logging with ScalaObject >>>> [error] val report = super.getApplicationReport(appId) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:34: >>>> ProtoUtils is not a member of org.apache.hadoop.yarn.util >>>> [error] import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, >>>> Records, ProtoUtils} >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:48: >>>> not found: type ContainerManager >>>> [error] var cm: ContainerManager = null >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:202: >>>> not found: type ContainerManager >>>> [error] def connectToCM: ContainerManager = { >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:63: >>>> value setContainerId is not a member of >>>> org.apache.hadoop.yarn.api.records.ContainerLaunchContext >>>> [error] ctx.setContainerId(container.getId()) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:64: >>>> value setResource is not a member of >>>> org.apache.hadoop.yarn.api.records.ContainerLaunchContext >>>> [error] ctx.setResource(container.getResource()) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:103: >>>> value setUser is not a member of >>>> org.apache.hadoop.yarn.api.records.ContainerLaunchContext >>>> [error] >>>> ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:108: >>>> value setContainerTokens is not a member of >>>> org.apache.hadoop.yarn.api.records.ContainerLaunchContext >>>> [error] ctx.setContainerTokens(ByteBuffer.wrap(dob.getData())) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:212: >>>> not found: value ProtoUtils >>>> [error] >>>> user.addToken(ProtoUtils.convertFromProtoFormat(containerToken, cmAddress)) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:216: >>>> not found: type ContainerManager >>>> [error] .doAs(new PrivilegedExceptionAction[ContainerManager] { >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:49: >>>> not found: type AMRMProtocol >>>> [error] private[yarn] class YarnAllocationHandler(val conf: >>>> Configuration, val resourceManager: AMRMProtocol, >>>> [error] >>>> ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:87: >>>> value getAMResponse is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse >>>> [error] val amResp = >>>> allocateWorkerResources(workersToRequest).getAMResponse >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:296: >>>> value getHostName is not a member of >>>> org.apache.hadoop.yarn.api.records.ResourceRequest >>>> [error] val candidateHost = container.getHostName >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:374: >>>> value setApplicationAttemptId is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest >>>> [error] req.setApplicationAttemptId(appAttemptId) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:376: >>>> value addAllAsks is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest >>>> [error] req.addAllAsks(resourceRequests) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:379: >>>> value addAllReleases is not a member of >>>> org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest >>>> [error] req.addAllReleases(releasedContainerList) >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:391: >>>> value getHostName is not a member of >>>> org.apache.hadoop.yarn.api.records.ResourceRequest >>>> [error] logInfo("rsrcRequest ... host : " + req.getHostName + ", >>>> numContainers : " + req.getNumContainers + >>>> [error] ^ >>>> [error] >>>> /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:441: >>>> value setHostName is not a member of >>>> org.apache.hadoop.yarn.api.records.ResourceRequest >>>> [error] rsrcRequest.setHostName(hostname) >>>> [error] ^ >>>> [error] 43 errors found >>>> >>>> [-- snip --] >>>> >>>> [warn] Merging >>>> 'org/yaml/snakeyaml/constructor/SafeConstructor$ConstructYamlBool.class' >>>> with strategy 'first' >>>> [warn] Merging >>>> 'org/yaml/snakeyaml/emitter/Emitter$ExpectStreamStart.class' with strategy >>>> 'first' >>>> [warn] Strategy 'concat' was applied to 2 files >>>> [warn] Strategy 'discard' was applied to 2 files >>>> [warn] Strategy 'first' was applied to 794 files >>>> [info] Checking every *.class/*.jar file's SHA-1. >>>> [info] SHA-1: deebf2bd4f022965649cfe78d51ff1c8780c92a2 >>>> [info] Packaging >>>> /tmp/spark/spark-0.8.0-incubating/examples/target/scala-2.9.3/spark-examples-assembly-0.8.0-incubating.jar >>>> ... >>>> [info] Done packaging. >>>> [error] (yarn/compile:compile) Compilation failed >>>> [error] Total time: 668 s, completed 12 Dec, 2013 10:15:25 AM >>>> >>>> >>> >>> >>> -- >>> s >>> >> >> > > > -- > s >
