I don't think yarn 2.2 is supported in 0.8 and very soon it will not be supported in master either. Read this thread http://mail-archives.apache.org/mod_mbox/spark-dev/201312.mbox/browser.
On Thu, Dec 12, 2013 at 4:24 PM, Pinak Pani < [email protected]> wrote: > I am trying to setup Spark with YARN 2.2.0. My Hadoop is plain Hadoop from > Apache Hadoop website. When I SBT build against 2.2.0 it fails. While it > compiles with a lot of warnings when I try against Hadoop 2.0.5-alpha. > > How can I compile Spark against YARN 2.2.0. > > There is a related thread here: > https://groups.google.com/forum/#!topic/spark-users/8Gm6ByvdNME > It did help. > > Also, I am a novice to SBT. > > Here is the error log: > > [root@ip-10-110-241-90 spark-0.8.0-incubating]# > SPARK_HADOOP_VERSION=2.2.0 SPARK_YARN=true sbt/sbt clean assembly > [info] Loading project definition from > /tmp/spark/spark-0.8.0-incubating/project/project > > [-- snip --] > > [warn] jobCommitter.cleanupJob(jobTaskContext) > [warn] ^ > [warn] > /tmp/spark/spark-0.8.0-incubating/core/src/main/scala/org/apache/spark/scheduler/InputFormatInfo.scala:98: > constructor Job in class Job is deprecated: see corresponding Javadoc for > more information. > [warn] val job = new Job(conf) > [warn] ^ > [warn] 9 warnings found > [warn] Note: Some input files use unchecked or unsafe operations. > [warn] Note: Recompile with -Xlint:unchecked for details. > [info] Compiling 8 Scala sources to > /tmp/spark/spark-0.8.0-incubating/yarn/target/scala-2.9.3/classes... > [info] Compiling 50 Scala sources to > /tmp/spark/spark-0.8.0-incubating/streaming/target/scala-2.9.3/classes... > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:42: > not found: type AMRMProtocol > [error] private var resourceManager: AMRMProtocol = null > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:126: > not found: type AMRMProtocol > [error] private def registerWithResourceManager(): AMRMProtocol = { > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:119: > value AM_CONTAINER_ID_ENV is not a member of object > org.apache.hadoop.yarn.api.ApplicationConstants > [error] val containerIdString = > envs.get(ApplicationConstants.AM_CONTAINER_ID_ENV) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:131: > not found: type AMRMProtocol > [error] return rpc.getProxy(classOf[AMRMProtocol], rmAddress, > conf).asInstanceOf[AMRMProtocol] > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:131: > not found: type AMRMProtocol > [error] return rpc.getProxy(classOf[AMRMProtocol], rmAddress, > conf).asInstanceOf[AMRMProtocol] > [error] > ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:138: > value setApplicationAttemptId is not a member of > org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest > [error] appMasterRequest.setApplicationAttemptId(appAttemptId) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:30: > AMRMProtocol is not a member of org.apache.hadoop.yarn.api > [error] import org.apache.hadoop.yarn.api.AMRMProtocol > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:504: > not found: type AMRMProtocol > [error] resourceManager: AMRMProtocol, appAttemptId: > ApplicationAttemptId, > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:24: > AMResponse is not a member of org.apache.hadoop.yarn.api.records > [error] import org.apache.hadoop.yarn.api.records.{AMResponse, > ApplicationAttemptId, ContainerId, Priority, Resource, ResourceRequest, > ContainerStatus, Container} > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:493: > not found: type AMRMProtocol > [error] resourceManager: AMRMProtocol, appAttemptId: > ApplicationAttemptId, > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:485: > not found: type AMRMProtocol > [error] resourceManager: AMRMProtocol, appAttemptId: > ApplicationAttemptId, > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:325: > value setAppAttemptId is not a member of > org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest > [error] finishReq.setAppAttemptId(appAttemptId) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala:326: > value setFinishApplicationStatus is not a member of > org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest > [error] finishReq.setFinishApplicationStatus(status) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:31: > YarnClientImpl is not a member of org.apache.hadoop.yarn.client > [error] import org.apache.hadoop.yarn.client.YarnClientImpl > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:42: > not found: type YarnClientImpl > [error] class Client(conf: Configuration, args: ClientArguments) extends > YarnClientImpl with Logging { > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:51: > not found: value init > [error] init(yarnConf) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:52: > not found: value start > [error] start() > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:55: > value getNewApplication is not a member of AnyRef with > org.apache.spark.Logging with ScalaObject > [error] val newApp = super.getNewApplication() > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:66: > value setUser is not a member of > org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext > [error] > appContext.setUser(UserGroupInformation.getCurrentUser().getShortUserName()) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:286: > value submitApplication is not a member of AnyRef with > org.apache.spark.Logging with ScalaObject > [error] super.submitApplication(appContext) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:76: > value getYarnClusterMetrics is not a member of AnyRef with > org.apache.spark.Logging with ScalaObject > [error] val clusterMetrics: YarnClusterMetrics = > super.getYarnClusterMetrics > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:79: > value getQueueInfo is not a member of AnyRef with org.apache.spark.Logging > with ScalaObject > [error] val queueInfo: QueueInfo = super.getQueueInfo(args.amQueue) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:216: > value getMinimumResourceCapability is not a member of > org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse > [error] val minResMemory: Int = > newApp.getMinimumResourceCapability().getMemory() > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:273: > value setResource is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] amContainer.setResource(capability) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:278: > value setContainerTokens is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] amContainer.setContainerTokens(ByteBuffer.wrap(dob.getData())) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala:292: > value getApplicationReport is not a member of AnyRef with > org.apache.spark.Logging with ScalaObject > [error] val report = super.getApplicationReport(appId) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:34: > ProtoUtils is not a member of org.apache.hadoop.yarn.util > [error] import org.apache.hadoop.yarn.util.{Apps, ConverterUtils, Records, > ProtoUtils} > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:48: > not found: type ContainerManager > [error] var cm: ContainerManager = null > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:202: > not found: type ContainerManager > [error] def connectToCM: ContainerManager = { > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:63: > value setContainerId is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] ctx.setContainerId(container.getId()) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:64: > value setResource is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] ctx.setResource(container.getResource()) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:103: > value setUser is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] > ctx.setUser(UserGroupInformation.getCurrentUser().getShortUserName()) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:108: > value setContainerTokens is not a member of > org.apache.hadoop.yarn.api.records.ContainerLaunchContext > [error] ctx.setContainerTokens(ByteBuffer.wrap(dob.getData())) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:212: > not found: value ProtoUtils > [error] > user.addToken(ProtoUtils.convertFromProtoFormat(containerToken, cmAddress)) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/WorkerRunnable.scala:216: > not found: type ContainerManager > [error] .doAs(new PrivilegedExceptionAction[ContainerManager] { > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:49: > not found: type AMRMProtocol > [error] private[yarn] class YarnAllocationHandler(val conf: Configuration, > val resourceManager: AMRMProtocol, > [error] > ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:87: > value getAMResponse is not a member of > org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse > [error] val amResp = > allocateWorkerResources(workersToRequest).getAMResponse > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:296: > value getHostName is not a member of > org.apache.hadoop.yarn.api.records.ResourceRequest > [error] val candidateHost = container.getHostName > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:374: > value setApplicationAttemptId is not a member of > org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest > [error] req.setApplicationAttemptId(appAttemptId) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:376: > value addAllAsks is not a member of > org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest > [error] req.addAllAsks(resourceRequests) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:379: > value addAllReleases is not a member of > org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest > [error] req.addAllReleases(releasedContainerList) > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:391: > value getHostName is not a member of > org.apache.hadoop.yarn.api.records.ResourceRequest > [error] logInfo("rsrcRequest ... host : " + req.getHostName + ", > numContainers : " + req.getNumContainers + > [error] ^ > [error] > /tmp/spark/spark-0.8.0-incubating/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala:441: > value setHostName is not a member of > org.apache.hadoop.yarn.api.records.ResourceRequest > [error] rsrcRequest.setHostName(hostname) > [error] ^ > [error] 43 errors found > > [-- snip --] > > [warn] Merging > 'org/yaml/snakeyaml/constructor/SafeConstructor$ConstructYamlBool.class' > with strategy 'first' > [warn] Merging > 'org/yaml/snakeyaml/emitter/Emitter$ExpectStreamStart.class' with strategy > 'first' > [warn] Strategy 'concat' was applied to 2 files > [warn] Strategy 'discard' was applied to 2 files > [warn] Strategy 'first' was applied to 794 files > [info] Checking every *.class/*.jar file's SHA-1. > [info] SHA-1: deebf2bd4f022965649cfe78d51ff1c8780c92a2 > [info] Packaging > /tmp/spark/spark-0.8.0-incubating/examples/target/scala-2.9.3/spark-examples-assembly-0.8.0-incubating.jar > ... > [info] Done packaging. > [error] (yarn/compile:compile) Compilation failed > [error] Total time: 668 s, completed 12 Dec, 2013 10:15:25 AM > > -- s
