[
https://issues.apache.org/jira/browse/AMBARI-13601?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Andrew Onischuk updated AMBARI-13601:
-------------------------------------
Description:
This happens due to NN start doesn't create a bunch of directories which are
used by a bunch of service checks like /tmp /user/ambari-qa
Traceback (most recent call last):
File
"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py",
line 142, in <module>
ServiceCheck().execute()
File
"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py",
line 218, in execute
method(env)
File
"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py",
line 104, in service_check
user=params.smokeuser,
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 70,
in inner
result = function(command, **kwargs)
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 92,
in checked_call
tries=tries, try_sleep=try_sleep)
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 140,
in _call_wrapper
result = _call(command, **kwargs_copy)
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 291,
in _call
raise Fail(err_msg)
resource_management.core.exceptions.Fail: Execution of '/usr/bin/kinit -kt
/etc/security/keytabs/smokeuser.headless.keytab [email protected]; yarn
org.apache.hadoop.yarn.applications.distributedshell.Client -shell_command ls
-num_containers 1 -jar
/usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar'
returned 1. Hey guys
sup
15/10/28 12:22:01 INFO impl.TimelineClientImpl: Timeline service address:
http://c6402.org:8188/ws/v1/timeline/
15/10/28 12:22:02 INFO distributedshell.Client: Initializing Client
15/10/28 12:22:02 INFO distributedshell.Client: Running Client
15/10/28 12:22:02 INFO client.RMProxy: Connecting to ResourceManager at
c6402.org/192.168.64.102:8050
15/10/28 12:22:02 INFO distributedshell.Client: Got Cluster metric info
from ASM, numNodeManagers=1
15/10/28 12:22:03 INFO distributedshell.Client: Got Cluster node info from
ASM
15/10/28 12:22:03 INFO distributedshell.Client: Got node report from ASM
for, nodeId=c6402.org:45454, nodeAddressc6402.org:8042,
nodeRackName/default-rack, nodeNumContainers0
15/10/28 12:22:03 INFO distributedshell.Client: Queue info,
queueName=default, queueCurrentCapacity=0.0, queueMaxCapacity=1.0,
queueApplicationCount=0, queueChildQueueCount=0
15/10/28 12:22:03 INFO distributedshell.Client: User ACL Info for Queue,
queueName=root, userAcl=SUBMIT_APPLICATIONS
15/10/28 12:22:03 INFO distributedshell.Client: User ACL Info for Queue,
queueName=default, userAcl=SUBMIT_APPLICATIONS
15/10/28 12:22:03 INFO distributedshell.Client: Max mem capabililty of
resources in this cluster 1024
15/10/28 12:22:03 INFO distributedshell.Client: Max virtual cores
capabililty of resources in this cluster 3
15/10/28 12:22:03 INFO distributedshell.Client: Copy App Master jar from
local filesystem and add to local environment
15/10/28 12:22:04 FATAL distributedshell.Client: Error running Client
org.apache.hadoop.security.AccessControlException: Permission denied:
user=ambari-qa, access=WRITE,
inode="/user/ambari-qa/DistributedShell/application_1446034885508_0001/AppMaster.jar":hdfs:hdfs:drwxr-xr-x
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:292)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:213)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1755)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkAncestorAccess(FSDirectory.java:1738)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2509)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2444)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2328)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:624)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:397)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2137)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2133)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2131)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
at
org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1697)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1703)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1638)
at
org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:448)
at
org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:444)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:444)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:387)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:909)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:890)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:365)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:338)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1965)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1933)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1898)
at
org.apache.hadoop.yarn.applications.distributedshell.Client.addToLocalResources(Client.java:820)
at
org.apache.hadoop.yarn.applications.distributedshell.Client.run(Client.java:532)
at
org.apache.hadoop.yarn.applications.distributedshell.Client.main(Client.java:215)
Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.AccessControlException):
Permission denied: user=ambari-qa, access=WRITE,
inode="/user/ambari-qa/DistributedShell/application_1446034885508_0001/AppMaster.jar":hdfs:hdfs:drwxr-xr-x
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:292)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:213)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1755)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkAncestorAccess(FSDirectory.java:1738)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2509)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2444)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2328)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:624)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:397)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2137)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2133)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2131)
at org.apache.hadoop.ipc.Client.call(Client.java:1427)
at org.apache.hadoop.ipc.Client.call(Client.java:1358)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
at com.sun.proxy.$Proxy22.create(Unknown Source)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:296)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy23.create(Unknown Source)
at
org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1692)
... 18 more
was:
This happens due to NN start doesn't create a bunch of directories which are
used by a bunch of service checks like /tmp /ambari-qa
Traceback (most recent call last):
File
"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py",
line 142, in <module>
ServiceCheck().execute()
File
"/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py",
line 218, in execute
method(env)
File
"/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py",
line 104, in service_check
user=params.smokeuser,
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 70,
in inner
result = function(command, **kwargs)
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 92,
in checked_call
tries=tries, try_sleep=try_sleep)
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 140,
in _call_wrapper
result = _call(command, **kwargs_copy)
File
"/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line 291,
in _call
raise Fail(err_msg)
resource_management.core.exceptions.Fail: Execution of '/usr/bin/kinit -kt
/etc/security/keytabs/smokeuser.headless.keytab [email protected]; yarn
org.apache.hadoop.yarn.applications.distributedshell.Client -shell_command ls
-num_containers 1 -jar
/usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar'
returned 1. Hey guys
sup
15/10/28 12:22:01 INFO impl.TimelineClientImpl: Timeline service address:
http://c6402.org:8188/ws/v1/timeline/
15/10/28 12:22:02 INFO distributedshell.Client: Initializing Client
15/10/28 12:22:02 INFO distributedshell.Client: Running Client
15/10/28 12:22:02 INFO client.RMProxy: Connecting to ResourceManager at
c6402.org/192.168.64.102:8050
15/10/28 12:22:02 INFO distributedshell.Client: Got Cluster metric info
from ASM, numNodeManagers=1
15/10/28 12:22:03 INFO distributedshell.Client: Got Cluster node info from
ASM
15/10/28 12:22:03 INFO distributedshell.Client: Got node report from ASM
for, nodeId=c6402.org:45454, nodeAddressc6402.org:8042,
nodeRackName/default-rack, nodeNumContainers0
15/10/28 12:22:03 INFO distributedshell.Client: Queue info,
queueName=default, queueCurrentCapacity=0.0, queueMaxCapacity=1.0,
queueApplicationCount=0, queueChildQueueCount=0
15/10/28 12:22:03 INFO distributedshell.Client: User ACL Info for Queue,
queueName=root, userAcl=SUBMIT_APPLICATIONS
15/10/28 12:22:03 INFO distributedshell.Client: User ACL Info for Queue,
queueName=default, userAcl=SUBMIT_APPLICATIONS
15/10/28 12:22:03 INFO distributedshell.Client: Max mem capabililty of
resources in this cluster 1024
15/10/28 12:22:03 INFO distributedshell.Client: Max virtual cores
capabililty of resources in this cluster 3
15/10/28 12:22:03 INFO distributedshell.Client: Copy App Master jar from
local filesystem and add to local environment
15/10/28 12:22:04 FATAL distributedshell.Client: Error running Client
org.apache.hadoop.security.AccessControlException: Permission denied:
user=ambari-qa, access=WRITE,
inode="/user/ambari-qa/DistributedShell/application_1446034885508_0001/AppMaster.jar":hdfs:hdfs:drwxr-xr-x
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:292)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:213)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1755)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkAncestorAccess(FSDirectory.java:1738)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2509)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2444)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2328)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:624)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:397)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2137)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2133)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2131)
at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
at
sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
at
sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
at
org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
at
org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
at
org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1697)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1703)
at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1638)
at
org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:448)
at
org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:444)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:444)
at
org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:387)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:909)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:890)
at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:365)
at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:338)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1965)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1933)
at
org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1898)
at
org.apache.hadoop.yarn.applications.distributedshell.Client.addToLocalResources(Client.java:820)
at
org.apache.hadoop.yarn.applications.distributedshell.Client.run(Client.java:532)
at
org.apache.hadoop.yarn.applications.distributedshell.Client.main(Client.java:215)
Caused by:
org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.AccessControlException):
Permission denied: user=ambari-qa, access=WRITE,
inode="/user/ambari-qa/DistributedShell/application_1446034885508_0001/AppMaster.jar":hdfs:hdfs:drwxr-xr-x
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:292)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:213)
at
org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1755)
at
org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkAncestorAccess(FSDirectory.java:1738)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2509)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2444)
at
org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2328)
at
org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:624)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:397)
at
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2137)
at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2133)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Subject.java:422)
at
org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2131)
at org.apache.hadoop.ipc.Client.call(Client.java:1427)
at org.apache.hadoop.ipc.Client.call(Client.java:1358)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
at com.sun.proxy.$Proxy22.create(Unknown Source)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:296)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:497)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy23.create(Unknown Source)
at
org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1692)
... 18 more
> Yarn service check and some other fail (unless NN HA is enabled)
> ----------------------------------------------------------------
>
> Key: AMBARI-13601
> URL: https://issues.apache.org/jira/browse/AMBARI-13601
> Project: Ambari
> Issue Type: Bug
> Reporter: Andrew Onischuk
> Assignee: Andrew Onischuk
> Fix For: 2.1.3
>
>
> This happens due to NN start doesn't create a bunch of directories which are
> used by a bunch of service checks like /tmp /user/ambari-qa
>
>
> Traceback (most recent call last):
> File
> "/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py",
> line 142, in <module>
> ServiceCheck().execute()
> File
> "/usr/lib/python2.6/site-packages/resource_management/libraries/script/script.py",
> line 218, in execute
> method(env)
> File
> "/var/lib/ambari-agent/cache/common-services/YARN/2.1.0.2.0/package/scripts/service_check.py",
> line 104, in service_check
> user=params.smokeuser,
> File
> "/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line
> 70, in inner
> result = function(command, **kwargs)
> File
> "/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line
> 92, in checked_call
> tries=tries, try_sleep=try_sleep)
> File
> "/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line
> 140, in _call_wrapper
> result = _call(command, **kwargs_copy)
> File
> "/usr/lib/python2.6/site-packages/resource_management/core/shell.py", line
> 291, in _call
> raise Fail(err_msg)
> resource_management.core.exceptions.Fail: Execution of '/usr/bin/kinit
> -kt /etc/security/keytabs/smokeuser.headless.keytab [email protected];
> yarn org.apache.hadoop.yarn.applications.distributedshell.Client
> -shell_command ls -num_containers 1 -jar
> /usr/hdp/current/hadoop-yarn-client/hadoop-yarn-applications-distributedshell.jar'
> returned 1. Hey guys
> sup
> 15/10/28 12:22:01 INFO impl.TimelineClientImpl: Timeline service address:
> http://c6402.org:8188/ws/v1/timeline/
> 15/10/28 12:22:02 INFO distributedshell.Client: Initializing Client
> 15/10/28 12:22:02 INFO distributedshell.Client: Running Client
> 15/10/28 12:22:02 INFO client.RMProxy: Connecting to ResourceManager at
> c6402.org/192.168.64.102:8050
> 15/10/28 12:22:02 INFO distributedshell.Client: Got Cluster metric info
> from ASM, numNodeManagers=1
> 15/10/28 12:22:03 INFO distributedshell.Client: Got Cluster node info
> from ASM
> 15/10/28 12:22:03 INFO distributedshell.Client: Got node report from ASM
> for, nodeId=c6402.org:45454, nodeAddressc6402.org:8042,
> nodeRackName/default-rack, nodeNumContainers0
> 15/10/28 12:22:03 INFO distributedshell.Client: Queue info,
> queueName=default, queueCurrentCapacity=0.0, queueMaxCapacity=1.0,
> queueApplicationCount=0, queueChildQueueCount=0
> 15/10/28 12:22:03 INFO distributedshell.Client: User ACL Info for Queue,
> queueName=root, userAcl=SUBMIT_APPLICATIONS
> 15/10/28 12:22:03 INFO distributedshell.Client: User ACL Info for Queue,
> queueName=default, userAcl=SUBMIT_APPLICATIONS
> 15/10/28 12:22:03 INFO distributedshell.Client: Max mem capabililty of
> resources in this cluster 1024
> 15/10/28 12:22:03 INFO distributedshell.Client: Max virtual cores
> capabililty of resources in this cluster 3
> 15/10/28 12:22:03 INFO distributedshell.Client: Copy App Master jar from
> local filesystem and add to local environment
> 15/10/28 12:22:04 FATAL distributedshell.Client: Error running Client
> org.apache.hadoop.security.AccessControlException: Permission denied:
> user=ambari-qa, access=WRITE,
> inode="/user/ambari-qa/DistributedShell/application_1446034885508_0001/AppMaster.jar":hdfs:hdfs:drwxr-xr-x
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319)
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:292)
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:213)
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1755)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkAncestorAccess(FSDirectory.java:1738)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2509)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2444)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2328)
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:624)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:397)
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2137)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2133)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2131)
>
> at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
> at
> sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
> at
> sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
> at java.lang.reflect.Constructor.newInstance(Constructor.java:422)
> at
> org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:106)
> at
> org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:73)
> at
> org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1697)
> at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1703)
> at org.apache.hadoop.hdfs.DFSClient.create(DFSClient.java:1638)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:448)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem$7.doCall(DistributedFileSystem.java:444)
> at
> org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:444)
> at
> org.apache.hadoop.hdfs.DistributedFileSystem.create(DistributedFileSystem.java:387)
> at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:909)
> at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:890)
> at org.apache.hadoop.fs.FileSystem.create(FileSystem.java:787)
> at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:365)
> at org.apache.hadoop.fs.FileUtil.copy(FileUtil.java:338)
> at
> org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1965)
> at
> org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1933)
> at
> org.apache.hadoop.fs.FileSystem.copyFromLocalFile(FileSystem.java:1898)
> at
> org.apache.hadoop.yarn.applications.distributedshell.Client.addToLocalResources(Client.java:820)
> at
> org.apache.hadoop.yarn.applications.distributedshell.Client.run(Client.java:532)
> at
> org.apache.hadoop.yarn.applications.distributedshell.Client.main(Client.java:215)
> Caused by:
> org.apache.hadoop.ipc.RemoteException(org.apache.hadoop.security.AccessControlException):
> Permission denied: user=ambari-qa, access=WRITE,
> inode="/user/ambari-qa/DistributedShell/application_1446034885508_0001/AppMaster.jar":hdfs:hdfs:drwxr-xr-x
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:319)
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.check(FSPermissionChecker.java:292)
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:213)
> at
> org.apache.hadoop.hdfs.server.namenode.FSPermissionChecker.checkPermission(FSPermissionChecker.java:190)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1771)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkPermission(FSDirectory.java:1755)
> at
> org.apache.hadoop.hdfs.server.namenode.FSDirectory.checkAncestorAccess(FSDirectory.java:1738)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInternal(FSNamesystem.java:2509)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFileInt(FSNamesystem.java:2444)
> at
> org.apache.hadoop.hdfs.server.namenode.FSNamesystem.startFile(FSNamesystem.java:2328)
> at
> org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.create(NameNodeRpcServer.java:624)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.create(ClientNamenodeProtocolServerSideTranslatorPB.java:397)
> at
> org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:616)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:969)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2137)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2133)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:422)
> at
> org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1657)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2131)
>
> at org.apache.hadoop.ipc.Client.call(Client.java:1427)
> at org.apache.hadoop.ipc.Client.call(Client.java:1358)
> at
> org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:229)
> at com.sun.proxy.$Proxy22.create(Unknown Source)
> at
> org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.create(ClientNamenodeProtocolTranslatorPB.java:296)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:497)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
> at
> org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
> at com.sun.proxy.$Proxy23.create(Unknown Source)
> at
> org.apache.hadoop.hdfs.DFSOutputStream.newStreamForCreate(DFSOutputStream.java:1692)
> ... 18 more
>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
