[
https://issues.apache.org/jira/browse/HADOOP-13988?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15830995#comment-15830995
]
Greg Senia commented on HADOOP-13988:
-------------------------------------
yes its running in our cluster. Just put the newest patch out there here is log
output from DN getting the request from Knox:
2017-01-19 20:33:12,835 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logPrivilegedAction(1767)) - PrivilegedAction
as:gss2002 (auth:PROXY) via knox (auth:TOKEN)
from:org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.channelRead0(WebHdfsHandler.java:114)
2017-01-19 20:33:12,835 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logPrivilegedAction(1767)) - PrivilegedAction
as:gss2002 (auth:PROXY) via knox (auth:TOKEN)
from:org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.channelRead0(WebHdfsHandler.java:114)
2017-01-19 20:33:12,873 DEBUG security.SecurityUtil
(SecurityUtil.java:setTokenService(421)) - Acquired token Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.6:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:12,873 DEBUG security.SecurityUtil
(SecurityUtil.java:setTokenService(421)) - Acquired token Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.6:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:12,874 DEBUG security.SecurityUtil
(SecurityUtil.java:setTokenService(421)) - Acquired token Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.7:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:12,874 DEBUG security.SecurityUtil
(SecurityUtil.java:setTokenService(421)) - Acquired token Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.7:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,061 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logPrivilegedAction(1767)) - PrivilegedAction
as:knox (auth:TOKEN)
from:org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:758)
2017-01-19 20:33:13,061 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logPrivilegedAction(1767)) - PrivilegedAction
as:knox (auth:TOKEN)
from:org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:758)
2017-01-19 20:33:13,099 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1774)) - UGI: gss2002 (auth:PROXY)
via knox (auth:TOKEN)
2017-01-19 20:33:13,099 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1774)) - UGI: gss2002 (auth:PROXY)
via knox (auth:TOKEN)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1776)) - +RealUGI: knox (auth:TOKEN)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1776)) - +RealUGI: knox (auth:TOKEN)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1777)) - +RealUGI: shortName: knox
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1777)) - +RealUGI: shortName: knox
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1780)) - +LoginUGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1780)) - +LoginUGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1781)) - +LoginUGI shortName: hdfs
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1781)) - +LoginUGI shortName: hdfs
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1784)) - +UGI token:Kind:
HDFS_DELEGATION_TOKEN, Service: ha-hdfs:tech, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1784)) - +UGI token:Kind:
HDFS_DELEGATION_TOKEN, Service: ha-hdfs:tech, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1784)) - +UGI token:Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.7:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,100 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1784)) - +UGI token:Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.7:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1784)) - +UGI token:Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.6:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1784)) - +UGI token:Kind:
HDFS_DELEGATION_TOKEN, Service: 10.70.33.6:8020, Ident: (HDFS_DELEGATION_TOKEN
token 14666 for gss2002)
2017-01-19 20:33:13,101 DEBUG kms.KMSClientProvider
(KMSClientProvider.java:getActualUgi(1055)) - using RealUser for proxyUser
2017-01-19 20:33:13,101 DEBUG kms.KMSClientProvider
(KMSClientProvider.java:getActualUgi(1055)) - using RealUser for proxyUser
2017-01-19 20:33:13,101 DEBUG kms.KMSClientProvider
(KMSClientProvider.java:getActualUgi(1060)) - doAsUser exists
2017-01-19 20:33:13,101 DEBUG kms.KMSClientProvider
(KMSClientProvider.java:getActualUgi(1060)) - doAsUser exists
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1774)) - UGI: knox (auth:TOKEN)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1774)) - UGI: knox (auth:TOKEN)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1780)) - +LoginUGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1780)) - +LoginUGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1781)) - +LoginUGI shortName: hdfs
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1781)) - +LoginUGI shortName: hdfs
2017-01-19 20:33:13,101 DEBUG kms.KMSClientProvider
(KMSClientProvider.java:getActualUgi(1068)) - currentUGI.realUser does not
match UGI processUser
2017-01-19 20:33:13,101 DEBUG kms.KMSClientProvider
(KMSClientProvider.java:getActualUgi(1068)) - currentUGI.realUser does not
match UGI processUser
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1774)) - UGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1774)) - UGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1780)) - +LoginUGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,101 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1780)) - +LoginUGI:
dn/[email protected] (auth:KERBEROS)
2017-01-19 20:33:13,102 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1781)) - +LoginUGI shortName: hdfs
2017-01-19 20:33:13,102 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logAllUserInfo(1781)) - +LoginUGI shortName: hdfs
2017-01-19 20:33:13,102 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logPrivilegedAction(1767)) - PrivilegedAction
as:dn/[email protected] (auth:KERBEROS)
from:org.apache.hadoop.crypto.key.kms.KMSClientProvider.createConnection(KMSClientProvider.java:524)
2017-01-19 20:33:13,102 DEBUG security.UserGroupInformation
(UserGroupInformation.java:logPrivilegedAction(1767)) - PrivilegedAction
as:dn/[email protected] (auth:KERBEROS)
from:org.apache.hadoop.crypto.key.kms.KMSClientProvider.createConnection(KMSClientProvider.java:524)
2017-01-19 20:33:13,107 DEBUG security.UserGroupInformation
(UserGroupInformation.java:getTGT(898)) - Found tgt Ticket (hex) =
Client Principal = dn/[email protected]
Server Principal = krbtgt/[email protected]
Session Key = EncryptionKey: keyType=18 keyBytes (hex dump)=
Forwardable Ticket true
Forwarded Ticket false
Proxiable Ticket false
Proxy Ticket false
Postdated Ticket false
Renewable Ticket false
Initial Ticket false
Auth Time = Thu Jan 19 20:22:30 EST 2017
Start Time = Thu Jan 19 20:22:30 EST 2017
End Time = Fri Jan 20 06:22:30 EST 2017
Renew Till = null
Client Addresses Null
2017-01-19 20:33:13,107 DEBUG security.UserGroupInformation
(UserGroupInformation.java:getTGT(898)) - Found tgt Ticket (hex) =
Client Principal = dn/[email protected]
Server Principal = krbtgt/[email protected]
Session Key = EncryptionKey: keyType=18 keyBytes (hex dump)=
Forwardable Ticket true
Forwarded Ticket false
Proxiable Ticket false
Proxy Ticket false
Postdated Ticket false
Renewable Ticket false
Initial Ticket false
Auth Time = Thu Jan 19 20:22:30 EST 2017
Start Time = Thu Jan 19 20:22:30 EST 2017
End Time = Fri Jan 20 06:22:30 EST 2017
Renew Till = null
Client Addresses Null
2017-01-19 20:33:13,122 DEBUG client.KerberosAuthenticator
(KerberosAuthenticator.java:authenticate(192)) - JDK performed authentication
on our behalf.
2017-01-19 20:33:13,122 DEBUG client.KerberosAuthenticator
(KerberosAuthenticator.java:authenticate(192)) - JDK performed authentication
on our behalf.
2017-01-19 20:33:13,257 INFO DataNode.clienttrace
(DataXceiver.java:requestShortCircuitShm(468)) - cliID:
DFSClient_NONMAPREDUCE_513733485_146, src: 127.0.0.1, dest: 127.0.0.1, op:
REQUEST_SHORT_CIRCUIT_SHM, shmId: e7f6cfb0dd48d8112883cc97c9292c4d, srvID:
faca0b23-bfbe-413c-a2db-cc23c8817e87, success: true
2017-01-19 20:33:13,262 INFO DataNode.clienttrace
(DataXceiver.java:requestShortCircuitFds(369)) - src: 127.0.0.1, dest:
127.0.0.1, op: REQUEST_SHORT_CIRCUIT_FDS, blockid: 1073781194, srvID:
faca0b23-bfbe-413c-a2db-cc23c8817e87, success: true
> KMSClientProvider does not work with WebHDFS and Apache Knox w/ProxyUser
> ------------------------------------------------------------------------
>
> Key: HADOOP-13988
> URL: https://issues.apache.org/jira/browse/HADOOP-13988
> Project: Hadoop Common
> Issue Type: Bug
> Components: common, kms
> Affects Versions: 2.8.0, 2.7.3
> Environment: HDP 2.5.3.0
> WebHDFSUser --> Knox --> HA NameNodes(WebHDFS) --> DataNodes
> Reporter: Greg Senia
> Attachments: HADOOP-13988.patch, HADOOP-13988.patch
>
>
> After upgrading to HDP 2.5.3.0 noticed that all of the KMSClientProvider
> issues have not been resolved. We put a test build together and applied
> HADOOP-13558 and HADOOP-13749 these two fixes did still not solve the issue
> with requests coming from WebHDFS through to Knox to a TDE zone.
> So we added some debug to our build and determined effectively what is
> happening here is a double proxy situation which does not seem to work. So we
> propose the following fix in getActualUgi Method:
> {noformat}
> }
> // Use current user by default
> UserGroupInformation actualUgi = currentUgi;
> if (currentUgi.getRealUser() != null) {
> // Use real user for proxy user
> if (LOG.isDebugEnabled()) {
> LOG.debug("using RealUser for proxyUser);
> }
> actualUgi = currentUgi.getRealUser();
> if (getDoAsUser() != null) {
> if (LOG.isDebugEnabled()) {
> LOG.debug("doAsUser exists");
> LOG.debug("currentUGI realUser shortName: {}",
> currentUgi.getRealUser().getShortUserName());
> LOG.debug("processUGI loginUser shortName: {}",
> UserGroupInformation.getLoginUser().getShortUserName());
> }
> if (currentUgi.getRealUser().getShortUserName() !=
> UserGroupInformation.getLoginUser().getShortUserName()) {
> if (LOG.isDebugEnabled()) {
> LOG.debug("currentUGI.realUser does not match
> UGI.processUser);
> }
> actualUgi = UserGroupInformation.getLoginUser();
> if (LOG.isDebugEnabled()) {
> LOG.debug("LoginUser for Proxy: {}",
> actualUgi.getLoginUser());
> }
> }
> }
>
> } else if (!currentUgiContainsKmsDt() &&
> !currentUgi.hasKerberosCredentials()) {
> // Use login user for user that does not have either
> // Kerberos credential or KMS delegation token for KMS operations
> if (LOG.isDebugEnabled()) {
> LOG.debug("using loginUser no KMS Delegation Token no Kerberos
> Credentials");
> }
> actualUgi = currentUgi.getLoginUser();
> }
> return actualUgi;
> }
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
