Author: acmurthy
Date: Sat Feb 27 03:07:08 2010
New Revision: 916902
URL: http://svn.apache.org/viewvc?rev=916902&view=rev
Log:
HDFS-994. Allow fetching of delegation token from NameNode for hftp.
Contributed by Jakob Homan.
Added:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DelegationTokenServlet.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=916902&r1=916901&r2=916902&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Sat Feb 27 03:07:08 2010
@@ -80,6 +80,9 @@
HDFS-991. Allow authentication to the web ui via a delegation token.
(omalley)
+ HDFS-994. Allow fetching of delegation token from NameNode for hftp.
+ (Jakob Homan via acmurthy)
+
OPTIMIZATIONS
HDFS-946. NameNode should not return full path name when lisitng a
Added:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DelegationTokenServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DelegationTokenServlet.java?rev=916902&view=auto
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DelegationTokenServlet.java
(added)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/DelegationTokenServlet.java
Sat Feb 27 03:07:08 2010
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
under
+ * the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+
+import javax.servlet.ServletContext;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.TokenStorage;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+
+/**
+ * Serve delegation tokens over http for use in hftp.
+ */
+...@suppresswarnings("serial")
+public class DelegationTokenServlet extends DfsServlet {
+ private static final Log LOG =
LogFactory.getLog(DelegationTokenServlet.class);
+ public static final String PATH_SPEC = "/getDelegationToken";
+
+ @Override
+ protected void doGet(final HttpServletRequest req, final HttpServletResponse
resp)
+ throws ServletException, IOException {
+ final UserGroupInformation ugi;
+ try {
+ ugi = getUGI(req, new Configuration());
+ } catch(IOException ioe) {
+ LOG.info("Request for token received with no authentication from "
+ + req.getRemoteAddr(), ioe);
+ resp.sendError(HttpServletResponse.SC_FORBIDDEN,
+ "Unable to identify or authenticate user");
+ return;
+ }
+ LOG.info("Sending token: {" + ugi.getUserName() + "," +
req.getRemoteAddr() +"}");
+ final ServletContext context = getServletContext();
+ final NameNode nn = (NameNode) context.getAttribute("name.node");
+
+ DataOutputStream dos = null;
+ try {
+ dos = new DataOutputStream(resp.getOutputStream());
+ final DataOutputStream dosFinal = dos; // for doAs block
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+
+ Token<DelegationTokenIdentifier> token =
+ nn.getDelegationToken(new Text(req.getUserPrincipal().getName()));
+ String s = nn.rpcAddress.getAddress().getHostAddress()
+ + ":" + nn.rpcAddress.getPort();
+ token.setService(new Text(s));
+ TokenStorage ts = new TokenStorage();
+ ts.addToken(new Text(ugi.getShortUserName()), token);
+ ts.write(dosFinal);
+ dosFinal.close();
+ return null;
+ }
+ });
+
+ } catch(Exception e) {
+ LOG.info("Exception while sending token. Re-throwing. ", e);
+ resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ } finally {
+ if(dos != null) dos.close();
+ }
+ }
+}
Modified:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=916902&r1=916901&r2=916902&view=diff
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
(original)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java
Sat Feb 27 03:07:08 2010
@@ -368,6 +368,8 @@
this.httpServer.setAttribute("name.node.address", getNameNodeAddress());
this.httpServer.setAttribute("name.system.image", getFSImage());
this.httpServer.setAttribute("name.conf", conf);
+ this.httpServer.addInternalServlet("getDelegationToken",
+ DelegationTokenServlet.PATH_SPEC, DelegationTokenServlet.class);
this.httpServer.addInternalServlet("fsck", "/fsck", FsckServlet.class);
this.httpServer.addInternalServlet("getimage", "/getimage",
GetImageServlet.class);
this.httpServer.addInternalServlet("listPaths", "/listPaths/*",
ListPathsServlet.class);
Added:
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=916902&view=auto
==============================================================================
---
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
(added)
+++
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
Sat Feb 27 03:07:08 2010
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.InetAddress;
+import java.net.URL;
+import java.net.URLConnection;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.namenode.DelegationTokenServlet;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.TokenStorage;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.StringUtils;
+
+/**
+ * Fetch a DelegationToken from the current Namenode and store it in the
+ * specified file.
+ */
+public class DelegationTokenFetcher {
+ private static final String USAGE =
+ "fetchdt retrieves delegation tokens (optionally over http)\n" +
+ "and writes them to specified file.\n" +
+ "Usage: fetchdt [--webservice <namenode http addr>] <output filename>";
+
+ private final DistributedFileSystem dfs;
+ private final UserGroupInformation ugi;
+ private final DataOutputStream out;
+
+ /**
+ * Command-line interface
+ */
+ public static void main(final String [] args) throws Exception {
+ // Login the current user
+ UserGroupInformation.getCurrentUser().doAs(new
PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+
+ if(args.length == 3 && "--webservice".equals(args[0])) {
+ getDTfromRemote(args[1], args[2]);
+ return null;
+ }
+ // avoid annoying mistake
+ if(args.length == 1 && "--webservice".equals(args[0])) {
+ System.out.println(USAGE);
+ return null;
+ }
+ if(args.length != 1 || args[0].isEmpty()) {
+ System.out.println(USAGE);
+ return null;
+ }
+
+ DataOutputStream out = null;
+
+ try {
+ Configuration conf = new HdfsConfiguration();
+ DistributedFileSystem dfs = (DistributedFileSystem)
FileSystem.get(conf);
+ out = new DataOutputStream(new FileOutputStream(args[0]));
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
+ new DelegationTokenFetcher(dfs, out, ugi).go();
+
+ out.flush();
+ System.out.println("Succesfully wrote token of size " +
+ out.size() + " bytes to "+ args[0]);
+ } catch (IOException ioe) {
+ System.out.println("Exception encountered:\n" +
+ StringUtils.stringifyException(ioe));
+ } finally {
+ if(out != null) out.close();
+ }
+ return null;
+ }
+ });
+
+ }
+
+ public DelegationTokenFetcher(DistributedFileSystem dfs,
+ DataOutputStream out, UserGroupInformation ugi) {
+ checkNotNull("dfs", dfs); this.dfs = dfs;
+ checkNotNull("out", out); this.out = out;
+ checkNotNull("ugi", ugi); this.ugi = ugi;
+ }
+
+ private void checkNotNull(String s, Object o) {
+ if(o == null) throw new IllegalArgumentException(s + " cannot be null.");
+ }
+
+ public void go() throws IOException {
+ String fullName = ugi.getUserName();
+ String shortName = ugi.getShortUserName();
+ Token<DelegationTokenIdentifier> token =
+ dfs.getDelegationToken(new Text(fullName));
+
+ // Reconstruct the ip:port of the Namenode
+ String nnAddress =
+ InetAddress.getByName(dfs.getUri().getHost()).getHostAddress()
+ + ":" + dfs.getUri().getPort();
+ token.setService(new Text(nnAddress));
+
+ TokenStorage ts = new TokenStorage();
+ ts.addToken(new Text(shortName), token);
+ ts.write(out);
+ }
+
+ /**
+ * Utility method to obtain a delegation token over http
+ * @param nnHttpAddr Namenode http addr, such as http://namenode:50070
+ * @param filename Name of file to store token in
+ */
+ static private void getDTfromRemote(String nnAddr, String filename)
+ throws IOException {
+ // Enable Kerberos sockets
+ System.setProperty("https.cipherSuites", "TLS_KRB5_WITH_3DES_EDE_CBC_SHA");
+ String ugiPostfix = "";
+ DataOutputStream file = null;
+ DataInputStream dis = null;
+
+ if(nnAddr.startsWith("http:"))
+ ugiPostfix = "?ugi=" +
UserGroupInformation.getCurrentUser().getShortUserName();
+
+ try {
+ System.out.println("Retrieving token from: " +
+ nnAddr + DelegationTokenServlet.PATH_SPEC + ugiPostfix);
+ URL remoteURL = new URL(nnAddr + DelegationTokenServlet.PATH_SPEC +
ugiPostfix);
+ URLConnection connection = remoteURL.openConnection();
+
+ InputStream in = connection.getInputStream();
+ TokenStorage ts = new TokenStorage();
+ dis = new DataInputStream(in);
+ ts.readFields(dis);
+ file = new DataOutputStream(new FileOutputStream(filename));
+ ts.write(file);
+ file.flush();
+ System.out.println("Successfully wrote token of " + file.size()
+ + " bytes to " + filename);
+ } catch (Exception e) {
+ throw new IOException("Unable to obtain remote token", e);
+ } finally {
+ if(dis != null) dis.close();
+ if(file != null) file.close();
+ }
+ }
+}
Added:
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
URL:
http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java?rev=916902&view=auto
==============================================================================
---
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
(added)
+++
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/tools/TestDelegationTokenFetcher.java
Sat Feb 27 03:07:08 2010
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.tools;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.net.URI;
+
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.tools.DelegationTokenFetcher;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.TokenStorage;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestDelegationTokenFetcher {
+ private DistributedFileSystem dfs;
+ private DataOutputStream out;
+ private UserGroupInformation ugi;
+
+ @Before
+ public void init() {
+ dfs = mock(DistributedFileSystem.class);
+ out = mock(DataOutputStream.class);
+ ugi = mock(UserGroupInformation.class);
+ }
+
+ /**
+ * Verify that when the DelegationTokenFetcher runs, it talks to the
Namenode,
+ * pulls out the correct user's token and successfully serializes it to disk.
+ */
+ @Test
+ public void expectedTokenIsRetrievedFromDFS() throws Exception {
+ final String LONG_NAME = "thedoc...@tardis";
+ final String SHORT_NAME = "TheDoctor";
+ final String SERVICE_VALUE = "localhost:2005";
+
+ // Mock out the user's long and short names.
+ when(ugi.getUserName()).thenReturn(LONG_NAME);
+ when(ugi.getShortUserName()).thenReturn(SHORT_NAME);
+
+ // Create a token for the fetcher to fetch, wire NN to return it when asked
+ // for this particular user.
+ Token<DelegationTokenIdentifier> t = new
Token<DelegationTokenIdentifier>();
+ when(dfs.getDelegationToken(eq(new Text(LONG_NAME)))).thenReturn(t);
+
+ // Mock the NN's URI, which is stored as the service value
+ URI uri = new URI("hdfs://" + SERVICE_VALUE);
+ when(dfs.getUri()).thenReturn(uri);
+
+ // Now, actually let the TokenFetcher go fetch the token.
+ final ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ out = new DataOutputStream(baos);
+ new DelegationTokenFetcher(dfs, out, ugi).go();
+
+ // now read the data back in and verify correct values
+ TokenStorage ts = new TokenStorage();
+ DataInputStream dis =
+ new DataInputStream(new ByteArrayInputStream(baos.toByteArray()));
+ ts.readFields(dis);
+ Token<? extends TokenIdentifier> newToken = ts.getToken(new
Text(SHORT_NAME));
+
+ assertEquals("Should only be one token in storage", ts.numberOfTokens(),
1);
+ assertEquals("Service value should have survived",
+ "127.0.0.1:2005", newToken.getService().toString());
+ }
+
+ private void checkWithNullParam(String s) {
+ try {
+ new DelegationTokenFetcher(dfs, out, ugi);
+ } catch (IllegalArgumentException iae) {
+ assertEquals("Expected exception message not received",
+ s + " cannot be null.", iae.getMessage());
+ return; // received expected exception. We're good.
+ }
+ fail("null parameter should have failed.");
+ }
+
+ @Test
+ public void dfsCannotBeNull() {
+ dfs = null;
+ String s = "dfs";
+ checkWithNullParam(s);
+ }
+
+ @Test
+ public void dosCannotBeNull() {
+ out = null;
+ String s = "out";
+ checkWithNullParam(s);
+ }
+
+ @Test
+ public void ugiCannotBeNull() {
+ ugi = null;
+ String s = "ugi";
+ checkWithNullParam(s);
+ }
+}
