Author: omalley
Date: Fri Mar 4 03:45:07 2011
New Revision: 1077139
URL: http://svn.apache.org/viewvc?rev=1077139&view=rev
Log:
commit 0f7a852cc25e9e49753d15d3f3d9af6ab4162ac3
Author: Jitendra Nath Pandey <[email protected]>
Date: Mon Feb 1 10:28:11 2010 -0800
HADOOP-6520 from
https://issues.apache.org/jira/secure/attachment/12434423/HADOOP-6520-0_20.2.patch
+++ b/YAHOO-CHANGES.txt
+ HADOOP-6520. UGI should load tokens from the environment. (jitendra)
+
Added:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
Added:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java?rev=1077139&view=auto
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
(added)
+++
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/TokenStorage.java
Fri Mar 4 03:45:07 2011
@@ -0,0 +1,174 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * A class that provides the facilities of reading and writing
+ * secret keys and Tokens.
+ */
+public class TokenStorage implements Writable {
+
+ private Map<Text, byte[]> secretKeysMap = new HashMap<Text, byte[]>();
+ private Map<Text, Token<? extends TokenIdentifier>> tokenMap =
+ new HashMap<Text, Token<? extends TokenIdentifier>>();
+
+ /**
+ * Returns the key bytes for the alias
+ * @param alias the alias for the key
+ * @return key for this alias
+ */
+ public byte[] getSecretKey(Text alias) {
+ return secretKeysMap.get(alias);
+ }
+
+ /**
+ * Returns the Token object for the alias
+ * @param alias the alias for the Token
+ * @return token for this alias
+ */
+ public Token<? extends TokenIdentifier> getToken(Text alias) {
+ return tokenMap.get(alias);
+ }
+
+ /**
+ * Add a token in the storage (in memory)
+ * @param alias the alias for the key
+ * @param t the token object
+ */
+ public void addToken(Text alias, Token<? extends TokenIdentifier> t) {
+ tokenMap.put(alias, t);
+ }
+
+ /**
+ * Return all the tokens in the in-memory map
+ */
+ public Collection<Token<? extends TokenIdentifier>> getAllTokens() {
+ return tokenMap.values();
+ }
+
+ /**
+ * @return number of Tokens in the in-memory map
+ */
+ public int numberOfTokens() {
+ return tokenMap.size();
+ }
+
+ /**
+ * @return number of keys in the in-memory map
+ */
+ public int numberOfSecretKeys() {
+ return secretKeysMap.size();
+ }
+
+ /**
+ * Set the key for an alias
+ * @param alias the alias for the key
+ * @param key the key bytes
+ */
+ public void addSecretKey(Text alias, byte[] key) {
+ secretKeysMap.put(alias, key);
+ }
+
+ /**
+ * Convenience method for reading a file, and loading the Tokens
+ * therein in the passed UGI
+ * @param filename
+ * @param conf
+ * @param ugi
+ * @throws IOException
+ */
+ public static void readTokensAndLoadInUGI(String filename, Configuration
conf,
+ UserGroupInformation ugi) throws IOException {
+ Path localTokensFile = new Path (filename);
+ FileSystem localFS = FileSystem.getLocal(conf);
+ FSDataInputStream in = localFS.open(localTokensFile);
+ TokenStorage ts = new TokenStorage();
+ ts.readFields(in);
+ for (Token<? extends TokenIdentifier> token : ts.getAllTokens()) {
+ ugi.addToken(token);
+ }
+ }
+ /**
+ * Stores all the keys to DataOutput
+ * @param out
+ * @throws IOException
+ */
+ @Override
+ public void write(DataOutput out) throws IOException {
+ // write out tokens first
+ WritableUtils.writeVInt(out, tokenMap.size());
+ for(Map.Entry<Text,
+ Token<? extends TokenIdentifier>> e: tokenMap.entrySet()) {
+ e.getKey().write(out);
+ e.getValue().write(out);
+ }
+
+ // now write out secret keys
+ WritableUtils.writeVInt(out, secretKeysMap.size());
+ for(Map.Entry<Text, byte[]> e : secretKeysMap.entrySet()) {
+ e.getKey().write(out);
+ WritableUtils.writeCompressedByteArray(out, e.getValue());
+ }
+ }
+
+ /**
+ * Loads all the keys
+ * @param in
+ * @throws IOException
+ */
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ secretKeysMap.clear();
+ tokenMap.clear();
+
+ int size = WritableUtils.readVInt(in);
+ for(int i=0; i<size; i++) {
+ Text alias = new Text();
+ alias.readFields(in);
+ Token<? extends TokenIdentifier> t = new Token<TokenIdentifier>();
+ t.readFields(in);
+ tokenMap.put(alias, t);
+ }
+
+ size = WritableUtils.readVInt(in);
+ for(int i=0; i<size; i++) {
+ Text alias = new Text();
+ alias.readFields(in);
+ byte[] key = WritableUtils.readCompressedByteArray(in);
+ secretKeysMap.put(alias, key);
+ }
+ }
+}
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java?rev=1077139&r1=1077138&r2=1077139&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/core/org/apache/hadoop/security/UserGroupInformation.java
Fri Mar 4 03:45:07 2011
@@ -30,7 +30,6 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -127,6 +126,10 @@ public class UserGroupInformation {
/** Server-side groups fetching service */
private static Groups groups;
+ /**Environment variable pointing to the token cache file*/
+ public static final String HADOOP_TOKEN_FILE_LOCATION =
+ "HADOOP_TOKEN_FILE_LOCATION";
+
/**
* A method to initialize the fields that depend on a configuration.
* Must be called before useKerberos or groups is used.
@@ -316,6 +319,10 @@ public class UserGroupInformation {
}
login.login();
loginUser = new UserGroupInformation(login.getSubject());
+ String tokenFile = System.getenv(HADOOP_TOKEN_FILE_LOCATION);
+ if (tokenFile != null && isSecurityEnabled()) {
+ TokenStorage.readTokensAndLoadInUGI(tokenFile, new Configuration(),
loginUser);
+ }
} catch (LoginException le) {
throw new IOException("failure to login", le);
}
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java?rev=1077139&r1=1077138&r2=1077139&view=diff
==============================================================================
---
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
(original)
+++
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/security/TestTokenStorage.java
Fri Mar 4 03:45:07 2011
@@ -29,19 +29,23 @@ import java.io.IOException;
import java.security.Key;
import java.security.NoSuchAlgorithmException;
import java.util.HashMap;
+import java.util.List;
+import java.util.ArrayList;
import java.util.Map;
+import java.util.Collection;
+
+import static org.mockito.Mockito.mock;
import javax.crypto.KeyGenerator;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
-import org.apache.hadoop.mapreduce.security.token.JobTokenSecretManager;
-import org.apache.hadoop.record.Utils;
+import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.security.TokenStorage;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.*;
-import org.apache.hadoop.mapreduce.security.TokenStorage;
public class TestTokenStorage {
private static final String DEFAULT_HMAC_ALGORITHM = "HmacSHA1";
@@ -52,19 +56,27 @@ public class TestTokenStorage {
public void setUp() {
tmpDir.mkdir();
}
-
+
+ @SuppressWarnings("unchecked")
@Test
- public void testReadWriteStorage() throws IOException,
NoSuchAlgorithmException{
+ public <T extends TokenIdentifier> void testReadWriteStorage()
+ throws IOException, NoSuchAlgorithmException{
// create tokenStorage Object
TokenStorage ts = new TokenStorage();
- // create a token
- JobTokenSecretManager jtSecretManager = new JobTokenSecretManager();
- JobTokenIdentifier identifier = new JobTokenIdentifier(new
Text("fakeJobId"));
- Token<JobTokenIdentifier> jt = new Token<JobTokenIdentifier>(identifier,
- jtSecretManager);
- // store it
- ts.setJobToken(jt);
+ Token<T> token1 = new Token();
+ Token<T> token2 = new Token();
+ Text service1 = new Text("service1");
+ Text service2 = new Text("service2");
+ Collection<Text> services = new ArrayList<Text>();
+
+ services.add(service1);
+ services.add(service2);
+
+ token1.setService(service1);
+ token2.setService(service2);
+ ts.addToken(new Text("sometoken1"), token1);
+ ts.addToken(new Text("sometoken2"), token2);
// create keys and put it in
final KeyGenerator kg = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM);
@@ -78,33 +90,44 @@ public class TestTokenStorage {
// create file to store
File tmpFileName = new File(tmpDir, "tokenStorageTest");
- DataOutputStream dos = new DataOutputStream(new
FileOutputStream(tmpFileName));
+ DataOutputStream dos =
+ new DataOutputStream(new FileOutputStream(tmpFileName));
ts.write(dos);
dos.close();
// open and read it back
- DataInputStream dis = new DataInputStream(new
FileInputStream(tmpFileName));
+ DataInputStream dis =
+ new DataInputStream(new FileInputStream(tmpFileName));
ts = new TokenStorage();
ts.readFields(dis);
dis.close();
- // get the token and compare the passwords
- byte[] tp1 = ts.getJobToken().getPassword();
- byte[] tp2 = jt.getPassword();
- int comp = Utils.compareBytes(tp1, 0, tp1.length, tp2, 0, tp2.length);
- assertTrue("shuffleToken doesn't match", comp==0);
-
+ // get the tokens and compare the services
+ Collection<Token<? extends TokenIdentifier>> list = ts.getAllTokens();
+ assertEquals("getAllTokens should return collection of size 2",
+ list.size(), 2);
+ boolean foundFirst = false;
+ boolean foundSecond = false;
+ for (Token<? extends TokenIdentifier> token : list) {
+ if (token.getService().equals(service1)) {
+ foundFirst = true;
+ }
+ if (token.getService().equals(service2)) {
+ foundSecond = true;
+ }
+ }
+ assertTrue("Tokens for services service1 and service2 must be present",
+ foundFirst && foundSecond);
// compare secret keys
int mapLen = m.size();
- assertEquals("wrong number of keys in the Storage", mapLen,
ts.numberOfSecretKeys());
+ assertEquals("wrong number of keys in the Storage",
+ mapLen, ts.numberOfSecretKeys());
for(Text a : m.keySet()) {
byte [] kTS = ts.getSecretKey(a);
byte [] kLocal = m.get(a);
assertTrue("keys don't match for " + a,
- Utils.compareBytes(kTS, 0, kTS.length, kLocal, 0, kLocal.length)==0);
- }
-
- assertEquals("All tokens should return collection of size 1",
- ts.getAllTokens().size(), 1);
+ WritableComparator.compareBytes(kTS, 0, kTS.length, kLocal,
+ 0, kLocal.length)==0);
+ }
}
}
