[
https://issues.apache.org/jira/browse/HADOOP-12563?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Matthew Paduano updated HADOOP-12563:
-------------------------------------
Attachment: HADOOP-12563.15.patch
diff of patch 14,15
{code}
218c218
< + readProto(in);
---
> + readProtos(in);
333c333
< + public void readProto(DataInput in) throws IOException {
---
> + public void readProtos(DataInput in) throws IOException {
< public class TestCredentials {
...
< + @Test
< + public void testBasicReadWriteProtoEmpty()
< + throws IOException, NoSuchAlgorithmException {
< + String testname ="testBasicReadWriteProtoEmpty";
< + Credentials ts = new Credentials();
< + writeCredentialsProto(ts, testname);
< + Credentials ts2 = readCredentialsProto(testname);
< + assertEquals("test empty tokens", 0, ts2.numberOfTokens());
< + assertEquals("test empty keys", 0, ts2.numberOfSecretKeys());
< + }
< +
< + @Test
< + public void testBasicReadWriteProto()
< + throws IOException, NoSuchAlgorithmException {
< + String testname ="testBasicReadWriteProto";
< + Text tok1 = new Text("token1");
< + Text tok2 = new Text("token2");
< + Text key1 = new Text("key1");
< + Credentials ts = generateCredentials(tok1, tok2, key1);
< + writeCredentialsProto(ts, testname);
< + Credentials ts2 = readCredentialsProto(testname);
< + assertCredentials(testname, tok1, key1, ts, ts2);
< + assertCredentials(testname, tok2, key1, ts, ts2);
< + }
< +
< + @Test
< + public void testBasicReadWriteStreamEmpty()
< + throws IOException, NoSuchAlgorithmException {
< + String testname ="testBasicReadWriteStreamEmpty";
< + Credentials ts = new Credentials();
< + writeCredentialsStream(ts, testname);
< + Credentials ts2 = readCredentialsStream(testname);
< + assertEquals("test empty tokens", 0, ts2.numberOfTokens());
< + assertEquals("test empty keys", 0, ts2.numberOfSecretKeys());
< + }
< +
< + @Test
< + public void testBasicReadWriteStream()
< + throws IOException, NoSuchAlgorithmException {
< + String testname ="testBasicReadWriteStream";
< + Text tok1 = new Text("token1");
< + Text tok2 = new Text("token2");
< + Text key1 = new Text("key1");
< + Credentials ts = generateCredentials(tok1, tok2, key1);
< + writeCredentialsStream(ts, testname);
< + Credentials ts2 = readCredentialsStream(testname);
< + assertCredentials(testname, tok1, key1, ts, ts2);
< + assertCredentials(testname, tok2, key1, ts, ts2);
< + }
< +
< + @Test
< + /**
< + * Verify the suitability of read/writeProto for use with Writable interfac
< + * This test uses only empty credentials.
< + */
< + public void testWritablePropertiesEmpty()
< + throws IOException, NoSuchAlgorithmException {
< + String testname ="testWritablePropertiesEmpty";
< + Credentials ts = new Credentials();
< + Credentials ts2 = new Credentials();
< + writeCredentialsProtos(ts, ts2, testname);
< + List<Credentials> clist = readCredentialsProtos(testname);
< + assertEquals("test empty tokens 0", 0, clist.get(0).numberOfTokens());
< + assertEquals("test empty keys 0", 0, clist.get(0).numberOfSecretKeys());
< + assertEquals("test empty tokens 1", 0, clist.get(1).numberOfTokens());
< + assertEquals("test empty keys 1", 0, clist.get(1).numberOfSecretKeys());
< + }
< +
< + @Test
< + /**
< + * Verify the suitability of read/writeProto for use with Writable interfac
< + */
< + public void testWritableProperties()
< + throws IOException, NoSuchAlgorithmException {
< + String testname ="testWritableProperties";
< + Text tok1 = new Text("token1");
< + Text tok2 = new Text("token2");
< + Text key1 = new Text("key1");
< + Credentials ts = generateCredentials(tok1, tok2, key1);
< + Text tok3 = new Text("token3");
< + Text key2 = new Text("key2");
< + Credentials ts2 = generateCredentials(tok1, tok3, key2);
< + writeCredentialsProtos(ts, ts2, testname);
< + List<Credentials> clist = readCredentialsProtos(testname);
< + assertCredentials(testname, tok1, key1, ts, clist.get(0));
< + assertCredentials(testname, tok2, key1, ts, clist.get(0));
< + assertCredentials(testname, tok1, key2, ts2, clist.get(1));
< + assertCredentials(testname, tok3, key2, ts2, clist.get(1));
< + }
< +
< + private Credentials generateCredentials(Text t1, Text t2, Text t3)
< + throws NoSuchAlgorithmException {
< + Text kind = new Text("TESTTOK");
< + byte[] id1 = {0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72}
< + byte[] pass1 = {0x70, 0x61, 0x73, 0x73, 0x77, 0x6f, 0x72, 0x64};
< + byte[] id2 = {0x68, 0x63, 0x64, 0x6d, 0x73, 0x68, 0x65, 0x68, 0x64, 0x71}
< + byte[] pass2 = {0x6f, 0x60, 0x72, 0x72, 0x76, 0x6e, 0x71, 0x63};
< + Credentials ts = new Credentials();
< + generateToken(ts, id1, pass1, kind, t1);
< + generateToken(ts, id2, pass2, kind, t2);
< + generateKey(ts, t3);
< + return ts;
< + }
< +
< + private void assertCredentials(String tag, Text alias, Text keykey,
< + Credentials a, Credentials b) {
< + assertEquals(tag + ": test token count", a.numberOfTokens(),
< + b.numberOfTokens());
< + assertEquals(tag + ": test service", a.getToken(alias).getService(),
< + b.getToken(alias).getService());
< + assertEquals(tag + ": test kind", a.getToken(alias).getKind(),
< + b.getToken(alias).getKind());
< + assertTrue(tag + ": test password",
< + Arrays.equals(a.getToken(alias).getPassword(),
< + b.getToken(alias).getPassword()));
< + assertTrue(tag + ": test identifier",
< + Arrays.equals(a.getToken(alias).getIdentifier(),
< + b.getToken(alias).getIdentifier()));
< + assertEquals(tag + ": test number of keys", a.numberOfSecretKeys(),
< + b.numberOfSecretKeys());
< + assertTrue(tag + ":test key values", Arrays.equals(a.getSecretKey(keykey)
< + b.getSecretKey(keykey)
< + }
< +
< + private void writeCredentialsStream(Credentials creds, String filename)
< + throws IOException, FileNotFoundException {
< + DataOutputStream dos = new DataOutputStream(
< + new FileOutputStream(new File(tmpDir, filename)));
< + creds.writeTokenStorageToStream(dos);
< + }
< +
< + private Credentials readCredentialsStream(String filename)
< + throws IOException, FileNotFoundException {
< + Credentials creds = new Credentials();
< + DataInputStream dis = new DataInputStream(
< + new FileInputStream(new File(tmpDir, filename)));
< + creds.readTokenStorageStream(dis);
< + return creds;
< + }
< +
< + private void writeCredentialsProto(Credentials creds, String filename)
< + throws IOException, FileNotFoundException {
< + DataOutputStream dos = new DataOutputStream(
< + new FileOutputStream(new File(tmpDir, filename)));
< + creds.writeProto(dos);
< + }
< +
< + private Credentials readCredentialsProto(String filename)
< + throws IOException, FileNotFoundException {
< + Credentials creds = new Credentials();
< + DataInputStream dis = new DataInputStream(
< + new FileInputStream(new File(tmpDir, filename)));
< + creds.readProto(dis);
< + return creds;
< + }
< +
< + private void writeCredentialsProtos(Credentials c1, Credentials c2,
< + String filename) throws IOException, FileNotFoundException {
< + DataOutputStream dos = new DataOutputStream(
< + new FileOutputStream(new File(tmpDir, filename)));
< + c1.writeProto(dos);
< + c2.writeProto(dos);
< + }
< +
< + private List<Credentials> readCredentialsProtos(String filename)
< + throws IOException, FileNotFoundException {
< + Credentials c1 = new Credentials();
< + Credentials c2 = new Credentials();
< + DataInputStream dis = new DataInputStream(
< + new FileInputStream(new File(tmpDir, filename)));
< + c1.readProto(dis);
< + c2.readProto(dis);
< + List<Credentials> r = new ArrayList<Credentials>(2);
< + r.add(0, c1);
< + r.add(1, c2);
< + return r;
< + }
< +
< + private <T extends TokenIdentifier> void generateToken(
< + Credentials creds, byte[] ident, byte[] pass, Text kind, Text service)
< + Token<T> token = new Token(ident, pass, kind, service);
< + creds.addToken(service, token);
< + }
< +
< + private void generateKey(Credentials creds, Text alias)
< + throws NoSuchAlgorithmException {
< + final KeyGenerator kg = KeyGenerator.getInstance(DEFAULT_HMAC_ALGORITHM);
< + Key key = kg.generateKey();
< + creds.addSecretKey(alias, key.getEncoded());
< + }
< +
{code}
> Updated utility to create/modify token files
> --------------------------------------------
>
> Key: HADOOP-12563
> URL: https://issues.apache.org/jira/browse/HADOOP-12563
> Project: Hadoop Common
> Issue Type: New Feature
> Affects Versions: 3.0.0
> Reporter: Allen Wittenauer
> Assignee: Matthew Paduano
> Fix For: 3.0.0
>
> Attachments: HADOOP-12563.01.patch, HADOOP-12563.02.patch,
> HADOOP-12563.03.patch, HADOOP-12563.04.patch, HADOOP-12563.05.patch,
> HADOOP-12563.06.patch, HADOOP-12563.07.patch, HADOOP-12563.07.patch,
> HADOOP-12563.08.patch, HADOOP-12563.09.patch, HADOOP-12563.10.patch,
> HADOOP-12563.11.patch, HADOOP-12563.12.patch, HADOOP-12563.13.patch,
> HADOOP-12563.14.patch, HADOOP-12563.15.patch, dtutil-test-out,
> example_dtutil_commands_and_output.txt, generalized_token_case.pdf
>
>
> hdfs fetchdt is missing some critical features and is geared almost
> exclusively towards HDFS operations. Additionally, the token files that are
> created use Java serializations which are hard/impossible to deal with in
> other languages. It should be replaced with a better utility in common that
> can read/write protobuf-based token files, has enough flexibility to be used
> with other services, and offers key functionality such as append and rename.
> The old version file format should still be supported for backward
> compatibility, but will be effectively deprecated.
> A follow-on JIRA will deprecrate fetchdt.
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
