Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm Thu Aug 7 07:38:23 2014 @@ -296,9 +296,24 @@ User Commands * <<<classpath>>> Prints the class path needed to get the Hadoop jar and the required - libraries. + libraries. If called without arguments, then prints the classpath set up by + the command scripts, which is likely to contain wildcards in the classpath + entries. Additional options print the classpath after wildcard expansion or + write the classpath into the manifest of a jar file. The latter is useful in + environments where wildcards cannot be used and the expanded classpath exceeds + the maximum supported command line length. - Usage: <<<hadoop classpath>>> + Usage: <<<hadoop classpath [--glob|--jar <path>|-h|--help]>>> + +*-----------------+-----------------------------------------------------------+ +|| COMMAND_OPTION || Description +*-----------------+-----------------------------------------------------------+ +| --glob | expand wildcards +*-----------------+-----------------------------------------------------------+ +| --jar <path> | write classpath as manifest in jar named <path> +*-----------------+-----------------------------------------------------------+ +| -h, --help | print help +*-----------------+-----------------------------------------------------------+ Administration Commands
Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java Thu Aug 7 07:38:23 2014 @@ -26,10 +26,10 @@ import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion; import org.junit.BeforeClass; import org.junit.Test; - import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion; import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; @@ -118,8 +118,15 @@ public class TestKeyProviderCryptoExtens new IvParameterSpec(KeyProviderCryptoExtension.EncryptedKeyVersion .deriveIV(encryptedKeyIv))); final byte[] manualMaterial = cipher.doFinal(encryptedKeyMaterial); + + // Test the createForDecryption factory method + EncryptedKeyVersion eek2 = + EncryptedKeyVersion.createForDecryption( + eek.getEncryptionKeyVersionName(), eek.getEncryptedKeyIv(), + eek.getEncryptedKeyVersion().getMaterial()); + // Decrypt it with the API - KeyVersion decryptedKey = kpExt.decryptEncryptedKey(eek); + KeyVersion decryptedKey = kpExt.decryptEncryptedKey(eek2); final byte[] apiMaterial = decryptedKey.getMaterial(); assertArrayEquals("Wrong key material from decryptEncryptedKey", Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderFactory.java Thu Aug 7 07:38:23 2014 @@ -100,9 +100,9 @@ public class TestKeyProviderFactory { static void checkSpecificProvider(Configuration conf, String ourUrl) throws Exception { KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0); - byte[] key1 = new byte[32]; - byte[] key2 = new byte[32]; - byte[] key3 = new byte[32]; + byte[] key1 = new byte[16]; + byte[] key2 = new byte[16]; + byte[] key3 = new byte[16]; for(int i =0; i < key1.length; ++i) { key1[i] = (byte) i; key2[i] = (byte) (i * 2); @@ -146,7 +146,7 @@ public class TestKeyProviderFactory { KeyProvider.options(conf).setBitLength(8)); assertTrue("should throw", false); } catch (IOException e) { - assertEquals("Wrong key length. Required 8, but got 256", e.getMessage()); + assertEquals("Wrong key length. Required 8, but got 128", e.getMessage()); } provider.createKey("key4", new byte[]{1}, KeyProvider.options(conf).setBitLength(8)); @@ -162,7 +162,7 @@ public class TestKeyProviderFactory { provider.rollNewVersion("key4", key1); assertTrue("should throw", false); } catch (IOException e) { - assertEquals("Wrong key length. Required 8, but got 256", e.getMessage()); + assertEquals("Wrong key length. Required 8, but got 128", e.getMessage()); } try { provider.rollNewVersion("no-such-key", key1); @@ -228,7 +228,7 @@ public class TestKeyProviderFactory { public void checkPermissionRetention(Configuration conf, String ourUrl, Path path) throws Exception { KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0); // let's add a new key and flush and check that permissions are still set to 777 - byte[] key = new byte[32]; + byte[] key = new byte[16]; for(int i =0; i < key.length; ++i) { key[i] = (byte) i; } @@ -261,7 +261,7 @@ public class TestKeyProviderFactory { conf.set(JavaKeyStoreProvider.KEYSTORE_PASSWORD_FILE_KEY, "javakeystoreprovider.password"); KeyProvider provider = KeyProviderFactory.getProviders(conf).get(0); - provider.createKey("key3", new byte[32], KeyProvider.options(conf)); + provider.createKey("key3", new byte[16], KeyProvider.options(conf)); provider.flush(); } catch (Exception ex) { Assert.fail("could not create keystore with password file"); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyShell.java Thu Aug 7 07:38:23 2014 @@ -73,7 +73,7 @@ public class TestKeyShell { private void deleteKey(KeyShell ks, String keyName) throws Exception { int rc; outContent.reset(); - final String[] delArgs = {"delete", keyName, "--provider", jceksProvider}; + final String[] delArgs = {"delete", keyName, "-provider", jceksProvider}; rc = ks.run(delArgs); assertEquals(0, rc); assertTrue(outContent.toString().contains(keyName + " has been " + @@ -90,8 +90,8 @@ public class TestKeyShell { private String listKeys(KeyShell ks, boolean wantMetadata) throws Exception { int rc; outContent.reset(); - final String[] listArgs = {"list", "--provider", jceksProvider }; - final String[] listArgsM = {"list", "--metadata", "--provider", jceksProvider }; + final String[] listArgs = {"list", "-provider", jceksProvider }; + final String[] listArgsM = {"list", "-metadata", "-provider", jceksProvider }; rc = ks.run(wantMetadata ? listArgsM : listArgs); assertEquals(0, rc); return outContent.toString(); @@ -106,11 +106,11 @@ public class TestKeyShell { ks.setConf(new Configuration()); outContent.reset(); - final String[] args1 = {"create", keyName, "--provider", jceksProvider}; + final String[] args1 = {"create", keyName, "-provider", jceksProvider}; rc = ks.run(args1); assertEquals(0, rc); assertTrue(outContent.toString().contains(keyName + " has been " + - "successfully created.")); + "successfully created")); String listOut = listKeys(ks, false); assertTrue(listOut.contains(keyName)); @@ -121,7 +121,7 @@ public class TestKeyShell { assertTrue(listOut.contains("created")); outContent.reset(); - final String[] args2 = {"roll", keyName, "--provider", jceksProvider}; + final String[] args2 = {"roll", keyName, "-provider", jceksProvider}; rc = ks.run(args2); assertEquals(0, rc); assertTrue(outContent.toString().contains("key1 has been successfully " + @@ -137,15 +137,15 @@ public class TestKeyShell { @Test public void testKeySuccessfulCreationWithDescription() throws Exception { outContent.reset(); - final String[] args1 = {"create", "key1", "--provider", jceksProvider, - "--description", "someDescription"}; + final String[] args1 = {"create", "key1", "-provider", jceksProvider, + "-description", "someDescription"}; int rc = 0; KeyShell ks = new KeyShell(); ks.setConf(new Configuration()); rc = ks.run(args1); assertEquals(0, rc); assertTrue(outContent.toString().contains("key1 has been successfully " + - "created.")); + "created")); String listOut = listKeys(ks, true); assertTrue(listOut.contains("description")); @@ -154,7 +154,7 @@ public class TestKeyShell { @Test public void testInvalidKeySize() throws Exception { - final String[] args1 = {"create", "key1", "--size", "56", "--provider", + final String[] args1 = {"create", "key1", "-size", "56", "-provider", jceksProvider}; int rc = 0; @@ -167,7 +167,7 @@ public class TestKeyShell { @Test public void testInvalidCipher() throws Exception { - final String[] args1 = {"create", "key1", "--cipher", "LJM", "--provider", + final String[] args1 = {"create", "key1", "-cipher", "LJM", "-provider", jceksProvider}; int rc = 0; @@ -180,7 +180,7 @@ public class TestKeyShell { @Test public void testInvalidProvider() throws Exception { - final String[] args1 = {"create", "key1", "--cipher", "AES", "--provider", + final String[] args1 = {"create", "key1", "-cipher", "AES", "-provider", "sdff://file/tmp/keystore.jceks"}; int rc = 0; @@ -194,7 +194,7 @@ public class TestKeyShell { @Test public void testTransientProviderWarning() throws Exception { - final String[] args1 = {"create", "key1", "--cipher", "AES", "--provider", + final String[] args1 = {"create", "key1", "-cipher", "AES", "-provider", "user:///"}; int rc = 0; @@ -224,8 +224,8 @@ public class TestKeyShell { @Test public void testFullCipher() throws Exception { final String keyName = "key1"; - final String[] args1 = {"create", keyName, "--cipher", "AES/CBC/pkcs5Padding", - "--provider", jceksProvider}; + final String[] args1 = {"create", keyName, "-cipher", "AES/CBC/pkcs5Padding", + "-provider", jceksProvider}; int rc = 0; KeyShell ks = new KeyShell(); @@ -233,7 +233,7 @@ public class TestKeyShell { rc = ks.run(args1); assertEquals(0, rc); assertTrue(outContent.toString().contains(keyName + " has been " + - "successfully " + "created.")); + "successfully created")); deleteKey(ks, keyName); } @@ -245,12 +245,12 @@ public class TestKeyShell { ks.setConf(new Configuration()); /* Simple creation test */ - final String[] args1 = {"create", "keyattr1", "--provider", jceksProvider, - "--attr", "foo=bar"}; + final String[] args1 = {"create", "keyattr1", "-provider", jceksProvider, + "-attr", "foo=bar"}; rc = ks.run(args1); assertEquals(0, rc); assertTrue(outContent.toString().contains("keyattr1 has been " + - "successfully " + "created.")); + "successfully created")); /* ...and list to see that we have the attr */ String listOut = listKeys(ks, true); @@ -259,8 +259,8 @@ public class TestKeyShell { /* Negative tests: no attribute */ outContent.reset(); - final String[] args2 = {"create", "keyattr2", "--provider", jceksProvider, - "--attr", "=bar"}; + final String[] args2 = {"create", "keyattr2", "-provider", jceksProvider, + "-attr", "=bar"}; rc = ks.run(args2); assertEquals(1, rc); @@ -288,10 +288,10 @@ public class TestKeyShell { /* Test several attrs together... */ outContent.reset(); - final String[] args3 = {"create", "keyattr3", "--provider", jceksProvider, - "--attr", "foo = bar", - "--attr", " glarch =baz ", - "--attr", "abc=def"}; + final String[] args3 = {"create", "keyattr3", "-provider", jceksProvider, + "-attr", "foo = bar", + "-attr", " glarch =baz ", + "-attr", "abc=def"}; rc = ks.run(args3); assertEquals(0, rc); @@ -304,9 +304,9 @@ public class TestKeyShell { /* Negative test - repeated attributes should fail */ outContent.reset(); - final String[] args4 = {"create", "keyattr4", "--provider", jceksProvider, - "--attr", "foo=bar", - "--attr", "foo=glarch"}; + final String[] args4 = {"create", "keyattr4", "-provider", jceksProvider, + "-attr", "foo=bar", + "-attr", "foo=glarch"}; rc = ks.run(args4); assertEquals(1, rc); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java Thu Aug 7 07:38:23 2014 @@ -414,7 +414,7 @@ public class TestHttpServer extends Http assertEquals(HttpURLConnection.HTTP_OK, getHttpStatusCode(serverURL + servlet, user)); } - assertEquals(HttpURLConnection.HTTP_UNAUTHORIZED, getHttpStatusCode( + assertEquals(HttpURLConnection.HTTP_FORBIDDEN, getHttpStatusCode( serverURL + servlet, "userE")); } myServer.stop(); @@ -474,7 +474,7 @@ public class TestHttpServer extends Http response = Mockito.mock(HttpServletResponse.class); conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION, true); Assert.assertFalse(HttpServer2.hasAdministratorAccess(context, request, response)); - Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString()); + Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_FORBIDDEN), Mockito.anyString()); //authorization ON & user NOT NULL & ACLs NULL response = Mockito.mock(HttpServletResponse.class); @@ -487,7 +487,7 @@ public class TestHttpServer extends Http Mockito.when(acls.isUserAllowed(Mockito.<UserGroupInformation>any())).thenReturn(false); Mockito.when(context.getAttribute(HttpServer2.ADMINS_ACL)).thenReturn(acls); Assert.assertFalse(HttpServer2.hasAdministratorAccess(context, request, response)); - Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_UNAUTHORIZED), Mockito.anyString()); + Mockito.verify(response).sendError(Mockito.eq(HttpServletResponse.SC_FORBIDDEN), Mockito.anyString()); //authorization ON & user NOT NULL & ACLs NOT NULL & user in in ACLs response = Mockito.mock(HttpServletResponse.class); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestLdapGroupsMapping.java Thu Aug 7 07:38:23 2014 @@ -17,6 +17,8 @@ */ package org.apache.hadoop.security; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.*; import java.io.File; @@ -38,6 +40,9 @@ import javax.naming.directory.SearchCont import javax.naming.directory.SearchResult; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -154,4 +159,57 @@ public class TestLdapGroupsMapping { Assert.assertEquals("hadoop", mapping.extractPassword(secretFile.getPath())); } + + @Test + public void testConfGetPassword() throws Exception { + File testDir = new File(System.getProperty("test.build.data", + "target/test-dir")); + Configuration conf = new Configuration(); + final String ourUrl = + JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + + File file = new File(testDir, "test.jks"); + file.delete(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + + CredentialProvider provider = + CredentialProviderFactory.getProviders(conf).get(0); + char[] bindpass = {'b', 'i', 'n', 'd', 'p', 'a', 's', 's'}; + char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'}; + + // ensure that we get nulls when the key isn't there + assertEquals(null, provider.getCredentialEntry( + LdapGroupsMapping.BIND_PASSWORD_KEY)); + assertEquals(null, provider.getCredentialEntry + (LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY)); + + // create new aliases + try { + provider.createCredentialEntry( + LdapGroupsMapping.BIND_PASSWORD_KEY, bindpass); + + provider.createCredentialEntry( + LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY, storepass); + provider.flush(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + // make sure we get back the right key + assertArrayEquals(bindpass, provider.getCredentialEntry( + LdapGroupsMapping.BIND_PASSWORD_KEY).getCredential()); + assertArrayEquals(storepass, provider.getCredentialEntry( + LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY).getCredential()); + + LdapGroupsMapping mapping = new LdapGroupsMapping(); + Assert.assertEquals("bindpass", + mapping.getPassword(conf, LdapGroupsMapping.BIND_PASSWORD_KEY, "")); + Assert.assertEquals("storepass", + mapping.getPassword(conf, LdapGroupsMapping.LDAP_KEYSTORE_PASSWORD_KEY, + "")); + // let's make sure that a password that doesn't exist returns an + // empty string as currently expected and used to trigger a call to + // extract password + Assert.assertEquals("", mapping.getPassword(conf,"invalid-alias", "")); + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java Thu Aug 7 07:38:23 2014 @@ -17,16 +17,18 @@ */ package org.apache.hadoop.security.alias; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.PrintStream; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.security.alias.CredentialShell.PasswordReader; import org.junit.Before; import org.junit.Test; @@ -45,7 +47,7 @@ public class TestCredShell { @Test public void testCredentialSuccessfulLifecycle() throws Exception { outContent.reset(); - String[] args1 = {"create", "credential1", "--value", "p@ssw0rd", "--provider", + String[] args1 = {"create", "credential1", "-value", "p@ssw0rd", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; int rc = 0; CredentialShell cs = new CredentialShell(); @@ -56,14 +58,14 @@ public class TestCredShell { "created.")); outContent.reset(); - String[] args2 = {"list", "--provider", + String[] args2 = {"list", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; rc = cs.run(args2); assertEquals(0, rc); assertTrue(outContent.toString().contains("credential1")); outContent.reset(); - String[] args4 = {"delete", "credential1", "--provider", + String[] args4 = {"delete", "credential1", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; rc = cs.run(args4); assertEquals(0, rc); @@ -71,7 +73,7 @@ public class TestCredShell { "deleted.")); outContent.reset(); - String[] args5 = {"list", "--provider", + String[] args5 = {"list", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; rc = cs.run(args5); assertEquals(0, rc); @@ -80,21 +82,21 @@ public class TestCredShell { @Test public void testInvalidProvider() throws Exception { - String[] args1 = {"create", "credential1", "--value", "p@ssw0rd", "--provider", + String[] args1 = {"create", "credential1", "-value", "p@ssw0rd", "-provider", "sdff://file/tmp/credstore.jceks"}; int rc = 0; CredentialShell cs = new CredentialShell(); cs.setConf(new Configuration()); rc = cs.run(args1); - assertEquals(-1, rc); + assertEquals(1, rc); assertTrue(outContent.toString().contains("There are no valid " + "CredentialProviders configured.")); } @Test public void testTransientProviderWarning() throws Exception { - String[] args1 = {"create", "credential1", "--value", "p@ssw0rd", "--provider", + String[] args1 = {"create", "credential1", "-value", "p@ssw0rd", "-provider", "user:///"}; int rc = 0; @@ -105,7 +107,7 @@ public class TestCredShell { assertTrue(outContent.toString().contains("WARNING: you are modifying a " + "transient provider.")); - String[] args2 = {"delete", "credential1", "--provider", "user:///"}; + String[] args2 = {"delete", "credential1", "-provider", "user:///"}; rc = cs.run(args2); assertEquals(outContent.toString(), 0, rc); assertTrue(outContent.toString().contains("credential1 has been successfully " + @@ -122,14 +124,14 @@ public class TestCredShell { config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, "user:///"); cs.setConf(config); rc = cs.run(args1); - assertEquals(-1, rc); + assertEquals(1, rc); assertTrue(outContent.toString().contains("There are no valid " + "CredentialProviders configured.")); } @Test public void testPromptForCredentialWithEmptyPasswd() throws Exception { - String[] args1 = {"create", "credential1", "--provider", + String[] args1 = {"create", "credential1", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; ArrayList<String> passwords = new ArrayList<String>(); passwords.add(null); @@ -139,13 +141,13 @@ public class TestCredShell { shell.setConf(new Configuration()); shell.setPasswordReader(new MockPasswordReader(passwords)); rc = shell.run(args1); - assertEquals(outContent.toString(), -1, rc); + assertEquals(outContent.toString(), 1, rc); assertTrue(outContent.toString().contains("Passwords don't match")); } @Test public void testPromptForCredential() throws Exception { - String[] args1 = {"create", "credential1", "--provider", + String[] args1 = {"create", "credential1", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; ArrayList<String> passwords = new ArrayList<String>(); passwords.add("p@ssw0rd"); @@ -159,7 +161,7 @@ public class TestCredShell { assertTrue(outContent.toString().contains("credential1 has been successfully " + "created.")); - String[] args2 = {"delete", "credential1", "--provider", + String[] args2 = {"delete", "credential1", "-provider", "jceks://file" + tmpDir + "/credstore.jceks"}; rc = shell.run(args2); assertEquals(0, rc); @@ -186,4 +188,21 @@ public class TestCredShell { System.out.println(message); } } + + @Test + public void testEmptyArgList() throws Exception { + CredentialShell shell = new CredentialShell(); + shell.setConf(new Configuration()); + assertEquals(1, shell.init(new String[0])); + } + + @Test + public void testCommandHelpExitsNormally() throws Exception { + for (String cmd : Arrays.asList("create", "list", "delete")) { + CredentialShell shell = new CredentialShell(); + shell.setConf(new Configuration()); + assertEquals("Expected help argument on " + cmd + " to return 0", + 0, shell.init(new String[] {cmd, "-help"})); + } + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/KeyStoreTestUtil.java Thu Aug 7 07:38:23 2014 @@ -19,6 +19,10 @@ package org.apache.hadoop.security.ssl; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; + import sun.security.x509.AlgorithmId; import sun.security.x509.CertificateAlgorithmId; import sun.security.x509.CertificateIssuerName; @@ -382,4 +386,41 @@ public class KeyStoreTestUtil { writer.close(); } } + + public static void provisionPasswordsToCredentialProvider() throws Exception { + File testDir = new File(System.getProperty("test.build.data", + "target/test-dir")); + + Configuration conf = new Configuration(); + final String ourUrl = + JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + + File file = new File(testDir, "test.jks"); + file.delete(); + conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + + CredentialProvider provider = + CredentialProviderFactory.getProviders(conf).get(0); + char[] keypass = {'k', 'e', 'y', 'p', 'a', 's', 's'}; + char[] storepass = {'s', 't', 'o', 'r', 'e', 'p', 'a', 's', 's'}; + + // create new aliases + try { + provider.createCredentialEntry( + FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_KEYSTORE_PASSWORD_TPL_KEY), + storepass); + + provider.createCredentialEntry( + FileBasedKeyStoresFactory.resolvePropertyName(SSLFactory.Mode.SERVER, + FileBasedKeyStoresFactory.SSL_KEYSTORE_KEYPASSWORD_TPL_KEY), + keypass); + + // write out so that it can be found in checks + provider.flush(); + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + } } Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/ssl/TestSSLFactory.java Thu Aug 7 07:38:23 2014 @@ -17,8 +17,14 @@ */ package org.apache.hadoop.security.ssl; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.security.alias.CredentialProvider; +import org.apache.hadoop.security.alias.CredentialProviderFactory; +import org.apache.hadoop.security.alias.JavaKeyStoreProvider; import org.junit.After; import org.junit.Assert; import org.junit.Before; @@ -211,6 +217,13 @@ public class TestSSLFactory { "password", "password", null); } + @Test + public void testServerCredProviderPasswords() throws Exception { + KeyStoreTestUtil.provisionPasswordsToCredentialProvider(); + checkSSLFactoryInitWithPasswords(SSLFactory.Mode.SERVER, + "storepass", "keypass", null, null, true); + } + /** * Checks that SSLFactory initialization is successful with the given * arguments. This is a helper method for writing test cases that cover @@ -218,7 +231,7 @@ public class TestSSLFactory { * It takes care of bootstrapping a keystore, a truststore, and SSL client or * server configuration. Then, it initializes an SSLFactory. If no exception * is thrown, then initialization was successful. - * + * * @param mode SSLFactory.Mode mode to test * @param password String store password to set on keystore * @param keyPassword String key password to set on keystore @@ -231,6 +244,34 @@ public class TestSSLFactory { private void checkSSLFactoryInitWithPasswords(SSLFactory.Mode mode, String password, String keyPassword, String confPassword, String confKeyPassword) throws Exception { + checkSSLFactoryInitWithPasswords(mode, password, keyPassword, + confPassword, confKeyPassword, false); + } + + /** + * Checks that SSLFactory initialization is successful with the given + * arguments. This is a helper method for writing test cases that cover + * different combinations of settings for the store password and key password. + * It takes care of bootstrapping a keystore, a truststore, and SSL client or + * server configuration. Then, it initializes an SSLFactory. If no exception + * is thrown, then initialization was successful. + * + * @param mode SSLFactory.Mode mode to test + * @param password String store password to set on keystore + * @param keyPassword String key password to set on keystore + * @param confPassword String store password to set in SSL config file, or null + * to avoid setting in SSL config file + * @param confKeyPassword String key password to set in SSL config file, or + * null to avoid setting in SSL config file + * @param useCredProvider boolean to indicate whether passwords should be set + * into the config or not. When set to true nulls are set and aliases are + * expected to be resolved through credential provider API through the + * Configuration.getPassword method + * @throws Exception for any error + */ + private void checkSSLFactoryInitWithPasswords(SSLFactory.Mode mode, + String password, String keyPassword, String confPassword, + String confKeyPassword, boolean useCredProvider) throws Exception { String keystore = new File(KEYSTORES_DIR, "keystore.jks").getAbsolutePath(); String truststore = new File(KEYSTORES_DIR, "truststore.jks") .getAbsolutePath(); @@ -249,10 +290,25 @@ public class TestSSLFactory { // Create SSL configuration file, for either server or client. final String sslConfFileName; final Configuration sslConf; + + // if the passwords are provisioned in a cred provider then don't set them + // in the configuration properly - expect them to be resolved through the + // provider + if (useCredProvider) { + confPassword = null; + confKeyPassword = null; + } if (mode == SSLFactory.Mode.SERVER) { sslConfFileName = "ssl-server.xml"; sslConf = KeyStoreTestUtil.createServerSSLConfig(keystore, confPassword, confKeyPassword, truststore); + if (useCredProvider) { + File testDir = new File(System.getProperty("test.build.data", + "target/test-dir")); + final String ourUrl = + JavaKeyStoreProvider.SCHEME_NAME + "://file/" + testDir + "/test.jks"; + sslConf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl); + } } else { sslConfFileName = "ssl-client.xml"; sslConf = KeyStoreTestUtil.createClientSSLConfig(keystore, confPassword, Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java Thu Aug 7 07:38:23 2014 @@ -181,12 +181,19 @@ public class KMSWebApp implements Servle keyProvider = new CachingKeyProvider(keyProvider, keyTimeOutMillis, currKeyTimeOutMillis); } + LOG.info("Initialized KeyProvider " + keyProvider); + keyProviderCryptoExtension = KeyProviderCryptoExtension. createKeyProviderCryptoExtension(keyProvider); keyProviderCryptoExtension = new EagerKeyGeneratorKeyProviderCryptoExtension(kmsConf, keyProviderCryptoExtension); - + LOG.info("Initialized KeyProviderCryptoExtension " + + keyProviderCryptoExtension); + final int defaultBitlength = kmsConf + .getInt(KeyProvider.DEFAULT_BITLENGTH_NAME, + KeyProvider.DEFAULT_BITLENGTH); + LOG.info("Default key bitlength is {}", defaultBitlength); LOG.info("KMS Started"); } catch (Throwable ex) { System.out.println(); Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/main/webapp/WEB-INF/web.xml Thu Aug 7 07:38:23 2014 @@ -42,7 +42,7 @@ <servlet> <servlet-name>jmx-servlet</servlet-name> - <servlet-class>org.apache.hadoop.jmx.JMXJsonServlet</servlet-class> + <servlet-class>org.apache.hadoop.crypto.key.kms.server.KMSJMXServlet</servlet-class> </servlet> <servlet-mapping> Modified: hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm?rev=1616428&r1=1616427&r2=1616428&view=diff ============================================================================== --- hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm (original) +++ hadoop/common/branches/HDFS-6584/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm Thu Aug 7 07:38:23 2014 @@ -106,14 +106,14 @@ Hadoop Key Management Server (KMS) - Doc ** KMS Aggregated Audit logs -Audit logs are aggregated for API accesses to the GET_KEY_VERSION, -GET_CURRENT_KEY, DECRYPT_EEK, GENERATE_EEK operations. + Audit logs are aggregated for API accesses to the GET_KEY_VERSION, + GET_CURRENT_KEY, DECRYPT_EEK, GENERATE_EEK operations. -Entries are grouped by the (user,key,operation) combined key for a configurable -aggregation interval after which the number of accesses to the specified -end-point by the user for a given key is flushed to the audit log. + Entries are grouped by the (user,key,operation) combined key for a + configurable aggregation interval after which the number of accesses to the + specified end-point by the user for a given key is flushed to the audit log. -The Aggregation interval is configured via the property : + The Aggregation interval is configured via the property : +---+ <property>