git commit: Fix hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt .
Repository: hadoop Updated Branches: refs/heads/HDFS-6584 aaa7e2175 - 91f6ddeb3 Fix hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt . Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91f6ddeb Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91f6ddeb Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91f6ddeb Branch: refs/heads/HDFS-6584 Commit: 91f6ddeb34dba3041848838af650ac4b1fddb731 Parents: aaa7e21 Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com Authored: Wed Sep 17 14:09:08 2014 +0800 Committer: Tsz-Wo Nicholas Sze szets...@hortonworks.com Committed: Wed Sep 17 14:09:08 2014 +0800 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 49 +++- 1 file changed, 47 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/91f6ddeb/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 59a130e..e859ca2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -20,11 +20,56 @@ HDFS-6584: Archival Storage HDFS-6686. Change BlockPlacementPolicy to use fallback when some storage types are unavailable. (szetszwo) +HDFS-6835. Add a new API to set storage policy. (jing9) + HDFS-6847. Support storage policy on directories and include storage policy in HdfsFileStatus. (Jing Zhao via szetszwo) -HDFS-7072. Fix TestBlockManager and TestStorageMover. (Jing Zhao -via szetszwo) +HDFS-6801. Add a new data migration tool, Mover, for archiving data. +(szetszwo via jing9) + +HDFS-6863. Support migration for snapshot paths. (jing9) + +HDFS-6906. Add more tests for BlockStoragePolicy. (szetszwo via jing9) + +HDFS-6911. check if a block is already scheduled in Mover. +(szetszwo via jing9) + +HDFS-6920. Check the storage type of delNodeHintStorage when deleting +a replica. (szetszwo via jing9) + +HDFS-6944. Add retry and termination logic for Mover. (jing9) + +HDFS-6969. INode#getStoragePolicyID should always return the latest +storage policy. (jing9) + +HDFS-6961. BlockPlacementPolicy#chooseTarget should check each valid +storage type in each choosing round. (jing9) + +HDFS-6876. support set/get storage policy in DFSAdmin. (jing9) + +HDFS-6997. Add more tests for data migration and replicaion. (szetszwo) + +HDFS-6875. Support migration for a list of specified paths. (jing9) + +HDFS-7027. Mover does not terminate when some storage type is out of space. +(szetszwo via jing9) + +HDFS-7029. Fix TestDFSInotifyEventInputStream and TestDistributedFileSystem. +(szetszwo via jing9) + +HDFS-7028. FSDirectory should not get storage policy id from symlinks. +(szetszwo) + +HDFS-7034. Fix TestBlockPlacement and TestStorageMover. (jing9) + +HDFS-7039. Fix Balancer tests. (szetszwo via jing9) + +HDFS-7062. Skip under construction block for migration. (jing9) + +HDFS-7052. Add Mover into hdfs script. (jing9) + +HDFS-7072. Fix TestBlockManager and TestStorageMover. (jing9 via szetszwo) Trunk (Unreleased)
git commit: HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion belongs to the keyname on decrypt. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0e7d1dbf9 - e14e71d5f
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e14e71d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e14e71d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e14e71d5
Branch: refs/heads/trunk
Commit: e14e71d5feff961b681d828b00e6f12cb197ebf5
Parents: 0e7d1db
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 14:32:49 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/KeyProviderCryptoExtension.java | 8 +--
.../key/TestKeyProviderCryptoExtension.java | 2 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 12 +
.../server/TestKeyAuthorizationKeyProvider.java | 53
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
6 files changed, 76 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3bf9d4b..9324acd 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -815,6 +815,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
+belongs to the keyname on decrypt. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index fed7e9e..968e341 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -91,6 +91,8 @@ public class KeyProviderCryptoExtension extends
* returned EncryptedKeyVersion will only partially be populated; it is not
* necessarily suitable for operations besides decryption.
*
+ * @param keyName Key name of the encryption key use to encrypt the
+ *encrypted key.
* @param encryptionKeyVersionName Version name of the encryption key used
* to encrypt the encrypted key.
* @param encryptedKeyIv Initialization vector of the encrypted
@@ -100,12 +102,12 @@ public class KeyProviderCryptoExtension extends
* @param encryptedKeyMaterial Key material of the encrypted key.
* @return EncryptedKeyVersion suitable for decryption.
*/
-public static EncryptedKeyVersion createForDecryption(String
-encryptionKeyVersionName, byte[] encryptedKeyIv,
+public static EncryptedKeyVersion createForDecryption(String keyName,
+String encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
- return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ return new EncryptedKeyVersion(keyName, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 70ec6fe..62e3310 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
git commit: HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion belongs to the keyname on decrypt. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c6b9768b3 - 94a1e68aa
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94a1e68a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94a1e68a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94a1e68a
Branch: refs/heads/branch-2
Commit: 94a1e68aa5aa3ea633b3af7b09aa2b9012498101
Parents: c6b9768
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 14:32:49 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:21:17 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/KeyProviderCryptoExtension.java | 8 +--
.../key/TestKeyProviderCryptoExtension.java | 2 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 12 +
.../server/TestKeyAuthorizationKeyProvider.java | 53
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
6 files changed, 76 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94a1e68a/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0ec1264..939af25 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -480,6 +480,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
+belongs to the keyname on decrypt. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94a1e68a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index 5d3281c..f800689 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -91,6 +91,8 @@ public class KeyProviderCryptoExtension extends
* returned EncryptedKeyVersion will only partially be populated; it is not
* necessarily suitable for operations besides decryption.
*
+ * @param keyName Key name of the encryption key use to encrypt the
+ *encrypted key.
* @param encryptionKeyVersionName Version name of the encryption key used
* to encrypt the encrypted key.
* @param encryptedKeyIv Initialization vector of the encrypted
@@ -100,12 +102,12 @@ public class KeyProviderCryptoExtension extends
* @param encryptedKeyMaterial Key material of the encrypted key.
* @return EncryptedKeyVersion suitable for decryption.
*/
-public static EncryptedKeyVersion createForDecryption(String
-encryptionKeyVersionName, byte[] encryptedKeyIv,
+public static EncryptedKeyVersion createForDecryption(String keyName,
+String encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
- return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ return new EncryptedKeyVersion(keyName, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94a1e68a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 9893515..0b202ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++
git commit: HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 94a1e68aa - 75bd79231
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb
via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75bd7923
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75bd7923
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75bd7923
Branch: refs/heads/branch-2
Commit: 75bd79231ca30cb7a16107101c175c5b6fa06f56
Parents: 94a1e68
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:47:55 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:21:17 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 8
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75bd7923/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 939af25..d6b05f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -483,6 +483,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
+HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
+(clamb via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75bd7923/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index be6c8f1..02ca1c5 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -197,22 +197,22 @@ hadoop-${project.version} $ sbin/kms.sh start
*** KMS Proxyuser Configuration
- Each proxyusers must be configured in etc/hadoop/kms-site.xml using the
+ Each proxyuser must be configured in etc/hadoop/kms-site.xml using the
following properties:
+---+
property
-namehadoop.kms.proxyusers.#USER#.users/name
+namehadoop.kms.proxyuser.#USER#.users/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.groups/name
+namehadoop.kms.proxyuser.#USER#.groups/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.hosts/name
+namehadoop.kms.proxyuser.#USER#.hosts/name
value*/value
/property
+---+
git commit: HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb via tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk e14e71d5f - 8cf1052be
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb
via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cf1052b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cf1052b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cf1052b
Branch: refs/heads/trunk
Commit: 8cf1052beb7cab68be1a6319c0a4d7e1c790d58a
Parents: e14e71d
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:47:55 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 8
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9324acd..11151f0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -818,6 +818,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
+HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
+(clamb via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index c76ca3b..d70f2a6 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -197,22 +197,22 @@ hadoop-${project.version} $ sbin/kms.sh start
*** KMS Proxyuser Configuration
- Each proxyusers must be configured in etc/hadoop/kms-site.xml using the
+ Each proxyuser must be configured in etc/hadoop/kms-site.xml using the
following properties:
+---+
property
-namehadoop.kms.proxyusers.#USER#.users/name
+namehadoop.kms.proxyuser.#USER#.users/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.groups/name
+namehadoop.kms.proxyuser.#USER#.groups/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.hosts/name
+namehadoop.kms.proxyuser.#USER#.hosts/name
value*/value
/property
+---+
git commit: HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if -Pnative is used. (asuresh via tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 75bd79231 - 1c847fdd6
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if
-Pnative is used. (asuresh via tucu)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c847fdd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c847fdd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c847fdd
Branch: refs/heads/branch-2
Commit: 1c847fdd61414f7f564de2cc477621edac8164b5
Parents: 75bd792
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 23:36:10 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:37:21 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/pom.xml | 3 +++
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 18 --
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 11 +++
4 files changed, 33 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c847fdd/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d6b05f7..0fad37d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -486,6 +486,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
(clamb via tucu)
+HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
+only if -Pnative is used. (asuresh via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c847fdd/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index cb6bafa..4a9fae3 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -390,6 +390,7 @@
systemPropertyVariables
startKdc${startKdc}/startKdc
kdc.resource.dir${kdc.resource.dir}/kdc.resource.dir
+runningWithNative${runningWithNative}/runningWithNative
/systemPropertyVariables
/configuration
/plugin
@@ -528,6 +529,7 @@
openssl.lib/openssl.lib
openssl.include/openssl.include
require.opensslfalse/require.openssl
+runningWithNativetrue/runningWithNative
/properties
build
plugins
@@ -647,6 +649,7 @@
openssl.lib/openssl.lib
openssl.include/openssl.include
require.opensslfalse/require.openssl
+runningWithNativetrue/runningWithNative
bundle.openssl.in.bintrue/bundle.openssl.in.bin
/properties
build
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c847fdd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 298f4ef..79987ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -59,7 +59,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!true.equalsIgnoreCase(System.getProperty(runningWithNative))) {
+ LOG.warn(Skipping since test was not run with -Pnative flag);
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn(Skipping test since openSSL library not loaded);
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
@@ -68,7 +75,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!true.equalsIgnoreCase(System.getProperty(runningWithNative))) {
+ LOG.warn(Skipping since test was not run
git commit: HDFS-6864. Archival Storage: add user documentation. Contributed by Tsz Wo Nicholas Sze.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 91f6ddeb3 - b014e83bc
HDFS-6864. Archival Storage: add user documentation. Contributed by Tsz Wo
Nicholas Sze.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b014e83b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b014e83b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b014e83b
Branch: refs/heads/HDFS-6584
Commit: b014e83bc5899ec135b1e7a54ca1902c970047a5
Parents: 91f6dde
Author: Jing Zhao j...@hortonworks.com
Authored: Wed Sep 17 09:40:17 2014 -0700
Committer: Jing Zhao j...@hortonworks.com
Committed: Wed Sep 17 09:40:17 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hadoop/hdfs/DistributedFileSystem.java | 6 +
.../apache/hadoop/hdfs/server/mover/Mover.java | 6 +-
.../src/site/apt/ArchivalStorage.apt.vm | 302 +++
.../src/site/apt/HDFSCommands.apt.vm| 43 ++-
hadoop-project/src/site/site.xml| 1 +
6 files changed, 349 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b014e83b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e859ca2..7a9c723 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -71,6 +71,8 @@ HDFS-6584: Archival Storage
HDFS-7072. Fix TestBlockManager and TestStorageMover. (jing9 via szetszwo)
+HDFS-6864. Archival Storage: add user documentation. (szetszwo via jing9)
+
Trunk (Unreleased)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b014e83b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 1c60e7b..6bce8b9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -472,6 +472,12 @@ public class DistributedFileSystem extends FileSystem {
}.resolve(this, absF);
}
+ /**
+ * Set the source path to the specified storage policy.
+ *
+ * @param src The source path referring to either a directory or a file.
+ * @param policyName The name of the storage policy.
+ */
public void setStoragePolicy(final Path src, final String policyName)
throws IOException {
statistics.incrementWriteOps(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b014e83b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 0812c03..f1837ae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -498,9 +498,9 @@ public class Mover {
static class Cli extends Configured implements Tool {
private static final String USAGE = Usage: java
-+ Mover.class.getSimpleName()
-+ [-p space separated files/dirs specify a list of files/dirs to
migrate]
-+ [-f local file namespecify a local file containing
files/dirs to migrate];
++ Mover.class.getSimpleName() + [-p files/dirs | -f local file]
++ \n\t-p files/dirs\ta space separated list of HDFS files/dirs to
migrate.
++ \n\t-f local file\ta local file containing a list of HDFS
files/dirs to migrate.;
private static Options buildCliOptions() {
Options opts = new Options();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b014e83b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
new file mode 100644
index 000..5301d52
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/ArchivalStorage.apt.vm
@@ -0,0 +1,302 @@
+~~ Licensed under the Apache License, Version 2.0 (the
[2/3] git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a495bef Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a495bef Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a495bef Branch: refs/heads/trunk Commit: 0a495bef5cd675dce4c928cb5331588bb198accf Parents: e4ddb6d Author: Alejandro Abdelnur t...@apache.org Authored: Tue Sep 16 21:21:17 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 11:08:00 2014 -0700 -- hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 6 files changed, 370 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 2c225cb..e6b21aa 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency +dependency + groupIdorg.apache.curator/groupId + artifactIdcurator-test/artifactId + scopetest/scope +/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ /description /property + !-- Authentication cookie signature source -- + + property +namehadoop.kms.authentication.signer.secret.provider/name +valuerandom/value +description + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. +/description + /property + + !-- Configuration for 'zookeeper' authentication cookie signature source -- + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name +value/hadoop-kms/hadoop-auth-signature-secret/value +description + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name +value#HOSTNAME#:#PORT#,.../value +description + The Zookeeper connection string, a list of hostnames and port comma + separated. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name +valuekerberos/value +description + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name +value/etc/hadoop/conf/kms.keytab/value +description + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal/name +valuekms/#HOSTNAME#/value +description + The Kerberos service principal used to connect to Zookeeper. +/description + /property + /configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 4df6db5..79652f3 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++
[3/3] git commit: HADOOP-10982
HADOOP-10982
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9a86031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9a86031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9a86031
Branch: refs/heads/trunk
Commit: d9a86031a077184d429dd5463e7da156df112011
Parents: 0a495be
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 23:07:01 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:08:00 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 ++
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 72 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 899b6c4..a97463a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+} catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 5fded92..682f479 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ HTTP/#HOSTNAME#. This means that all KMS instances must have have a
+ Kerberos service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Kebero service name with its own hostname. This is
+ require for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+ property
+namehadoop.kms.authentication.kerberos.principal/name
+value*/value
+ /property
++---+
+
+ NOTE: If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ keytool SAN extension support for details on how to do this).
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index cdb3c7f..42afe19 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
[1/3] git commit: HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/trunk c0c7e6fab - d9a86031a
HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4ddb6da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4ddb6da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4ddb6da
Branch: refs/heads/trunk
Commit: e4ddb6da15420d5c13ec7ec99fed1e44b32290b0
Parents: c0c7e6f
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:29:09 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:07:56 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 2 ++
.../apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f0fcab5..a1dca66 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -824,6 +824,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
only if -Pnative is used. (asuresh via tucu)
+HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index 77b78ee..5cb0885 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -79,7 +79,7 @@ public class KMSExceptionsProvider implements
ExceptionMapperException {
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AuthorizationException) {
- status = Response.Status.UNAUTHORIZED;
+ status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AccessControlException) {
[1/2] git commit: Revert HADOOP-10982
Repository: hadoop
Updated Branches:
refs/heads/trunk d9a86031a - 8a7671d75
Revert HADOOP-10982
This reverts commit d9a86031a077184d429dd5463e7da156df112011.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f8f860c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f8f860c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f8f860c
Branch: refs/heads/trunk
Commit: 3f8f860cc65e179dd5766fea4d21cf30fa4b96e3
Parents: d9a8603
Author: Alejandro Abdelnur t...@apache.org
Authored: Wed Sep 17 11:11:15 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:11:15 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 --
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 11 insertions(+), 72 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a97463a..899b6c4 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,7 +45,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -401,8 +400,6 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
-} catch (UndeclaredThrowableException ex) {
- throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 682f479..5fded92 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,31 +602,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- When KMS instances are behind a load-balancer or VIP, clients will use the
- hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
- URL is used to construct the Kerberos service name of the server,
- HTTP/#HOSTNAME#. This means that all KMS instances must have have a
- Kerberos service name with the load-balancer or VIP hostname.
-
- In order to be able to access directly a specific KMS instance, the KMS
- instance must also have Kebero service name with its own hostname. This is
- require for monitoring and admin purposes.
-
- Both Kerberos service principal credentials (for the load-balancer/VIP
- hostname and for the actual KMS instance hostname) must be in the keytab file
- configured for authentication. And the principal name specified in the
- configuration must be '*'. For example:
-
-+---+
- property
-namehadoop.kms.authentication.kerberos.principal/name
-value*/value
- /property
-+---+
-
- NOTE: If using HTTPS, the SSL certificate used by the KMS instance must
- be configured to support multiple hostnames (see Java 7
- keytool SAN extension support for details on how to do this).
+ TBD
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 42afe19..cdb3c7f 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.minikdc.MiniKdc;
[2/2] git commit: Revert HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
Revert HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) This reverts commit 0a495bef5cd675dce4c928cb5331588bb198accf. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a7671d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a7671d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a7671d7 Branch: refs/heads/trunk Commit: 8a7671d7539bff0566cb87f2b347f71bcf148977 Parents: 3f8f860 Author: Alejandro Abdelnur t...@apache.org Authored: Wed Sep 17 11:11:33 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 11:11:33 2014 -0700 -- hadoop-common-project/hadoop-kms/pom.xml| 5 - .../hadoop-kms/src/main/conf/kms-site.xml | 57 -- .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 - .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 --- 6 files changed, 44 insertions(+), 370 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index e6b21aa..2c225cb 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,11 +187,6 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency -dependency - groupIdorg.apache.curator/groupId - artifactIdcurator-test/artifactId - scopetest/scope -/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index f55ce5f..20896fc 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,61 +68,4 @@ /description /property - !-- Authentication cookie signature source -- - - property -namehadoop.kms.authentication.signer.secret.provider/name -valuerandom/value -description - Indicates how the secret to sign the authentication cookies will be - stored. Options are 'random' (default), 'string' and 'zookeeper'. - If using a setup with multiple KMS instances, 'zookeeper' should be used. -/description - /property - - !-- Configuration for 'zookeeper' authentication cookie signature source -- - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name -value/hadoop-kms/hadoop-auth-signature-secret/value -description - The Zookeeper ZNode path where the KMS instances will store and retrieve - the secret from. -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name -value#HOSTNAME#:#PORT#,.../value -description - The Zookeeper connection string, a list of hostnames and port comma - separated. -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name -valuekerberos/value -description - The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name -value/etc/hadoop/conf/kms.keytab/value -description - The absolute path for the Kerberos keytab with the credentials to - connect to Zookeeper. -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal/name -valuekms/#HOSTNAME#/value -description - The Kerberos service principal used to connect to Zookeeper. -/description - /property - /configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 79652f3..4df6db5 100644 ---
git commit: HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1c847fdd6 - 6857c291a
HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
(cherry picked from commit e4ddb6da15420d5c13ec7ec99fed1e44b32290b0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6857c291
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6857c291
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6857c291
Branch: refs/heads/branch-2
Commit: 6857c291af05350064336ba12c121c7fada27a5d
Parents: 1c847fd
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:29:09 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:08:25 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 2 ++
.../apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6857c291/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0fad37d..40b0045 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -489,6 +489,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
only if -Pnative is used. (asuresh via tucu)
+HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6857c291/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index 77b78ee..5cb0885 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -79,7 +79,7 @@ public class KMSExceptionsProvider implements
ExceptionMapperException {
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AuthorizationException) {
- status = Response.Status.UNAUTHORIZED;
+ status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AccessControlException) {
git commit: HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file. (clamb via wang)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6857c291a - 5e54aae62
HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file.
(clamb via wang)
(cherry picked from commit ea4e2e843ecadd8019ea35413f4a34b97a424923)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e54aae6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e54aae6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e54aae6
Branch: refs/heads/branch-2
Commit: 5e54aae62b1d388398acd947054170e2ca4e4da1
Parents: 6857c29
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 11:23:47 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 11:26:17 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/common/HdfsServerConstants.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 42 +-
.../hdfs/server/namenode/FSNamesystem.java | 25 +++-
.../server/namenode/XAttrPermissionFilter.java | 14 ++
.../src/site/apt/ExtendedAttributes.apt.vm | 3 +-
.../hdfs/server/namenode/FSXAttrBaseTest.java | 148 +--
.../src/test/resources/testXAttrConf.xml| 73 +
8 files changed, 287 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e54aae6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c9bc8bc..1329ac6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -210,6 +210,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6851. Refactor EncryptionZoneWithId and EncryptionZone. (clamb via
wang)
+HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a
+file. (clamb via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e54aae6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 98c6398..106f489 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -299,5 +299,6 @@ public final class HdfsServerConstants {
raw.hdfs.crypto.encryption.zone;
public static final String CRYPTO_XATTR_FILE_ENCRYPTION_INFO =
raw.hdfs.crypto.file.encryption.info;
+ public static final String SECURITY_XATTR_UNREADABLE_BY_SUPERUSER =
+ security.hdfs.unreadable.by.superuser;
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e54aae6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 23c40b5..f31cf4a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
+import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
import static org.apache.hadoop.util.Time.now;
import java.io.Closeable;
@@ -90,6 +91,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.hadoop.security.AccessControlException;
/**
* Both FSDirectory and FSNamesystem manage the state of the namespace.
@@ -128,6 +130,8
[04/11] git commit: HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if -Pnative is used. (asuresh via tucu)
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if
-Pnative is used. (asuresh via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0c7e6fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0c7e6fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0c7e6fa
Branch: refs/heads/HDFS-6581
Commit: c0c7e6fabd573df85791d7ec4c536fd48280883f
Parents: 8cf1052
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 23:36:10 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:36:36 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/pom.xml | 3 +++
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 18 --
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 +++
4 files changed, 29 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 11151f0..f0fcab5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -821,6 +821,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
(clamb via tucu)
+HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
+only if -Pnative is used. (asuresh via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index ae495be..0183e29 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -375,6 +375,7 @@
systemPropertyVariables
startKdc${startKdc}/startKdc
kdc.resource.dir${kdc.resource.dir}/kdc.resource.dir
+runningWithNative${runningWithNative}/runningWithNative
/systemPropertyVariables
properties
property
@@ -507,6 +508,7 @@
openssl.lib/openssl.lib
openssl.include/openssl.include
require.opensslfalse/require.openssl
+runningWithNativetrue/runningWithNative
/properties
build
plugins
@@ -626,6 +628,7 @@
openssl.lib/openssl.lib
openssl.include/openssl.include
require.opensslfalse/require.openssl
+runningWithNativetrue/runningWithNative
bundle.openssl.in.bintrue/bundle.openssl.in.bin
/properties
build
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 298f4ef..79987ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -59,7 +59,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!true.equalsIgnoreCase(System.getProperty(runningWithNative))) {
+ LOG.warn(Skipping since test was not run with -Pnative flag);
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn(Skipping test since openSSL library not loaded);
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
@@ -68,7 +75,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!true.equalsIgnoreCase(System.getProperty(runningWithNative))) {
+ LOG.warn(Skipping since test was not run with -Pnative flag);
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn(Skipping
[01/11] git commit: YARN-1250. Addendum
Repository: hadoop Updated Branches: refs/heads/HDFS-6581 dcbc46730 - 24f815688 YARN-1250. Addendum Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e7d1dbf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e7d1dbf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e7d1dbf Branch: refs/heads/HDFS-6581 Commit: 0e7d1dbf9ab732dd04dccaacbf273e9ac437eba5 Parents: 90a0c03 Author: junping_du junping...@apache.org Authored: Tue Sep 16 18:25:45 2014 -0700 Committer: junping_du junping...@apache.org Committed: Tue Sep 16 18:25:45 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e7d1dbf/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ec59cba..51fe3cc 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -88,6 +88,9 @@ Release 2.6.0 - UNRELEASED and enforce/not-enforce strict control of per-container cpu usage. (Varun Vasudev via vinodkv) +YARN-1250. Generic history service should support application-acls. (Zhijie Shen +via junping_du) + IMPROVEMENTS YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
[05/11] git commit: HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4ddb6da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4ddb6da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4ddb6da
Branch: refs/heads/HDFS-6581
Commit: e4ddb6da15420d5c13ec7ec99fed1e44b32290b0
Parents: c0c7e6f
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:29:09 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:07:56 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 2 ++
.../apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f0fcab5..a1dca66 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -824,6 +824,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
only if -Pnative is used. (asuresh via tucu)
+HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index 77b78ee..5cb0885 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -79,7 +79,7 @@ public class KMSExceptionsProvider implements
ExceptionMapperException {
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AuthorizationException) {
- status = Response.Status.UNAUTHORIZED;
+ status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AccessControlException) {
[03/11] git commit: HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion belongs to the keyname on decrypt. (tucu)
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e14e71d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e14e71d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e14e71d5
Branch: refs/heads/HDFS-6581
Commit: e14e71d5feff961b681d828b00e6f12cb197ebf5
Parents: 0e7d1db
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 14:32:49 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/KeyProviderCryptoExtension.java | 8 +--
.../key/TestKeyProviderCryptoExtension.java | 2 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 12 +
.../server/TestKeyAuthorizationKeyProvider.java | 53
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
6 files changed, 76 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3bf9d4b..9324acd 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -815,6 +815,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
+belongs to the keyname on decrypt. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index fed7e9e..968e341 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -91,6 +91,8 @@ public class KeyProviderCryptoExtension extends
* returned EncryptedKeyVersion will only partially be populated; it is not
* necessarily suitable for operations besides decryption.
*
+ * @param keyName Key name of the encryption key use to encrypt the
+ *encrypted key.
* @param encryptionKeyVersionName Version name of the encryption key used
* to encrypt the encrypted key.
* @param encryptedKeyIv Initialization vector of the encrypted
@@ -100,12 +102,12 @@ public class KeyProviderCryptoExtension extends
* @param encryptedKeyMaterial Key material of the encrypted key.
* @return EncryptedKeyVersion suitable for decryption.
*/
-public static EncryptedKeyVersion createForDecryption(String
-encryptionKeyVersionName, byte[] encryptedKeyIv,
+public static EncryptedKeyVersion createForDecryption(String keyName,
+String encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
- return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ return new EncryptedKeyVersion(keyName, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 70ec6fe..62e3310 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
@@ -121,7 +121,7 @@ public class TestKeyProviderCryptoExtension {
//
[02/11] git commit: HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb via tucu)
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb
via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cf1052b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cf1052b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cf1052b
Branch: refs/heads/HDFS-6581
Commit: 8cf1052beb7cab68be1a6319c0a4d7e1c790d58a
Parents: e14e71d
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:47:55 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 8
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9324acd..11151f0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -818,6 +818,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
+HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
+(clamb via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index c76ca3b..d70f2a6 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -197,22 +197,22 @@ hadoop-${project.version} $ sbin/kms.sh start
*** KMS Proxyuser Configuration
- Each proxyusers must be configured in etc/hadoop/kms-site.xml using the
+ Each proxyuser must be configured in etc/hadoop/kms-site.xml using the
following properties:
+---+
property
-namehadoop.kms.proxyusers.#USER#.users/name
+namehadoop.kms.proxyuser.#USER#.users/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.groups/name
+namehadoop.kms.proxyuser.#USER#.groups/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.hosts/name
+namehadoop.kms.proxyuser.#USER#.hosts/name
value*/value
/property
+---+
[11/11] git commit: Merge branch 'trunk' into HDFS-6581
Merge branch 'trunk' into HDFS-6581 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24f81568 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24f81568 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24f81568 Branch: refs/heads/HDFS-6581 Commit: 24f81568868d4db1bcbe628d9ebd7bff7b7315de Parents: dcbc467 ea4e2e8 Author: arp a...@apache.org Authored: Wed Sep 17 12:10:50 2014 -0700 Committer: arp a...@apache.org Committed: Wed Sep 17 12:10:50 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 11 ++ hadoop-common-project/hadoop-common/pom.xml | 3 + .../crypto/key/KeyProviderCryptoExtension.java | 8 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 18 ++- .../key/TestKeyProviderCryptoExtension.java | 2 +- .../key/kms/server/KMSExceptionsProvider.java | 2 +- .../kms/server/KeyAuthorizationKeyProvider.java | 12 ++ .../hadoop-kms/src/site/apt/index.apt.vm| 8 +- .../server/TestKeyAuthorizationKeyProvider.java | 53 +++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +- .../hdfs/server/common/HdfsServerConstants.java | 3 +- .../hdfs/server/namenode/FSDirectory.java | 42 +- .../hdfs/server/namenode/FSNamesystem.java | 24 ++- .../server/namenode/XAttrPermissionFilter.java | 14 ++ .../src/site/apt/ExtendedAttributes.apt.vm | 3 +- .../hdfs/server/namenode/FSXAttrBaseTest.java | 148 +-- .../src/test/resources/testXAttrConf.xml| 73 + hadoop-yarn-project/CHANGES.txt | 3 + 20 files changed, 405 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/24f81568/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/24f81568/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java --
[10/11] git commit: HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file. (clamb via wang)
HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file.
(clamb via wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea4e2e84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea4e2e84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea4e2e84
Branch: refs/heads/HDFS-6581
Commit: ea4e2e843ecadd8019ea35413f4a34b97a424923
Parents: 8a7671d
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 11:23:47 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 11:23:47 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/common/HdfsServerConstants.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 42 +-
.../hdfs/server/namenode/FSNamesystem.java | 24 ++-
.../server/namenode/XAttrPermissionFilter.java | 14 ++
.../src/site/apt/ExtendedAttributes.apt.vm | 3 +-
.../hdfs/server/namenode/FSXAttrBaseTest.java | 148 +--
.../src/test/resources/testXAttrConf.xml| 73 +
8 files changed, 287 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea4e2e84/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 752e778..567a6ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -468,6 +468,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6851. Refactor EncryptionZoneWithId and EncryptionZone. (clamb via
wang)
+HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a
+file. (clamb via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea4e2e84/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 98c6398..106f489 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -299,5 +299,6 @@ public final class HdfsServerConstants {
raw.hdfs.crypto.encryption.zone;
public static final String CRYPTO_XATTR_FILE_ENCRYPTION_INFO =
raw.hdfs.crypto.file.encryption.info;
+ public static final String SECURITY_XATTR_UNREADABLE_BY_SUPERUSER =
+ security.hdfs.unreadable.by.superuser;
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea4e2e84/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 836ebd2..e33832d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
+import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
import static org.apache.hadoop.util.Time.now;
import java.io.Closeable;
@@ -90,6 +91,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.hadoop.security.AccessControlException;
/**
* Both FSDirectory and FSNamesystem manage the state of the namespace.
@@ -128,6 +130,8 @@ public class FSDirectory implements Closeable {
DFSUtil.string2Bytes(DOT_INODES_STRING);
private final XAttr KEYID_XATTR =
XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
+ private final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
+
[08/11] git commit: Revert HADOOP-10982
Revert HADOOP-10982
This reverts commit d9a86031a077184d429dd5463e7da156df112011.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f8f860c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f8f860c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f8f860c
Branch: refs/heads/HDFS-6581
Commit: 3f8f860cc65e179dd5766fea4d21cf30fa4b96e3
Parents: d9a8603
Author: Alejandro Abdelnur t...@apache.org
Authored: Wed Sep 17 11:11:15 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:11:15 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 --
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 11 insertions(+), 72 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a97463a..899b6c4 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,7 +45,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -401,8 +400,6 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
-} catch (UndeclaredThrowableException ex) {
- throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 682f479..5fded92 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,31 +602,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- When KMS instances are behind a load-balancer or VIP, clients will use the
- hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
- URL is used to construct the Kerberos service name of the server,
- HTTP/#HOSTNAME#. This means that all KMS instances must have have a
- Kerberos service name with the load-balancer or VIP hostname.
-
- In order to be able to access directly a specific KMS instance, the KMS
- instance must also have Kebero service name with its own hostname. This is
- require for monitoring and admin purposes.
-
- Both Kerberos service principal credentials (for the load-balancer/VIP
- hostname and for the actual KMS instance hostname) must be in the keytab file
- configured for authentication. And the principal name specified in the
- configuration must be '*'. For example:
-
-+---+
- property
-namehadoop.kms.authentication.kerberos.principal/name
-value*/value
- /property
-+---+
-
- NOTE: If using HTTPS, the SSL certificate used by the KMS instance must
- be configured to support multiple hostnames (see Java 7
- keytool SAN extension support for details on how to do this).
+ TBD
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 42afe19..cdb3c7f 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import
[07/11] git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a495bef Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a495bef Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a495bef Branch: refs/heads/HDFS-6581 Commit: 0a495bef5cd675dce4c928cb5331588bb198accf Parents: e4ddb6d Author: Alejandro Abdelnur t...@apache.org Authored: Tue Sep 16 21:21:17 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 11:08:00 2014 -0700 -- hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 6 files changed, 370 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 2c225cb..e6b21aa 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency +dependency + groupIdorg.apache.curator/groupId + artifactIdcurator-test/artifactId + scopetest/scope +/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ /description /property + !-- Authentication cookie signature source -- + + property +namehadoop.kms.authentication.signer.secret.provider/name +valuerandom/value +description + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. +/description + /property + + !-- Configuration for 'zookeeper' authentication cookie signature source -- + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name +value/hadoop-kms/hadoop-auth-signature-secret/value +description + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name +value#HOSTNAME#:#PORT#,.../value +description + The Zookeeper connection string, a list of hostnames and port comma + separated. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name +valuekerberos/value +description + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name +value/etc/hadoop/conf/kms.keytab/value +description + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal/name +valuekms/#HOSTNAME#/value +description + The Kerberos service principal used to connect to Zookeeper. +/description + /property + /configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 4df6db5..79652f3 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++
[06/11] git commit: HADOOP-10982
HADOOP-10982
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9a86031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9a86031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9a86031
Branch: refs/heads/HDFS-6581
Commit: d9a86031a077184d429dd5463e7da156df112011
Parents: 0a495be
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 23:07:01 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:08:00 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 ++
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 72 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 899b6c4..a97463a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+} catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 5fded92..682f479 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ HTTP/#HOSTNAME#. This means that all KMS instances must have have a
+ Kerberos service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Kebero service name with its own hostname. This is
+ require for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+ property
+namehadoop.kms.authentication.kerberos.principal/name
+value*/value
+ /property
++---+
+
+ NOTE: If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ keytool SAN extension support for details on how to do this).
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index cdb3c7f..42afe19 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import
[02/10] git commit: HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion belongs to the keyname on decrypt. (tucu)
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e14e71d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e14e71d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e14e71d5
Branch: refs/heads/HDFS-6584
Commit: e14e71d5feff961b681d828b00e6f12cb197ebf5
Parents: 0e7d1db
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 14:32:49 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
.../crypto/key/KeyProviderCryptoExtension.java | 8 +--
.../key/TestKeyProviderCryptoExtension.java | 2 +-
.../kms/server/KeyAuthorizationKeyProvider.java | 12 +
.../server/TestKeyAuthorizationKeyProvider.java | 53
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
6 files changed, 76 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3bf9d4b..9324acd 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -815,6 +815,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11088. Unittest TestKeyShell, TestCredShell and TestKMS assume UNIX
path separator for JECKS key store path. (Xiaoyu Yao via cnauroth)
+HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
+belongs to the keyname on decrypt. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index fed7e9e..968e341 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -91,6 +91,8 @@ public class KeyProviderCryptoExtension extends
* returned EncryptedKeyVersion will only partially be populated; it is not
* necessarily suitable for operations besides decryption.
*
+ * @param keyName Key name of the encryption key use to encrypt the
+ *encrypted key.
* @param encryptionKeyVersionName Version name of the encryption key used
* to encrypt the encrypted key.
* @param encryptedKeyIv Initialization vector of the encrypted
@@ -100,12 +102,12 @@ public class KeyProviderCryptoExtension extends
* @param encryptedKeyMaterial Key material of the encrypted key.
* @return EncryptedKeyVersion suitable for decryption.
*/
-public static EncryptedKeyVersion createForDecryption(String
-encryptionKeyVersionName, byte[] encryptedKeyIv,
+public static EncryptedKeyVersion createForDecryption(String keyName,
+String encryptionKeyVersionName, byte[] encryptedKeyIv,
byte[] encryptedKeyMaterial) {
KeyVersion encryptedKeyVersion = new KeyVersion(null, EEK,
encryptedKeyMaterial);
- return new EncryptedKeyVersion(null, encryptionKeyVersionName,
+ return new EncryptedKeyVersion(keyName, encryptionKeyVersionName,
encryptedKeyIv, encryptedKeyVersion);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e14e71d5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 70ec6fe..62e3310 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
@@ -121,7 +121,7 @@ public class TestKeyProviderCryptoExtension {
//
[10/10] git commit: Merge changes from trunk
Merge changes from trunk Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/911979c8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/911979c8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/911979c8 Branch: refs/heads/HDFS-6584 Commit: 911979c8ab2e6dd4fe82023ae022a1582c8590c2 Parents: b014e83 ea4e2e8 Author: Jing Zhao j...@hortonworks.com Authored: Wed Sep 17 13:12:45 2014 -0700 Committer: Jing Zhao j...@hortonworks.com Committed: Wed Sep 17 13:12:45 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 11 ++ hadoop-common-project/hadoop-common/pom.xml | 3 + .../crypto/key/KeyProviderCryptoExtension.java | 8 +- .../apache/hadoop/crypto/TestCryptoCodec.java | 18 ++- .../key/TestKeyProviderCryptoExtension.java | 2 +- .../key/kms/server/KMSExceptionsProvider.java | 2 +- .../kms/server/KeyAuthorizationKeyProvider.java | 12 ++ .../hadoop-kms/src/site/apt/index.apt.vm| 8 +- .../server/TestKeyAuthorizationKeyProvider.java | 53 +++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 + .../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +- .../hdfs/server/common/HdfsServerConstants.java | 3 +- .../hdfs/server/namenode/FSDirectory.java | 42 +- .../hdfs/server/namenode/FSNamesystem.java | 25 +++- .../server/namenode/XAttrPermissionFilter.java | 14 ++ .../src/site/apt/ExtendedAttributes.apt.vm | 3 +- .../hdfs/server/namenode/FSXAttrBaseTest.java | 148 +-- .../src/test/resources/testXAttrConf.xml| 73 + 19 files changed, 403 insertions(+), 35 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/911979c8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/911979c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/911979c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/911979c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java --
[04/10] git commit: HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4ddb6da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4ddb6da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4ddb6da
Branch: refs/heads/HDFS-6584
Commit: e4ddb6da15420d5c13ec7ec99fed1e44b32290b0
Parents: c0c7e6f
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:29:09 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:07:56 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 2 ++
.../apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f0fcab5..a1dca66 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -824,6 +824,8 @@ Release 2.6.0 - UNRELEASED
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
only if -Pnative is used. (asuresh via tucu)
+HADOOP-11099. KMS return HTTP UNAUTHORIZED 401 on ACL failure. (tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4ddb6da/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
index 77b78ee..5cb0885 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSExceptionsProvider.java
@@ -79,7 +79,7 @@ public class KMSExceptionsProvider implements
ExceptionMapperException {
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AuthorizationException) {
- status = Response.Status.UNAUTHORIZED;
+ status = Response.Status.FORBIDDEN;
// we don't audit here because we did it already when checking access
doAudit = false;
} else if (throwable instanceof AccessControlException) {
[01/10] git commit: HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb via tucu)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 b014e83bc - 911979c8a
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params. (clamb
via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8cf1052b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8cf1052b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8cf1052b
Branch: refs/heads/HDFS-6584
Commit: 8cf1052beb7cab68be1a6319c0a4d7e1c790d58a
Parents: e14e71d
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 21:47:55 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:20:35 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm | 8
2 files changed, 7 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9324acd..11151f0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -818,6 +818,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11096. KMS: KeyAuthorizationKeyProvider should verify the keyversion
belongs to the keyname on decrypt. (tucu)
+HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
+(clamb via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8cf1052b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index c76ca3b..d70f2a6 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -197,22 +197,22 @@ hadoop-${project.version} $ sbin/kms.sh start
*** KMS Proxyuser Configuration
- Each proxyusers must be configured in etc/hadoop/kms-site.xml using the
+ Each proxyuser must be configured in etc/hadoop/kms-site.xml using the
following properties:
+---+
property
-namehadoop.kms.proxyusers.#USER#.users/name
+namehadoop.kms.proxyuser.#USER#.users/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.groups/name
+namehadoop.kms.proxyuser.#USER#.groups/name
value*/value
/property
property
-namehadoop.kms.proxyusers.#USER#.hosts/name
+namehadoop.kms.proxyuser.#USER#.hosts/name
value*/value
/property
+---+
[06/10] git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0a495bef Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0a495bef Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0a495bef Branch: refs/heads/HDFS-6584 Commit: 0a495bef5cd675dce4c928cb5331588bb198accf Parents: e4ddb6d Author: Alejandro Abdelnur t...@apache.org Authored: Tue Sep 16 21:21:17 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 11:08:00 2014 -0700 -- hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 6 files changed, 370 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 2c225cb..e6b21aa 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency +dependency + groupIdorg.apache.curator/groupId + artifactIdcurator-test/artifactId + scopetest/scope +/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ /description /property + !-- Authentication cookie signature source -- + + property +namehadoop.kms.authentication.signer.secret.provider/name +valuerandom/value +description + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. +/description + /property + + !-- Configuration for 'zookeeper' authentication cookie signature source -- + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name +value/hadoop-kms/hadoop-auth-signature-secret/value +description + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name +value#HOSTNAME#:#PORT#,.../value +description + The Zookeeper connection string, a list of hostnames and port comma + separated. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name +valuekerberos/value +description + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name +value/etc/hadoop/conf/kms.keytab/value +description + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal/name +valuekms/#HOSTNAME#/value +description + The Kerberos service principal used to connect to Zookeeper. +/description + /property + /configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/0a495bef/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 4df6db5..79652f3 100644 --- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java +++
[07/10] git commit: Revert HADOOP-10982
Revert HADOOP-10982
This reverts commit d9a86031a077184d429dd5463e7da156df112011.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f8f860c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f8f860c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f8f860c
Branch: refs/heads/HDFS-6584
Commit: 3f8f860cc65e179dd5766fea4d21cf30fa4b96e3
Parents: d9a8603
Author: Alejandro Abdelnur t...@apache.org
Authored: Wed Sep 17 11:11:15 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:11:15 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 --
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 11 insertions(+), 72 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index a97463a..899b6c4 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,7 +45,6 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
-import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -401,8 +400,6 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
-} catch (UndeclaredThrowableException ex) {
- throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 682f479..5fded92 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,31 +602,7 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- When KMS instances are behind a load-balancer or VIP, clients will use the
- hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
- URL is used to construct the Kerberos service name of the server,
- HTTP/#HOSTNAME#. This means that all KMS instances must have have a
- Kerberos service name with the load-balancer or VIP hostname.
-
- In order to be able to access directly a specific KMS instance, the KMS
- instance must also have Kebero service name with its own hostname. This is
- require for monitoring and admin purposes.
-
- Both Kerberos service principal credentials (for the load-balancer/VIP
- hostname and for the actual KMS instance hostname) must be in the keytab file
- configured for authentication. And the principal name specified in the
- configuration must be '*'. For example:
-
-+---+
- property
-namehadoop.kms.authentication.kerberos.principal/name
-value*/value
- /property
-+---+
-
- NOTE: If using HTTPS, the SSL certificate used by the KMS instance must
- be configured to support multiple hostnames (see Java 7
- keytool SAN extension support for details on how to do this).
+ TBD
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f8f860c/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 42afe19..cdb3c7f 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import
[08/10] git commit: Revert HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
Revert HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) This reverts commit 0a495bef5cd675dce4c928cb5331588bb198accf. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8a7671d7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8a7671d7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8a7671d7 Branch: refs/heads/HDFS-6584 Commit: 8a7671d7539bff0566cb87f2b347f71bcf148977 Parents: 3f8f860 Author: Alejandro Abdelnur t...@apache.org Authored: Wed Sep 17 11:11:33 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 11:11:33 2014 -0700 -- hadoop-common-project/hadoop-kms/pom.xml| 5 - .../hadoop-kms/src/main/conf/kms-site.xml | 57 -- .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 - .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 --- 6 files changed, 44 insertions(+), 370 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index e6b21aa..2c225cb 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,11 +187,6 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency -dependency - groupIdorg.apache.curator/groupId - artifactIdcurator-test/artifactId - scopetest/scope -/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index f55ce5f..20896fc 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,61 +68,4 @@ /description /property - !-- Authentication cookie signature source -- - - property -namehadoop.kms.authentication.signer.secret.provider/name -valuerandom/value -description - Indicates how the secret to sign the authentication cookies will be - stored. Options are 'random' (default), 'string' and 'zookeeper'. - If using a setup with multiple KMS instances, 'zookeeper' should be used. -/description - /property - - !-- Configuration for 'zookeeper' authentication cookie signature source -- - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name -value/hadoop-kms/hadoop-auth-signature-secret/value -description - The Zookeeper ZNode path where the KMS instances will store and retrieve - the secret from. -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name -value#HOSTNAME#:#PORT#,.../value -description - The Zookeeper connection string, a list of hostnames and port comma - separated. -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name -valuekerberos/value -description - The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name -value/etc/hadoop/conf/kms.keytab/value -description - The absolute path for the Kerberos keytab with the credentials to - connect to Zookeeper. -/description - /property - - property - namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal/name -valuekms/#HOSTNAME#/value -description - The Kerberos service principal used to connect to Zookeeper. -/description - /property - /configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/8a7671d7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java -- diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSAuthenticationFilter.java index 79652f3..4df6db5 100644 ---
[03/10] git commit: HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if -Pnative is used. (asuresh via tucu)
HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run only if
-Pnative is used. (asuresh via tucu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0c7e6fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0c7e6fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0c7e6fa
Branch: refs/heads/HDFS-6584
Commit: c0c7e6fabd573df85791d7ec4c536fd48280883f
Parents: 8cf1052
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 23:36:10 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Tue Sep 16 23:36:36 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/pom.xml | 3 +++
.../org/apache/hadoop/crypto/TestCryptoCodec.java | 18 --
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 +++
4 files changed, 29 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 11151f0..f0fcab5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -821,6 +821,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11097. kms docs say proxyusers, not proxyuser for config params.
(clamb via tucu)
+HADOOP-11062. CryptoCodec testcases requiring OpenSSL should be run
+only if -Pnative is used. (asuresh via tucu)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/pom.xml
--
diff --git a/hadoop-common-project/hadoop-common/pom.xml
b/hadoop-common-project/hadoop-common/pom.xml
index ae495be..0183e29 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -375,6 +375,7 @@
systemPropertyVariables
startKdc${startKdc}/startKdc
kdc.resource.dir${kdc.resource.dir}/kdc.resource.dir
+runningWithNative${runningWithNative}/runningWithNative
/systemPropertyVariables
properties
property
@@ -507,6 +508,7 @@
openssl.lib/openssl.lib
openssl.include/openssl.include
require.opensslfalse/require.openssl
+runningWithNativetrue/runningWithNative
/properties
build
plugins
@@ -626,6 +628,7 @@
openssl.lib/openssl.lib
openssl.include/openssl.include
require.opensslfalse/require.openssl
+runningWithNativetrue/runningWithNative
bundle.openssl.in.bintrue/bundle.openssl.in.bin
/properties
build
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0c7e6fa/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 298f4ef..79987ce 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -59,7 +59,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testJceAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!true.equalsIgnoreCase(System.getProperty(runningWithNative))) {
+ LOG.warn(Skipping since test was not run with -Pnative flag);
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn(Skipping test since openSSL library not loaded);
+ Assume.assumeTrue(false);
+}
Assert.assertEquals(null, OpensslCipher.getLoadingFailureReason());
cryptoCodecTest(conf, seed, 0, jceCodecClass, jceCodecClass);
cryptoCodecTest(conf, seed, count, jceCodecClass, jceCodecClass);
@@ -68,7 +75,14 @@ public class TestCryptoCodec {
@Test(timeout=12)
public void testOpensslAesCtrCryptoCodec() throws Exception {
-Assume.assumeTrue(NativeCodeLoader.buildSupportsOpenssl());
+if (!true.equalsIgnoreCase(System.getProperty(runningWithNative))) {
+ LOG.warn(Skipping since test was not run with -Pnative flag);
+ Assume.assumeTrue(false);
+}
+if (!NativeCodeLoader.buildSupportsOpenssl()) {
+ LOG.warn(Skipping
[05/10] git commit: HADOOP-10982
HADOOP-10982
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9a86031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9a86031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9a86031
Branch: refs/heads/HDFS-6584
Commit: d9a86031a077184d429dd5463e7da156df112011
Parents: 0a495be
Author: Alejandro Abdelnur t...@apache.org
Authored: Tue Sep 16 23:07:01 2014 -0700
Committer: Alejandro Abdelnur t...@apache.org
Committed: Wed Sep 17 11:08:00 2014 -0700
--
.../crypto/key/kms/KMSClientProvider.java | 3 ++
.../hadoop-kms/src/site/apt/index.apt.vm| 26 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 54
3 files changed, 72 insertions(+), 11 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 899b6c4..a97463a 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -45,6 +45,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.SocketTimeoutException;
import java.net.URI;
@@ -400,6 +401,8 @@ public class KMSClientProvider extends KeyProvider
implements CryptoExtension,
});
} catch (IOException ex) {
throw ex;
+} catch (UndeclaredThrowableException ex) {
+ throw new IOException(ex.getUndeclaredThrowable());
} catch (Exception ex) {
throw new IOException(ex);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
--
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
index 5fded92..682f479 100644
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
@@ -602,7 +602,31 @@ $ keytool -genkey -alias tomcat -keyalg RSA
*** HTTP Kerberos Principals Configuration
- TBD
+ When KMS instances are behind a load-balancer or VIP, clients will use the
+ hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
+ URL is used to construct the Kerberos service name of the server,
+ HTTP/#HOSTNAME#. This means that all KMS instances must have have a
+ Kerberos service name with the load-balancer or VIP hostname.
+
+ In order to be able to access directly a specific KMS instance, the KMS
+ instance must also have Kebero service name with its own hostname. This is
+ require for monitoring and admin purposes.
+
+ Both Kerberos service principal credentials (for the load-balancer/VIP
+ hostname and for the actual KMS instance hostname) must be in the keytab file
+ configured for authentication. And the principal name specified in the
+ configuration must be '*'. For example:
+
++---+
+ property
+namehadoop.kms.authentication.kerberos.principal/name
+value*/value
+ /property
++---+
+
+ NOTE: If using HTTPS, the SSL certificate used by the KMS instance must
+ be configured to support multiple hostnames (see Java 7
+ keytool SAN extension support for details on how to do this).
*** HTTP Authentication Signature
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9a86031/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index cdb3c7f..42afe19 100644
---
a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++
b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import
[09/10] git commit: HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file. (clamb via wang)
HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file.
(clamb via wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea4e2e84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea4e2e84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea4e2e84
Branch: refs/heads/HDFS-6584
Commit: ea4e2e843ecadd8019ea35413f4a34b97a424923
Parents: 8a7671d
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 11:23:47 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 11:23:47 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/common/HdfsServerConstants.java | 3 +-
.../hdfs/server/namenode/FSDirectory.java | 42 +-
.../hdfs/server/namenode/FSNamesystem.java | 24 ++-
.../server/namenode/XAttrPermissionFilter.java | 14 ++
.../src/site/apt/ExtendedAttributes.apt.vm | 3 +-
.../hdfs/server/namenode/FSXAttrBaseTest.java | 148 +--
.../src/test/resources/testXAttrConf.xml| 73 +
8 files changed, 287 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea4e2e84/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 752e778..567a6ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -468,6 +468,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6851. Refactor EncryptionZoneWithId and EncryptionZone. (clamb via
wang)
+HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a
+file. (clamb via wang)
+
OPTIMIZATIONS
HDFS-6690. Deduplicate xattr names in memory. (wang)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea4e2e84/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 98c6398..106f489 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -299,5 +299,6 @@ public final class HdfsServerConstants {
raw.hdfs.crypto.encryption.zone;
public static final String CRYPTO_XATTR_FILE_ENCRYPTION_INFO =
raw.hdfs.crypto.file.encryption.info;
+ public static final String SECURITY_XATTR_UNREADABLE_BY_SUPERUSER =
+ security.hdfs.unreadable.by.superuser;
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea4e2e84/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 836ebd2..e33832d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
+import static
org.apache.hadoop.hdfs.server.common.HdfsServerConstants.SECURITY_XATTR_UNREADABLE_BY_SUPERUSER;
import static org.apache.hadoop.util.Time.now;
import java.io.Closeable;
@@ -90,6 +91,7 @@ import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import org.apache.hadoop.security.AccessControlException;
/**
* Both FSDirectory and FSNamesystem manage the state of the namespace.
@@ -128,6 +130,8 @@ public class FSDirectory implements Closeable {
DFSUtil.string2Bytes(DOT_INODES_STRING);
private final XAttr KEYID_XATTR =
XAttrHelper.buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, null);
+ private final XAttr UNREADABLE_BY_SUPERUSER_XATTR =
+
git commit: HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk ea4e2e843 - e3803d002
HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3803d00
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3803d00
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3803d00
Branch: refs/heads/trunk
Commit: e3803d002c660f18a5c2ecf32344fd6f3f491a5b
Parents: ea4e2e8
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 12:55:35 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 12:55:35 2014 -0700
--
.../java/org/apache/hadoop/fs/FileStatus.java | 9 ++
.../hadoop/fs/permission/FsPermission.java | 7 ++
.../src/site/markdown/filesystem/filesystem.md | 31 +++
.../fs/contract/AbstractContractOpenTest.java | 12 +++
.../hadoop/hdfs/protocol/FsAclPermission.java | 77 -
.../hdfs/protocol/FsPermissionExtension.java| 89
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 +-
.../hdfs/server/namenode/FSDirectory.java | 36 +---
.../org/apache/hadoop/hdfs/web/JsonUtil.java| 16 +++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 88 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 5 +-
11 files changed, 280 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index b261f7f..da3807d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -200,6 +200,15 @@ public class FileStatus implements Writable, Comparable {
public FsPermission getPermission() {
return permission;
}
+
+ /**
+ * Tell whether the underlying file or directory is encrypted or not.
+ *
+ * @return true if the underlying file is encrypted.
+ */
+ public boolean isEncrypted() {
+return permission.getEncryptedBit();
+ }
/**
* Get the owner of the file.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index ee84437..264a095 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -294,6 +294,13 @@ public class FsPermission implements Writable {
return false;
}
+ /**
+ * Returns true if the file is encrypted or directory is in an encryption
zone
+ */
+ public boolean getEncryptedBit() {
+return false;
+ }
+
/** Set the user file creation mask (umask) */
public static void setUMask(Configuration conf, FsPermission umask) {
conf.set(UMASK_LABEL, String.format(%1$03o, umask.toShort()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
--
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 70796cc..e59fa1b 100644
---
a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++
b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -64,6 +64,33 @@ all operations on a valid FileSystem MUST result in a new
FileSystem that is als
def isSymlink(FS, p) = p in symlinks(FS)
+### 'boolean inEncryptionZone(Path p)'
+
+Return True if the data for p is encrypted. The nature of the encryption and
the
+mechanism for creating an encryption zone are implementation details not
covered
+in this specification. No guarantees are made about the quality of the
+encryption. The metadata is not encrypted.
+
+ Preconditions
+
+if not exists(FS, p) : raise FileNotFoundException
+
+ Postconditions
+
+ Invariants
+
+All files and directories under a
git commit: HDFS-7075. hadoop-fuse-dfs fails because it cannot find JavaKeyStoreProvider$Factory. (cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk f24ac429d - f23024852
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory. (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2302485
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2302485
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2302485
Branch: refs/heads/trunk
Commit: f23024852502441fc259012664e444e5e51c604a
Parents: f24ac42
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 14:27:32 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 14:27:32 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/key/KeyProviderFactory.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2302485/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8cb6c8d..31c09de 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -726,6 +726,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
+HDFS-7075. hadoop-fuse-dfs fails because it cannot find
+JavaKeyStoreProvider$Factory (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2302485/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index 6ca0425..ce99d79 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -46,7 +46,8 @@ public abstract class KeyProviderFactory {
) throws IOException;
private static final ServiceLoaderKeyProviderFactory serviceLoader =
- ServiceLoader.load(KeyProviderFactory.class);
+ ServiceLoader.load(KeyProviderFactory.class,
+ KeyProviderFactory.class.getClassLoader());
// Iterate through the serviceLoader to avoid lazy loading.
// Lazy loading would require synchronization in concurrent use cases.
git commit: HDFS-7075. hadoop-fuse-dfs fails because it cannot find JavaKeyStoreProvider$Factory. (cmccabe) (cherry picked from commit f23024852502441fc259012664e444e5e51c604a)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6cb8ed0d2 - 0ad613c36
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory. (cmccabe)
(cherry picked from commit f23024852502441fc259012664e444e5e51c604a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ad613c3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ad613c3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ad613c3
Branch: refs/heads/branch-2
Commit: 0ad613c369bb7ee7f23c2294799483b9eff58b30
Parents: 6cb8ed0
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 14:27:32 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 14:28:05 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/key/KeyProviderFactory.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ad613c3/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2153a59..e5a914e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -391,6 +391,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
+HDFS-7075. hadoop-fuse-dfs fails because it cannot find
+JavaKeyStoreProvider$Factory (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ad613c3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index cb63dcd..fd91284 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -46,7 +46,8 @@ public abstract class KeyProviderFactory {
) throws IOException;
private static final ServiceLoaderKeyProviderFactory serviceLoader =
- ServiceLoader.load(KeyProviderFactory.class);
+ ServiceLoader.load(KeyProviderFactory.class,
+ KeyProviderFactory.class.getClassLoader());
// Iterate through the serviceLoader to avoid lazy loading.
// Lazy loading would require synchronization in concurrent use cases.
git commit: YARN-2558. Updated ContainerTokenIdentifier#read/write to use ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
Repository: hadoop
Updated Branches:
refs/heads/trunk f23024852 - f4886111a
YARN-2558. Updated ContainerTokenIdentifier#read/write to use
ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4886111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4886111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4886111
Branch: refs/heads/trunk
Commit: f4886111aa573ec928de69e8ca9328d480bf673e
Parents: f230248
Author: Jian He jia...@apache.org
Authored: Wed Sep 17 15:12:17 2014 -0700
Committer: Jian He jia...@apache.org
Committed: Wed Sep 17 15:13:59 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/security/ContainerTokenIdentifier.java | 4 +-
.../server/TestContainerManagerSecurity.java| 92
3 files changed, 97 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 51fe3cc..bc828c6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -377,6 +377,9 @@ Release 2.6.0 - UNRELEASED
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. (Zhijie Shen via jianhe)
+YARN-2558. Updated ContainerTokenIdentifier#read/write to use
+ContainerId#getContainerId. (Tsuyoshi OZAWA via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 8b8177a..ca847e0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -128,7 +128,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
out.writeLong(applicationId.getClusterTimestamp());
out.writeInt(applicationId.getId());
out.writeInt(applicationAttemptId.getAttemptId());
-out.writeInt(this.containerId.getId());
+out.writeLong(this.containerId.getContainerId());
out.writeUTF(this.nmHostAddr);
out.writeUTF(this.appSubmitter);
out.writeInt(this.resource.getMemory());
@@ -147,7 +147,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, in.readInt());
this.containerId =
-ContainerId.newInstance(applicationAttemptId, in.readInt());
+ContainerId.newInstance(applicationAttemptId, in.readLong());
this.nmHostAddr = in.readUTF();
this.appSubmitter = in.readUTF();
int memory = in.readInt();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 6797165..9bb44ca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -28,6 +28,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.LinkedList;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteStreams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -158,6 +161,25 @@ public class TestContainerManagerSecurity extends
KerberosSecurityTestcase {
}
}
[4/4] git commit: Merge branch 'trunk' into HDFS-6581
Merge branch 'trunk' into HDFS-6581
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a186d514
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a186d514
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a186d514
Branch: refs/heads/HDFS-6581
Commit: a186d514bd0c5c5446faa6dd4896a8136d627837
Parents: 24f8156 f230248
Author: arp a...@apache.org
Authored: Wed Sep 17 15:03:55 2014 -0700
Committer: arp a...@apache.org
Committed: Wed Sep 17 15:03:55 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 5 ++
.../hadoop/crypto/key/KeyProviderFactory.java | 3 +-
.../java/org/apache/hadoop/fs/FileStatus.java | 9 ++
.../hadoop/fs/permission/FsPermission.java | 7 ++
.../src/site/markdown/filesystem/filesystem.md | 31 +++
.../fs/contract/AbstractContractOpenTest.java | 12 +++
.../hadoop/hdfs/protocol/FsAclPermission.java | 77 -
.../hdfs/protocol/FsPermissionExtension.java| 89
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 +-
.../hdfs/server/namenode/FSDirectory.java | 36 +---
.../org/apache/hadoop/hdfs/web/JsonUtil.java| 16 +++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 88 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 5 +-
13 files changed, 287 insertions(+), 95 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a186d514/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a186d514/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a186d514/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
--
diff --cc
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index bf6c25e,56105d9..8ea653a
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@@ -2315,14 -2313,22 +2315,24 @@@ public class FSDirectory implements Clo
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
+ boolean isLazyPersist = false;
+ final boolean isEncrypted;
+
+ final FileEncryptionInfo feInfo = isRawPath ? null :
+ getFileEncryptionInfo(node, snapshot);
+
if (node.isFile()) {
final INodeFile fileNode = node.asFile();
size = fileNode.computeFileSize(snapshot);
replication = fileNode.getFileReplication(snapshot);
blocksize = fileNode.getPreferredBlockSize();
+ isLazyPersist = fileNode.getLazyPersistFlag();
+isEncrypted = (feInfo != null) ||
+(isRawPath isInAnEZ(INodesInPath.fromINode(node)));
+ } else {
+isEncrypted = isInAnEZ(INodesInPath.fromINode(node));
}
+
int childrenNum = node.isDirectory() ?
node.asDirectory().getChildrenNum(snapshot) : 0;
@@@ -2334,10 -2337,9 +2341,10 @@@
node.isDirectory(),
replication,
blocksize,
+isLazyPersist,
node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
- getPermissionForFileStatus(node, snapshot),
+ getPermissionForFileStatus(node, snapshot, isEncrypted),
node.getUserName(snapshot),
node.getGroupName(snapshot),
node.isSymlink() ? node.asSymlink().getSymlink() : null,
@@@ -2356,8 -2358,8 +2363,9 @@@
long size = 0; // length is zero for directories
short replication = 0;
long blocksize = 0;
+boolean isLazyPersist = false;
LocatedBlocks loc = null;
+ final boolean isEncrypted;
final FileEncryptionInfo feInfo = isRawPath ? null :
getFileEncryptionInfo(node, snapshot);
if (node.isFile()) {
@@@ -2383,9 -2389,9 +2395,9 @@@
HdfsLocatedFileStatus status =
new HdfsLocatedFileStatus(size, node.isDirectory(), replication,
- blocksize, node.getModificationTime(snapshot),
+ blocksize, isLazyPersist, node.getModificationTime(snapshot),
node.getAccessTime(snapshot),
- getPermissionForFileStatus(node, snapshot),
+ getPermissionForFileStatus(node,
[2/4] git commit: HDFS-6843. Add to CHANGES.txt
HDFS-6843. Add to CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f24ac429 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f24ac429 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f24ac429 Branch: refs/heads/HDFS-6581 Commit: f24ac429d102777fe021e9852cfff38312643512 Parents: e3803d0 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Wed Sep 17 13:38:11 2014 -0700 Committer: Colin Patrick Mccabe cmcc...@cloudera.com Committed: Wed Sep 17 13:38:11 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24ac429/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a1dca66..8cb6c8d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -530,6 +530,8 @@ Release 2.6.0 - UNRELEASED HADOOP-10922. User documentation for CredentialShell. (Larry McCay via wang) +HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
[1/4] git commit: HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 24f815688 - a186d514b
HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3803d00
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3803d00
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3803d00
Branch: refs/heads/HDFS-6581
Commit: e3803d002c660f18a5c2ecf32344fd6f3f491a5b
Parents: ea4e2e8
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 12:55:35 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 12:55:35 2014 -0700
--
.../java/org/apache/hadoop/fs/FileStatus.java | 9 ++
.../hadoop/fs/permission/FsPermission.java | 7 ++
.../src/site/markdown/filesystem/filesystem.md | 31 +++
.../fs/contract/AbstractContractOpenTest.java | 12 +++
.../hadoop/hdfs/protocol/FsAclPermission.java | 77 -
.../hdfs/protocol/FsPermissionExtension.java| 89
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 +-
.../hdfs/server/namenode/FSDirectory.java | 36 +---
.../org/apache/hadoop/hdfs/web/JsonUtil.java| 16 +++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 88 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 5 +-
11 files changed, 280 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index b261f7f..da3807d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -200,6 +200,15 @@ public class FileStatus implements Writable, Comparable {
public FsPermission getPermission() {
return permission;
}
+
+ /**
+ * Tell whether the underlying file or directory is encrypted or not.
+ *
+ * @return true if the underlying file is encrypted.
+ */
+ public boolean isEncrypted() {
+return permission.getEncryptedBit();
+ }
/**
* Get the owner of the file.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index ee84437..264a095 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -294,6 +294,13 @@ public class FsPermission implements Writable {
return false;
}
+ /**
+ * Returns true if the file is encrypted or directory is in an encryption
zone
+ */
+ public boolean getEncryptedBit() {
+return false;
+ }
+
/** Set the user file creation mask (umask) */
public static void setUMask(Configuration conf, FsPermission umask) {
conf.set(UMASK_LABEL, String.format(%1$03o, umask.toShort()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
--
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 70796cc..e59fa1b 100644
---
a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++
b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -64,6 +64,33 @@ all operations on a valid FileSystem MUST result in a new
FileSystem that is als
def isSymlink(FS, p) = p in symlinks(FS)
+### 'boolean inEncryptionZone(Path p)'
+
+Return True if the data for p is encrypted. The nature of the encryption and
the
+mechanism for creating an encryption zone are implementation details not
covered
+in this specification. No guarantees are made about the quality of the
+encryption. The metadata is not encrypted.
+
+ Preconditions
+
+if not exists(FS, p) : raise FileNotFoundException
+
+ Postconditions
+
+ Invariants
+
+All files and directories
[3/4] git commit: HDFS-7075. hadoop-fuse-dfs fails because it cannot find JavaKeyStoreProvider$Factory. (cmccabe)
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory. (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2302485
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2302485
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2302485
Branch: refs/heads/HDFS-6581
Commit: f23024852502441fc259012664e444e5e51c604a
Parents: f24ac42
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 14:27:32 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 14:27:32 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/key/KeyProviderFactory.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2302485/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8cb6c8d..31c09de 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -726,6 +726,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
+HDFS-7075. hadoop-fuse-dfs fails because it cannot find
+JavaKeyStoreProvider$Factory (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2302485/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index 6ca0425..ce99d79 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -46,7 +46,8 @@ public abstract class KeyProviderFactory {
) throws IOException;
private static final ServiceLoaderKeyProviderFactory serviceLoader =
- ServiceLoader.load(KeyProviderFactory.class);
+ ServiceLoader.load(KeyProviderFactory.class,
+ KeyProviderFactory.class.getClassLoader());
// Iterate through the serviceLoader to avoid lazy loading.
// Lazy loading would require synchronization in concurrent use cases.
[1/2] git commit: YARN-2558. Updated ContainerTokenIdentifier#read/write to use ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 a186d514b - 900f6e52e
YARN-2558. Updated ContainerTokenIdentifier#read/write to use
ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4886111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4886111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4886111
Branch: refs/heads/HDFS-6581
Commit: f4886111aa573ec928de69e8ca9328d480bf673e
Parents: f230248
Author: Jian He jia...@apache.org
Authored: Wed Sep 17 15:12:17 2014 -0700
Committer: Jian He jia...@apache.org
Committed: Wed Sep 17 15:13:59 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/security/ContainerTokenIdentifier.java | 4 +-
.../server/TestContainerManagerSecurity.java| 92
3 files changed, 97 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 51fe3cc..bc828c6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -377,6 +377,9 @@ Release 2.6.0 - UNRELEASED
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. (Zhijie Shen via jianhe)
+YARN-2558. Updated ContainerTokenIdentifier#read/write to use
+ContainerId#getContainerId. (Tsuyoshi OZAWA via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 8b8177a..ca847e0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -128,7 +128,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
out.writeLong(applicationId.getClusterTimestamp());
out.writeInt(applicationId.getId());
out.writeInt(applicationAttemptId.getAttemptId());
-out.writeInt(this.containerId.getId());
+out.writeLong(this.containerId.getContainerId());
out.writeUTF(this.nmHostAddr);
out.writeUTF(this.appSubmitter);
out.writeInt(this.resource.getMemory());
@@ -147,7 +147,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, in.readInt());
this.containerId =
-ContainerId.newInstance(applicationAttemptId, in.readInt());
+ContainerId.newInstance(applicationAttemptId, in.readLong());
this.nmHostAddr = in.readUTF();
this.appSubmitter = in.readUTF();
int memory = in.readInt();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 6797165..9bb44ca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -28,6 +28,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.LinkedList;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteStreams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -158,6 +161,25 @@ public class TestContainerManagerSecurity extends
KerberosSecurityTestcase {
}
[2/2] git commit: Merge branch 'trunk' into HDFS-6581
Merge branch 'trunk' into HDFS-6581 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/900f6e52 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/900f6e52 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/900f6e52 Branch: refs/heads/HDFS-6581 Commit: 900f6e52ec33ca82edb0d306d0bfecc1ba4d60e6 Parents: a186d51 f488611 Author: arp a...@apache.org Authored: Wed Sep 17 15:19:43 2014 -0700 Committer: arp a...@apache.org Committed: Wed Sep 17 15:19:43 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 + .../yarn/security/ContainerTokenIdentifier.java | 4 +- .../server/TestContainerManagerSecurity.java| 92 3 files changed, 97 insertions(+), 2 deletions(-) --
git commit: YARN-2558. Updated ContainerTokenIdentifier#read/write to use ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0ad613c36 - 3746b1e90
YARN-2558. Updated ContainerTokenIdentifier#read/write to use
ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
(cherry picked from commit f4886111aa573ec928de69e8ca9328d480bf673e)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3746b1e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3746b1e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3746b1e9
Branch: refs/heads/branch-2
Commit: 3746b1e9053cf73ae1b6edc8d7aa0c4b38496fce
Parents: 0ad613c
Author: Jian He jia...@apache.org
Authored: Wed Sep 17 15:12:17 2014 -0700
Committer: Jian He jia...@apache.org
Committed: Wed Sep 17 15:22:02 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 7 +-
.../yarn/security/ContainerTokenIdentifier.java | 4 +-
.../server/TestContainerManagerSecurity.java| 92
3 files changed, 99 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3746b1e9/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f9acfc6..7d7eea0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -202,6 +202,9 @@ Release 2.6.0 - UNRELEASED
YARN-2547. Cross Origin Filter throws UnsupportedOperationException upon
destroy (Mit Desai via jeagles)
+YARN-2557. Add a parameter attempt_Failures_Validity_Interval into
+DistributedShell. (xgong)
+
OPTIMIZATIONS
BUG FIXES
@@ -344,8 +347,8 @@ Release 2.6.0 - UNRELEASED
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. (Zhijie Shen via jianhe)
-YARN-2557. Add a parameter attempt_Failures_Validity_Interval into
-DistributedShell. (xgong)
+YARN-2558. Updated ContainerTokenIdentifier#read/write to use
+ContainerId#getContainerId. (Tsuyoshi OZAWA via jianhe)
Release 2.5.1 - 2014-09-05
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3746b1e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 8b8177a..ca847e0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -128,7 +128,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
out.writeLong(applicationId.getClusterTimestamp());
out.writeInt(applicationId.getId());
out.writeInt(applicationAttemptId.getAttemptId());
-out.writeInt(this.containerId.getId());
+out.writeLong(this.containerId.getContainerId());
out.writeUTF(this.nmHostAddr);
out.writeUTF(this.appSubmitter);
out.writeInt(this.resource.getMemory());
@@ -147,7 +147,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, in.readInt());
this.containerId =
-ContainerId.newInstance(applicationAttemptId, in.readInt());
+ContainerId.newInstance(applicationAttemptId, in.readLong());
this.nmHostAddr = in.readUTF();
this.appSubmitter = in.readUTF();
int memory = in.readInt();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3746b1e9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 6797165..9bb44ca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++
git commit: HDFS-7080. Fix finalize and upgrade unit test failures. (Arpit Agarwal)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6581 900f6e52e - 4eab083b1
HDFS-7080. Fix finalize and upgrade unit test failures. (Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eab083b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eab083b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eab083b
Branch: refs/heads/HDFS-6581
Commit: 4eab083b1b7faf4485274d1d30256cde08e11915
Parents: 900f6e5
Author: arp a...@apache.org
Authored: Wed Sep 17 15:25:04 2014 -0700
Committer: arp a...@apache.org
Committed: Wed Sep 17 15:25:04 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt | 2 ++
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../server/datanode/fsdataset/impl/BlockPoolSlice.java| 10 ++
.../test/java/org/apache/hadoop/hdfs/TestDFSFinalize.java | 3 +++
.../test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java | 3 +++
5 files changed, 23 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eab083b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
index e1f51c1..98c0bca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-6581.txt
@@ -52,4 +52,6 @@
HDFS-6581. Few more unit test fixes for HDFS-6581. (Arpit Agarwal)
+HDFS-7080. Fix finalize and upgrade unit test failures. (Arpit Agarwal)
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eab083b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c5d8bd2..ea9efcf 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -133,6 +133,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final int
DFS_DATANODE_RAM_DISK_LOW_WATERMARK_PERCENT_DEFAULT = 10;
public static final String DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS =
dfs.datanode.ram.disk.low.watermark.replicas;
public static final int
DFS_DATANODE_RAM_DISK_LOW_WATERMARK_REPLICAS_DEFAULT = 3;
+
+ // This setting is for testing/internal use only.
+ public static final String DFS_DATANODE_DUPLICATE_REPLICA_DELETION =
dfs.datanode.duplicate.replica.deletion;
+ public static final boolean DFS_DATANODE_DUPLICATE_REPLICA_DELETION_DEFAULT
= true;
+
public static final String
DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT =
dfs.namenode.path.based.cache.block.map.allocation.percent;
public static final float
DFS_NAMENODE_PATH_BASED_CACHE_BLOCK_MAP_ALLOCATION_PERCENT_DEFAULT = 0.25f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eab083b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index f39ca16..a4bcc3e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -68,6 +68,7 @@ class BlockPoolSlice {
private static final String DU_CACHE_FILE = dfsUsed;
private volatile boolean dfsUsedSaved = false;
private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private final boolean deleteDuplicateReplicas;
// TODO:FEDERATION scalability issue - a thread per DU is needed
private final DU dfsUsage;
@@ -94,6 +95,10 @@ class BlockPoolSlice {
}
}
+this.deleteDuplicateReplicas = conf.getBoolean(
+DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION,
+DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION_DEFAULT);
+
// Files that were being written when the datanode was last shutdown
// are now moved back to the data directory. It is possible that
// in the future, we might want to do some sort of datanode-local
@@ -509,6
git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
Repository: hadoop Updated Branches: refs/heads/trunk f4886111a - 123f20d42 HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/123f20d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/123f20d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/123f20d4 Branch: refs/heads/trunk Commit: 123f20d42f6acffcde05392d689acd91a82462db Parents: f488611 Author: Alejandro Abdelnur t...@apache.org Authored: Wed Sep 17 14:27:35 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 15:29:17 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 7 files changed, 373 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 31c09de..d2671c3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -532,6 +532,9 @@ Release 2.6.0 - UNRELEASED HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) +HADOOP-11016. KMS should support signing cookies with zookeeper secret +manager. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 2c225cb..e6b21aa 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency +dependency + groupIdorg.apache.curator/groupId + artifactIdcurator-test/artifactId + scopetest/scope +/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/123f20d4/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ /description /property + !-- Authentication cookie signature source -- + + property +namehadoop.kms.authentication.signer.secret.provider/name +valuerandom/value +description + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. +/description + /property + + !-- Configuration for 'zookeeper' authentication cookie signature source -- + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name +value/hadoop-kms/hadoop-auth-signature-secret/value +description + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name +value#HOSTNAME#:#PORT#,.../value +description + The Zookeeper connection string, a list of hostnames and port comma + separated. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name +valuekerberos/value +description + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name +value/etc/hadoop/conf/kms.keytab/value +description + The absolute path for the Kerberos keytab with the credentials to + connect to Zookeeper. +/description + /property + + property +
git commit: HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu)
Repository: hadoop Updated Branches: refs/heads/branch-2 3746b1e90 - d3efebf4a HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) (cherry picked from commit 123f20d42f6acffcde05392d689acd91a82462db) Conflicts: hadoop-common-project/hadoop-common/CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3efebf4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3efebf4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3efebf4 Branch: refs/heads/branch-2 Commit: d3efebf4aaf4a8da602c9f134d5b0f9cf0b8b5b7 Parents: 3746b1e Author: Alejandro Abdelnur t...@apache.org Authored: Wed Sep 17 14:27:35 2014 -0700 Committer: Alejandro Abdelnur t...@apache.org Committed: Wed Sep 17 15:30:56 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 57 ++ .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../hadoop-kms/src/site/apt/index.apt.vm| 161 + .../hadoop/crypto/key/kms/server/TestKMS.java | 5 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ 7 files changed, 373 insertions(+), 44 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index e5a914e..6661bfb 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -194,6 +194,9 @@ Release 2.6.0 - UNRELEASED HADOOP-10922. User documentation for CredentialShell. (Larry McCay via wang) +HADOOP-11016. KMS should support signing cookies with zookeeper secret +manager. (tucu) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-kms/pom.xml -- diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml index 37dcb2c..9de5c45 100644 --- a/hadoop-common-project/hadoop-kms/pom.xml +++ b/hadoop-common-project/hadoop-kms/pom.xml @@ -187,6 +187,11 @@ artifactIdmetrics-core/artifactId scopecompile/scope /dependency +dependency + groupIdorg.apache.curator/groupId + artifactIdcurator-test/artifactId + scopetest/scope +/dependency /dependencies build http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3efebf4/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml -- diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml index 20896fc..f55ce5f 100644 --- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml +++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml @@ -68,4 +68,61 @@ /description /property + !-- Authentication cookie signature source -- + + property +namehadoop.kms.authentication.signer.secret.provider/name +valuerandom/value +description + Indicates how the secret to sign the authentication cookies will be + stored. Options are 'random' (default), 'string' and 'zookeeper'. + If using a setup with multiple KMS instances, 'zookeeper' should be used. +/description + /property + + !-- Configuration for 'zookeeper' authentication cookie signature source -- + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.path/name +value/hadoop-kms/hadoop-auth-signature-secret/value +description + The Zookeeper ZNode path where the KMS instances will store and retrieve + the secret from. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string/name +value#HOSTNAME#:#PORT#,.../value +description + The Zookeeper connection string, a list of hostnames and port comma + separated. +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type/name +valuekerberos/value +description + The Zookeeper authentication type, 'none' or 'sasl' (Kerberos). +/description + /property + + property + namehadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab/name +value/etc/hadoop/conf/kms.keytab/value +description + The absolute path for the Kerberos keytab with
git commit: HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is incorrect in some cases. (Yi Liu via wang)
Repository: hadoop
Updated Branches:
refs/heads/trunk 123f20d42 - 47e5e1983
HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is
incorrect in some cases. (Yi Liu via wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47e5e198
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47e5e198
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47e5e198
Branch: refs/heads/trunk
Commit: 47e5e19831a363aa4d675fd23ab0d06e86809094
Parents: 123f20d
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 17:58:56 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 17:58:56 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/crypto/CryptoInputStream.java | 11 ++-
.../hadoop/crypto/CryptoStreamsTestBase.java | 18 ++
3 files changed, 27 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47e5e198/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d2671c3..f2b4180 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -732,6 +732,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory (cmccabe)
+HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is
+incorrect in some cases. (Yi Liu via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47e5e198/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index e8964ed..68e9697 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -471,7 +471,16 @@ public class CryptoInputStream extends FilterInputStream
implements
streamOffset += n; // Read n bytes
decrypt(buf, n, pos);
}
- return n;
+
+ if (n = 0) {
+return unread + n;
+ } else {
+if (unread == 0) {
+ return -1;
+} else {
+ return unread;
+}
+ }
}
throw new UnsupportedOperationException(ByteBuffer read unsupported +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47e5e198/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
index f5acc73..86bb64d 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
@@ -469,6 +469,7 @@ public abstract class CryptoStreamsTestBase {
int bufPos) throws Exception {
buf.position(bufPos);
int n = ((ByteBufferReadable) in).read(buf);
+Assert.assertEquals(bufPos + n, buf.position());
byte[] readData = new byte[n];
buf.rewind();
buf.position(bufPos);
@@ -568,6 +569,7 @@ public abstract class CryptoStreamsTestBase {
// Read forward len1
ByteBuffer buf = ByteBuffer.allocate(len1);
int nRead = ((ByteBufferReadable) in).read(buf);
+Assert.assertEquals(nRead, buf.position());
readData = new byte[nRead];
buf.rewind();
buf.get(readData);
@@ -575,9 +577,10 @@ public abstract class CryptoStreamsTestBase {
System.arraycopy(data, (int)pos, expectedData, 0, nRead);
Assert.assertArrayEquals(readData, expectedData);
-// Pos should be len1 + 2 * len2 + nRead
+long lastPos = pos;
+// Pos should be lastPos + nRead
pos = ((Seekable) in).getPos();
-Assert.assertEquals(len1 + 2 * len2 + nRead, pos);
+Assert.assertEquals(lastPos + nRead, pos);
// Pos: 1/3 dataLen
positionedReadCheck(in , dataLen / 3);
@@ -589,13
git commit: HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is incorrect in some cases. (Yi Liu via wang)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d3efebf4a - a3aab30d6
HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is
incorrect in some cases. (Yi Liu via wang)
(cherry picked from commit 47e5e19831a363aa4d675fd23ab0d06e86809094)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3aab30d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3aab30d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3aab30d
Branch: refs/heads/branch-2
Commit: a3aab30d6fefd9493248d9b37dd9097f4c628f9a
Parents: d3efebf
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 17:58:56 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 18:03:06 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/crypto/CryptoInputStream.java | 11 ++-
.../hadoop/crypto/CryptoStreamsTestBase.java | 18 ++
3 files changed, 27 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3aab30d/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6661bfb..8d30ae8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -397,6 +397,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory (cmccabe)
+HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is
+incorrect in some cases. (Yi Liu via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3aab30d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index e8964ed..68e9697 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -471,7 +471,16 @@ public class CryptoInputStream extends FilterInputStream
implements
streamOffset += n; // Read n bytes
decrypt(buf, n, pos);
}
- return n;
+
+ if (n = 0) {
+return unread + n;
+ } else {
+if (unread == 0) {
+ return -1;
+} else {
+ return unread;
+}
+ }
}
throw new UnsupportedOperationException(ByteBuffer read unsupported +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3aab30d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
index f5acc73..86bb64d 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
@@ -469,6 +469,7 @@ public abstract class CryptoStreamsTestBase {
int bufPos) throws Exception {
buf.position(bufPos);
int n = ((ByteBufferReadable) in).read(buf);
+Assert.assertEquals(bufPos + n, buf.position());
byte[] readData = new byte[n];
buf.rewind();
buf.position(bufPos);
@@ -568,6 +569,7 @@ public abstract class CryptoStreamsTestBase {
// Read forward len1
ByteBuffer buf = ByteBuffer.allocate(len1);
int nRead = ((ByteBufferReadable) in).read(buf);
+Assert.assertEquals(nRead, buf.position());
readData = new byte[nRead];
buf.rewind();
buf.get(readData);
@@ -575,9 +577,10 @@ public abstract class CryptoStreamsTestBase {
System.arraycopy(data, (int)pos, expectedData, 0, nRead);
Assert.assertArrayEquals(readData, expectedData);
-// Pos should be len1 + 2 * len2 + nRead
+long lastPos = pos;
+// Pos should be lastPos + nRead
pos = ((Seekable) in).getPos();
-Assert.assertEquals(len1 + 2 * len2 + nRead, pos);
+Assert.assertEquals(lastPos + nRead, pos);
//
git commit: Move some HDFS JIRAs to the correct CHANGES.txt
Repository: hadoop Updated Branches: refs/heads/trunk 47e5e1983 - bf38793ce Move some HDFS JIRAs to the correct CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf38793c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf38793c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf38793c Branch: refs/heads/trunk Commit: bf38793ce169137bb3ef36e96db7ea62d89ce1c4 Parents: 47e5e19 Author: Andrew Wang w...@apache.org Authored: Wed Sep 17 18:08:34 2014 -0700 Committer: Andrew Wang w...@apache.org Committed: Wed Sep 17 18:08:34 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 8 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 8 2 files changed, 8 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf38793c/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f2b4180..0ca2953 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -530,8 +530,6 @@ Release 2.6.0 - UNRELEASED HADOOP-10922. User documentation for CredentialShell. (Larry McCay via wang) -HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) - HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) @@ -726,12 +724,6 @@ Release 2.6.0 - UNRELEASED HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors (yzhang via cmccabe) -HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files -(cmccabe) - -HDFS-7075. hadoop-fuse-dfs fails because it cannot find -JavaKeyStoreProvider$Factory (cmccabe) - HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is incorrect in some cases. (Yi Liu via wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf38793c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 567a6ab..0e01ca0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -471,6 +471,8 @@ Release 2.6.0 - UNRELEASED HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file. (clamb via wang) +HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) @@ -670,6 +672,12 @@ Release 2.6.0 - UNRELEASED and TestDFSClientFailover.testDoesntDnsResolveLogicalURI failing on jdk7. (Akira Ajisaka via wang) +HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files +(cmccabe) + +HDFS-7075. hadoop-fuse-dfs fails because it cannot find +JavaKeyStoreProvider$Factory (cmccabe) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating deleting an
git commit: HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
Repository: hadoop
Updated Branches:
refs/heads/trunk bf38793ce - 10e8602f3
HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10e8602f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10e8602f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10e8602f
Branch: refs/heads/trunk
Commit: 10e8602f32b553a1424f1a9b5f9f74f7b68a49d1
Parents: bf38793
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 20:14:40 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 20:14:40 2014 -0700
--
.../hadoop-kms/src/main/conf/kms-site.xml | 2 +-
.../crypto/key/kms/server/KMSConfiguration.java | 4 ++
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 14 +++
.../hadoop-kms/src/site/apt/index.apt.vm| 2 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 2 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java| 41 +++-
.../src/main/resources/hdfs-default.xml | 8
.../src/site/apt/TransparentEncryption.apt.vm | 6 +++
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 2 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +++--
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 3 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 3 +-
15 files changed, 61 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e8602f/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
--
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index f55ce5f..4f4694c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -16,7 +16,7 @@
!-- KMS Backend KeyProvider --
property
-namehadoop.security.key.provider.path/name
+namehadoop.kms.key.provider.uri/name
valuejceks://file@/${user.home}/kms.keystore/value
description
/description
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e8602f/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index f028119..c9b0491 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -40,6 +40,10 @@ public class KMSConfiguration {
public static final String KEY_ACL_PREFIX = key.acl.;
public static final String DEFAULT_KEY_ACL_PREFIX = default.key.acl.;
+ // Property to set the backing KeyProvider
+ public static final String KEY_PROVIDER_URI = CONFIG_PREFIX +
+ key.provider.uri;
+
// Property to Enable/Disable Caching
public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX +
cache.enable;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e8602f/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index 0827b78..c9eeb1d 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -39,6 +39,7 @@ import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.io.File;
+import java.net.URI;
import java.net.URL;
import java.util.List;
@@ -159,17 +160,12 @@ public class KMSWebApp implements ServletContextListener {
new AccessControlList(AccessControlList.WILDCARD_ACL_VALUE));
// intializing the KeyProvider
-
- ListKeyProvider providers = KeyProviderFactory.getProviders(kmsConf);
- if (providers.isEmpty()) {
+ String providerString = kmsConf.get(KMSConfiguration.KEY_PROVIDER_URI);
+ if (providerString == null) {
[1/2] git commit: Move some HDFS JIRAs to the correct CHANGES.txt
Repository: hadoop Updated Branches: refs/heads/branch-2 a3aab30d6 - b477d30e6 Move some HDFS JIRAs to the correct CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b05da10d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b05da10d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b05da10d Branch: refs/heads/branch-2 Commit: b05da10d0d03b8ab212443a9713e1620efa1895c Parents: a3aab30 Author: Andrew Wang w...@apache.org Authored: Wed Sep 17 18:08:24 2014 -0700 Committer: Andrew Wang w...@apache.org Committed: Wed Sep 17 18:08:24 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 8 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 8 2 files changed, 8 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b05da10d/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 8d30ae8..e710739 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -251,8 +251,6 @@ Release 2.6.0 - UNRELEASED HADOOP-10833. Remove unused cache in UserProvider. (Benoy Antony) -HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) - BUG FIXES HADOOP-10781. Unportable getgrouplist() usage breaks FreeBSD (Dmitry @@ -391,12 +389,6 @@ Release 2.6.0 - UNRELEASED HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors. (yzhang via cmccabe) -HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files -(cmccabe) - -HDFS-7075. hadoop-fuse-dfs fails because it cannot find -JavaKeyStoreProvider$Factory (cmccabe) - HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is incorrect in some cases. (Yi Liu via wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b05da10d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1329ac6..0b8c359 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -213,6 +213,8 @@ Release 2.6.0 - UNRELEASED HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file. (clamb via wang) +HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) @@ -412,6 +414,12 @@ Release 2.6.0 - UNRELEASED and TestDFSClientFailover.testDoesntDnsResolveLogicalURI failing on jdk7. (Akira Ajisaka via wang) +HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files +(cmccabe) + +HDFS-7075. hadoop-fuse-dfs fails because it cannot find +JavaKeyStoreProvider$Factory (cmccabe) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating deleting an
[2/2] git commit: HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
(cherry picked from commit 10e8602f32b553a1424f1a9b5f9f74f7b68a49d1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b477d30e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b477d30e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b477d30e
Branch: refs/heads/branch-2
Commit: b477d30e63bfb4ce9a660f5ffe88801758e7a985
Parents: b05da10
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 20:14:40 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 20:15:42 2014 -0700
--
.../hadoop-kms/src/main/conf/kms-site.xml | 2 +-
.../crypto/key/kms/server/KMSConfiguration.java | 4 ++
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 14 +++
.../hadoop-kms/src/site/apt/index.apt.vm| 2 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 2 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java| 41 +++-
.../src/main/resources/hdfs-default.xml | 8
.../src/site/apt/TransparentEncryption.apt.vm | 6 +++
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 2 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +++--
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 3 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 3 +-
15 files changed, 61 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b477d30e/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
--
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index f55ce5f..4f4694c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -16,7 +16,7 @@
!-- KMS Backend KeyProvider --
property
-namehadoop.security.key.provider.path/name
+namehadoop.kms.key.provider.uri/name
valuejceks://file@/${user.home}/kms.keystore/value
description
/description
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b477d30e/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index 76fb40c..56123f9 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -40,6 +40,10 @@ public class KMSConfiguration {
public static final String KEY_ACL_PREFIX = key.acl.;
public static final String DEFAULT_KEY_ACL_PREFIX = default.key.acl.;
+ // Property to set the backing KeyProvider
+ public static final String KEY_PROVIDER_URI = CONFIG_PREFIX +
+ key.provider.uri;
+
// Property to Enable/Disable Caching
public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX +
cache.enable;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b477d30e/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index e90c3ee..c36823a 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -39,6 +39,7 @@ import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.io.File;
+import java.net.URI;
import java.net.URL;
import java.util.List;
@@ -159,17 +160,12 @@ public class KMSWebApp implements ServletContextListener {
new AccessControlList(AccessControlList.WILDCARD_ACL_VALUE));
// intializing the KeyProvider
-
- ListKeyProvider providers = KeyProviderFactory.getProviders(kmsConf);
- if (providers.isEmpty()) {
+ String providerString = kmsConf.get(KMSConfiguration.KEY_PROVIDER_URI);
+ if (providerString == null) {
git commit: HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
Repository: hadoop
Updated Branches:
refs/heads/trunk 10e8602f3 - 0ecefe601
HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ecefe60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ecefe60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ecefe60
Branch: refs/heads/trunk
Commit: 0ecefe60179968984b1892a14411566b7a0c8df3
Parents: 10e8602
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 21:28:05 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 21:28:05 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../server/namenode/EncryptionZoneManager.java | 17 -
.../apache/hadoop/hdfs/TestEncryptionZones.java | 38 +---
3 files changed, 52 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecefe60/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7527463..26d5652 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -680,6 +680,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory (cmccabe)
+HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecefe60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index e72ae12..c428690 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -312,7 +312,22 @@ public class EncryptionZoneManager {
int count = 0;
for (EncryptionZoneInt ezi : tailMap.values()) {
- zones.add(new EncryptionZone(getFullPathName(ezi),
+ /*
+ Skip EZs that are only present in snapshots. Re-resolve the path to
+ see if the path's current inode ID matches EZ map's INode ID.
+
+ INode#getFullPathName simply calls getParent recursively, so will return
+ the INode's parents at the time it was snapshotted. It will not
+ contain a reference INode.
+ */
+ final String pathName = getFullPathName(ezi);
+ INodesInPath iip = dir.getINodesInPath(pathName, false);
+ INode lastINode = iip.getLastINode();
+ if (lastINode == null || lastINode.getId() != ezi.getINodeId()) {
+continue;
+ }
+ // Add the EZ to the result list
+ zones.add(new EncryptionZone(pathName,
ezi.getKeyName(), ezi.getINodeId()));
count++;
if (count = numResponses) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecefe60/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b4f6c1c..ff28200 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -27,6 +27,7 @@ import java.io.StringReader;
import java.io.StringWriter;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
@@ -1030,6 +1031,9 @@ public class TestEncryptionZones {
*/
@Test(timeout = 6)
public void testSnapshotsOnEncryptionZones() throws Exception {
+final String TEST_KEY2 = testkey2;
+DFSTestUtil.createKey(TEST_KEY2, cluster, conf);
+
final int len = 8196;
final Path zoneParent = new Path(/zones);
final Path zone = new Path(zoneParent, zone);
@@ -1044,7 +1048,8 @@ public class TestEncryptionZones {
assertEquals(Got unexpected ez path,
git commit: HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b477d30e6 - 008e2f68f
HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
(cherry picked from commit 0ecefe60179968984b1892a14411566b7a0c8df3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/008e2f68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/008e2f68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/008e2f68
Branch: refs/heads/branch-2
Commit: 008e2f68f1a929f6fcaa5ae71ccd0eeac8ecdf95
Parents: b477d30
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 21:28:05 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 21:28:15 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../server/namenode/EncryptionZoneManager.java | 17 -
.../apache/hadoop/hdfs/TestEncryptionZones.java | 38 +---
3 files changed, 52 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/008e2f68/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d9293c5..32f1df3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -422,6 +422,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory (cmccabe)
+HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/008e2f68/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index e72ae12..c428690 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -312,7 +312,22 @@ public class EncryptionZoneManager {
int count = 0;
for (EncryptionZoneInt ezi : tailMap.values()) {
- zones.add(new EncryptionZone(getFullPathName(ezi),
+ /*
+ Skip EZs that are only present in snapshots. Re-resolve the path to
+ see if the path's current inode ID matches EZ map's INode ID.
+
+ INode#getFullPathName simply calls getParent recursively, so will return
+ the INode's parents at the time it was snapshotted. It will not
+ contain a reference INode.
+ */
+ final String pathName = getFullPathName(ezi);
+ INodesInPath iip = dir.getINodesInPath(pathName, false);
+ INode lastINode = iip.getLastINode();
+ if (lastINode == null || lastINode.getId() != ezi.getINodeId()) {
+continue;
+ }
+ // Add the EZ to the result list
+ zones.add(new EncryptionZone(pathName,
ezi.getKeyName(), ezi.getINodeId()));
count++;
if (count = numResponses) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/008e2f68/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b4f6c1c..ff28200 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -27,6 +27,7 @@ import java.io.StringReader;
import java.io.StringWriter;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
@@ -1030,6 +1031,9 @@ public class TestEncryptionZones {
*/
@Test(timeout = 6)
public void testSnapshotsOnEncryptionZones() throws Exception {
+final String TEST_KEY2 = testkey2;
+DFSTestUtil.createKey(TEST_KEY2, cluster, conf);
+
final int len = 8196;
final Path zoneParent = new Path(/zones);
final Path zone = new Path(zoneParent, zone);
@@ -1044,7 +1048,8 @@ public
git commit: YARN-2559. Fixed NPE in SystemMetricsPublisher when retrieving FinalApplicationStatus. Contributed by Zhijie Shen
Repository: hadoop
Updated Branches:
refs/heads/trunk 0ecefe601 - ee21b13cb
YARN-2559. Fixed NPE in SystemMetricsPublisher when retrieving
FinalApplicationStatus. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee21b13c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee21b13c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee21b13c
Branch: refs/heads/trunk
Commit: ee21b13cbd4654d7181306404174329f12193613
Parents: 0ecefe6
Author: Jian He jia...@apache.org
Authored: Wed Sep 17 21:44:15 2014 -0700
Committer: Jian He jia...@apache.org
Committed: Wed Sep 17 21:44:15 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/metrics/SystemMetricsPublisher.java | 8 +---
.../resourcemanager/rmapp/attempt/RMAppAttemptImpl.java | 6 --
.../resourcemanager/metrics/TestSystemMetricsPublisher.java | 8
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 5 ++---
5 files changed, 18 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21b13c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bc828c6..5a23814 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -380,6 +380,9 @@ Release 2.6.0 - UNRELEASED
YARN-2558. Updated ContainerTokenIdentifier#read/write to use
ContainerId#getContainerId. (Tsuyoshi OZAWA via jianhe)
+YARN-2559. Fixed NPE in SystemMetricsPublisher when retrieving
+FinalApplicationStatus. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21b13c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index ecf37b0..5da006c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -160,7 +160,7 @@ public class SystemMetricsPublisher extends
CompositeService {
@SuppressWarnings(unchecked)
public void appAttemptFinished(RMAppAttempt appAttempt,
- RMAppAttemptState state, long finishedTime) {
+ RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
@@ -168,8 +168,10 @@ public class SystemMetricsPublisher extends
CompositeService {
appAttempt.getTrackingUrl(),
appAttempt.getOriginalTrackingUrl(),
appAttempt.getDiagnostics(),
- appAttempt.getFinalApplicationStatus(),
- RMServerUtils.createApplicationAttemptState(state),
+ // app will get the final status from app attempt, or create one
+ // based on app state if it doesn't exist
+ app.getFinalApplicationStatus(),
+ RMServerUtils.createApplicationAttemptState(appAttemtpState),
finishedTime));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21b13c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 863130f..7ca57ee 100644
---
[09/11] git commit: HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0ecefe60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0ecefe60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0ecefe60
Branch: refs/heads/HDFS-6584
Commit: 0ecefe60179968984b1892a14411566b7a0c8df3
Parents: 10e8602
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 21:28:05 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 21:28:05 2014 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../server/namenode/EncryptionZoneManager.java | 17 -
.../apache/hadoop/hdfs/TestEncryptionZones.java | 38 +---
3 files changed, 52 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecefe60/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7527463..26d5652 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -680,6 +680,8 @@ Release 2.6.0 - UNRELEASED
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory (cmccabe)
+HDFS-7078. Fix listEZs to work correctly with snapshots. (wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HDFS-6387. HDFS CLI admin tool for creating deleting an
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecefe60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index e72ae12..c428690 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -312,7 +312,22 @@ public class EncryptionZoneManager {
int count = 0;
for (EncryptionZoneInt ezi : tailMap.values()) {
- zones.add(new EncryptionZone(getFullPathName(ezi),
+ /*
+ Skip EZs that are only present in snapshots. Re-resolve the path to
+ see if the path's current inode ID matches EZ map's INode ID.
+
+ INode#getFullPathName simply calls getParent recursively, so will return
+ the INode's parents at the time it was snapshotted. It will not
+ contain a reference INode.
+ */
+ final String pathName = getFullPathName(ezi);
+ INodesInPath iip = dir.getINodesInPath(pathName, false);
+ INode lastINode = iip.getLastINode();
+ if (lastINode == null || lastINode.getId() != ezi.getINodeId()) {
+continue;
+ }
+ // Add the EZ to the result list
+ zones.add(new EncryptionZone(pathName,
ezi.getKeyName(), ezi.getINodeId()));
count++;
if (count = numResponses) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0ecefe60/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b4f6c1c..ff28200 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -27,6 +27,7 @@ import java.io.StringReader;
import java.io.StringWriter;
import java.net.URI;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.concurrent.Callable;
@@ -1030,6 +1031,9 @@ public class TestEncryptionZones {
*/
@Test(timeout = 6)
public void testSnapshotsOnEncryptionZones() throws Exception {
+final String TEST_KEY2 = testkey2;
+DFSTestUtil.createKey(TEST_KEY2, cluster, conf);
+
final int len = 8196;
final Path zoneParent = new Path(/zones);
final Path zone = new Path(zoneParent, zone);
@@ -1044,7 +1048,8 @@ public class TestEncryptionZones {
assertEquals(Got unexpected ez path, zone.toString(),
[04/11] git commit: YARN-2558. Updated ContainerTokenIdentifier#read/write to use ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
YARN-2558. Updated ContainerTokenIdentifier#read/write to use
ContainerId#getContainerId. Contributed by Tsuyoshi OZAWA.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4886111
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4886111
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4886111
Branch: refs/heads/HDFS-6584
Commit: f4886111aa573ec928de69e8ca9328d480bf673e
Parents: f230248
Author: Jian He jia...@apache.org
Authored: Wed Sep 17 15:12:17 2014 -0700
Committer: Jian He jia...@apache.org
Committed: Wed Sep 17 15:13:59 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/security/ContainerTokenIdentifier.java | 4 +-
.../server/TestContainerManagerSecurity.java| 92
3 files changed, 97 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 51fe3cc..bc828c6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -377,6 +377,9 @@ Release 2.6.0 - UNRELEASED
YARN-2529. Generic history service RPC interface doesn't work when service
authorization is enabled. (Zhijie Shen via jianhe)
+YARN-2558. Updated ContainerTokenIdentifier#read/write to use
+ContainerId#getContainerId. (Tsuyoshi OZAWA via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index 8b8177a..ca847e0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -128,7 +128,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
out.writeLong(applicationId.getClusterTimestamp());
out.writeInt(applicationId.getId());
out.writeInt(applicationAttemptId.getAttemptId());
-out.writeInt(this.containerId.getId());
+out.writeLong(this.containerId.getContainerId());
out.writeUTF(this.nmHostAddr);
out.writeUTF(this.appSubmitter);
out.writeInt(this.resource.getMemory());
@@ -147,7 +147,7 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
ApplicationAttemptId applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, in.readInt());
this.containerId =
-ContainerId.newInstance(applicationAttemptId, in.readInt());
+ContainerId.newInstance(applicationAttemptId, in.readLong());
this.nmHostAddr = in.readUTF();
this.appSubmitter = in.readUTF();
int memory = in.readInt();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4886111/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index 6797165..9bb44ca 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -28,6 +28,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.LinkedList;
+import com.google.common.io.ByteArrayDataInput;
+import com.google.common.io.ByteStreams;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -158,6 +161,25 @@ public class TestContainerManagerSecurity extends
KerberosSecurityTestcase {
}
}
}
+
+ @Test (timeout = 50)
+ public void
[08/11] git commit: HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
HDFS-7004. Update KeyProvider instantiation to create by URI. (wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10e8602f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10e8602f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10e8602f
Branch: refs/heads/HDFS-6584
Commit: 10e8602f32b553a1424f1a9b5f9f74f7b68a49d1
Parents: bf38793
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 20:14:40 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 20:14:40 2014 -0700
--
.../hadoop-kms/src/main/conf/kms-site.xml | 2 +-
.../crypto/key/kms/server/KMSConfiguration.java | 4 ++
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 14 +++
.../hadoop-kms/src/site/apt/index.apt.vm| 2 +-
.../hadoop/crypto/key/kms/server/MiniKMS.java | 2 +-
.../hadoop/crypto/key/kms/server/TestKMS.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 +
.../java/org/apache/hadoop/hdfs/DFSUtil.java| 41 +++-
.../src/main/resources/hdfs-default.xml | 8
.../src/site/apt/TransparentEncryption.apt.vm | 6 +++
.../apache/hadoop/cli/TestCryptoAdminCLI.java | 2 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +++--
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 3 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 3 +-
15 files changed, 61 insertions(+), 41 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e8602f/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
--
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
index f55ce5f..4f4694c 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-site.xml
@@ -16,7 +16,7 @@
!-- KMS Backend KeyProvider --
property
-namehadoop.security.key.provider.path/name
+namehadoop.kms.key.provider.uri/name
valuejceks://file@/${user.home}/kms.keystore/value
description
/description
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e8602f/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
index f028119..c9b0491 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSConfiguration.java
@@ -40,6 +40,10 @@ public class KMSConfiguration {
public static final String KEY_ACL_PREFIX = key.acl.;
public static final String DEFAULT_KEY_ACL_PREFIX = default.key.acl.;
+ // Property to set the backing KeyProvider
+ public static final String KEY_PROVIDER_URI = CONFIG_PREFIX +
+ key.provider.uri;
+
// Property to Enable/Disable Caching
public static final String KEY_CACHE_ENABLE = CONFIG_PREFIX +
cache.enable;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10e8602f/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
--
diff --git
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index 0827b78..c9eeb1d 100644
---
a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++
b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -39,6 +39,7 @@ import javax.servlet.ServletContextEvent;
import javax.servlet.ServletContextListener;
import java.io.File;
+import java.net.URI;
import java.net.URL;
import java.util.List;
@@ -159,17 +160,12 @@ public class KMSWebApp implements ServletContextListener {
new AccessControlList(AccessControlList.WILDCARD_ACL_VALUE));
// intializing the KeyProvider
-
- ListKeyProvider providers = KeyProviderFactory.getProviders(kmsConf);
- if (providers.isEmpty()) {
+ String providerString = kmsConf.get(KMSConfiguration.KEY_PROVIDER_URI);
+ if (providerString == null) {
throw new IllegalStateException(No KeyProvider has been defined);
[03/11] git commit: HDFS-7075. hadoop-fuse-dfs fails because it cannot find JavaKeyStoreProvider$Factory. (cmccabe)
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory. (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2302485
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2302485
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2302485
Branch: refs/heads/HDFS-6584
Commit: f23024852502441fc259012664e444e5e51c604a
Parents: f24ac42
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 14:27:32 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 14:27:32 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/key/KeyProviderFactory.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2302485/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8cb6c8d..31c09de 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -726,6 +726,9 @@ Release 2.6.0 - UNRELEASED
HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files
(cmccabe)
+HDFS-7075. hadoop-fuse-dfs fails because it cannot find
+JavaKeyStoreProvider$Factory (cmccabe)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2302485/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index 6ca0425..ce99d79 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -46,7 +46,8 @@ public abstract class KeyProviderFactory {
) throws IOException;
private static final ServiceLoaderKeyProviderFactory serviceLoader =
- ServiceLoader.load(KeyProviderFactory.class);
+ ServiceLoader.load(KeyProviderFactory.class,
+ KeyProviderFactory.class.getClassLoader());
// Iterate through the serviceLoader to avoid lazy loading.
// Lazy loading would require synchronization in concurrent use cases.
[10/11] git commit: YARN-2559. Fixed NPE in SystemMetricsPublisher when retrieving FinalApplicationStatus. Contributed by Zhijie Shen
YARN-2559. Fixed NPE in SystemMetricsPublisher when retrieving
FinalApplicationStatus. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee21b13c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee21b13c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee21b13c
Branch: refs/heads/HDFS-6584
Commit: ee21b13cbd4654d7181306404174329f12193613
Parents: 0ecefe6
Author: Jian He jia...@apache.org
Authored: Wed Sep 17 21:44:15 2014 -0700
Committer: Jian He jia...@apache.org
Committed: Wed Sep 17 21:44:15 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/metrics/SystemMetricsPublisher.java | 8 +---
.../resourcemanager/rmapp/attempt/RMAppAttemptImpl.java | 6 --
.../resourcemanager/metrics/TestSystemMetricsPublisher.java | 8
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 5 ++---
5 files changed, 18 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21b13c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bc828c6..5a23814 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -380,6 +380,9 @@ Release 2.6.0 - UNRELEASED
YARN-2558. Updated ContainerTokenIdentifier#read/write to use
ContainerId#getContainerId. (Tsuyoshi OZAWA via jianhe)
+YARN-2559. Fixed NPE in SystemMetricsPublisher when retrieving
+FinalApplicationStatus. (Zhijie Shen via jianhe)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21b13c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index ecf37b0..5da006c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -160,7 +160,7 @@ public class SystemMetricsPublisher extends
CompositeService {
@SuppressWarnings(unchecked)
public void appAttemptFinished(RMAppAttempt appAttempt,
- RMAppAttemptState state, long finishedTime) {
+ RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
@@ -168,8 +168,10 @@ public class SystemMetricsPublisher extends
CompositeService {
appAttempt.getTrackingUrl(),
appAttempt.getOriginalTrackingUrl(),
appAttempt.getDiagnostics(),
- appAttempt.getFinalApplicationStatus(),
- RMServerUtils.createApplicationAttemptState(state),
+ // app will get the final status from app attempt, or create one
+ // based on app state if it doesn't exist
+ app.getFinalApplicationStatus(),
+ RMServerUtils.createApplicationAttemptState(appAttemtpState),
finishedTime));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee21b13c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 863130f..7ca57ee 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
[06/11] git commit: HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is incorrect in some cases. (Yi Liu via wang)
HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is
incorrect in some cases. (Yi Liu via wang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47e5e198
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47e5e198
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47e5e198
Branch: refs/heads/HDFS-6584
Commit: 47e5e19831a363aa4d675fd23ab0d06e86809094
Parents: 123f20d
Author: Andrew Wang w...@apache.org
Authored: Wed Sep 17 17:58:56 2014 -0700
Committer: Andrew Wang w...@apache.org
Committed: Wed Sep 17 17:58:56 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/crypto/CryptoInputStream.java | 11 ++-
.../hadoop/crypto/CryptoStreamsTestBase.java | 18 ++
3 files changed, 27 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47e5e198/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d2671c3..f2b4180 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -732,6 +732,9 @@ Release 2.6.0 - UNRELEASED
HDFS-7075. hadoop-fuse-dfs fails because it cannot find
JavaKeyStoreProvider$Factory (cmccabe)
+HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is
+incorrect in some cases. (Yi Liu via wang)
+
BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS
HADOOP-10734. Implement high-performance secure random number sources.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47e5e198/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
index e8964ed..68e9697 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/CryptoInputStream.java
@@ -471,7 +471,16 @@ public class CryptoInputStream extends FilterInputStream
implements
streamOffset += n; // Read n bytes
decrypt(buf, n, pos);
}
- return n;
+
+ if (n = 0) {
+return unread + n;
+ } else {
+if (unread == 0) {
+ return -1;
+} else {
+ return unread;
+}
+ }
}
throw new UnsupportedOperationException(ByteBuffer read unsupported +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47e5e198/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
index f5acc73..86bb64d 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/CryptoStreamsTestBase.java
@@ -469,6 +469,7 @@ public abstract class CryptoStreamsTestBase {
int bufPos) throws Exception {
buf.position(bufPos);
int n = ((ByteBufferReadable) in).read(buf);
+Assert.assertEquals(bufPos + n, buf.position());
byte[] readData = new byte[n];
buf.rewind();
buf.position(bufPos);
@@ -568,6 +569,7 @@ public abstract class CryptoStreamsTestBase {
// Read forward len1
ByteBuffer buf = ByteBuffer.allocate(len1);
int nRead = ((ByteBufferReadable) in).read(buf);
+Assert.assertEquals(nRead, buf.position());
readData = new byte[nRead];
buf.rewind();
buf.get(readData);
@@ -575,9 +577,10 @@ public abstract class CryptoStreamsTestBase {
System.arraycopy(data, (int)pos, expectedData, 0, nRead);
Assert.assertArrayEquals(readData, expectedData);
-// Pos should be len1 + 2 * len2 + nRead
+long lastPos = pos;
+// Pos should be lastPos + nRead
pos = ((Seekable) in).getPos();
-Assert.assertEquals(len1 + 2 * len2 + nRead, pos);
+Assert.assertEquals(lastPos + nRead, pos);
// Pos: 1/3 dataLen
positionedReadCheck(in , dataLen / 3);
@@ -589,13 +592,15 @@ public abstract class CryptoStreamsTestBase {
[07/11] git commit: Move some HDFS JIRAs to the correct CHANGES.txt
Move some HDFS JIRAs to the correct CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf38793c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf38793c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf38793c Branch: refs/heads/HDFS-6584 Commit: bf38793ce169137bb3ef36e96db7ea62d89ce1c4 Parents: 47e5e19 Author: Andrew Wang w...@apache.org Authored: Wed Sep 17 18:08:34 2014 -0700 Committer: Andrew Wang w...@apache.org Committed: Wed Sep 17 18:08:34 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 8 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 8 2 files changed, 8 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf38793c/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index f2b4180..0ca2953 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -530,8 +530,6 @@ Release 2.6.0 - UNRELEASED HADOOP-10922. User documentation for CredentialShell. (Larry McCay via wang) -HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) - HADOOP-11016. KMS should support signing cookies with zookeeper secret manager. (tucu) @@ -726,12 +724,6 @@ Release 2.6.0 - UNRELEASED HADOOP-11056. OsSecureRandom.setConf() might leak file descriptors (yzhang via cmccabe) -HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files -(cmccabe) - -HDFS-7075. hadoop-fuse-dfs fails because it cannot find -JavaKeyStoreProvider$Factory (cmccabe) - HADOOP-11040. Return value of read(ByteBuffer buf) in CryptoInputStream is incorrect in some cases. (Yi Liu via wang) http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf38793c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 567a6ab..0e01ca0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -471,6 +471,8 @@ Release 2.6.0 - UNRELEASED HDFS-6705. Create an XAttr that disallows the HDFS admin from accessing a file. (clamb via wang) +HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) + OPTIMIZATIONS HDFS-6690. Deduplicate xattr names in memory. (wang) @@ -670,6 +672,12 @@ Release 2.6.0 - UNRELEASED and TestDFSClientFailover.testDoesntDnsResolveLogicalURI failing on jdk7. (Akira Ajisaka via wang) +HDFS-6912. SharedFileDescriptorFactory should not allocate sparse files +(cmccabe) + +HDFS-7075. hadoop-fuse-dfs fails because it cannot find +JavaKeyStoreProvider$Factory (cmccabe) + BREAKDOWN OF HDFS-6134 AND HADOOP-10150 SUBTASKS AND RELATED JIRAS HDFS-6387. HDFS CLI admin tool for creating deleting an
[01/11] git commit: HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-6584 911979c8a - 2d2b0009e
HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e3803d00
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e3803d00
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e3803d00
Branch: refs/heads/HDFS-6584
Commit: e3803d002c660f18a5c2ecf32344fd6f3f491a5b
Parents: ea4e2e8
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Wed Sep 17 12:55:35 2014 -0700
Committer: Colin Patrick Mccabe cmcc...@cloudera.com
Committed: Wed Sep 17 12:55:35 2014 -0700
--
.../java/org/apache/hadoop/fs/FileStatus.java | 9 ++
.../hadoop/fs/permission/FsPermission.java | 7 ++
.../src/site/markdown/filesystem/filesystem.md | 31 +++
.../fs/contract/AbstractContractOpenTest.java | 12 +++
.../hadoop/hdfs/protocol/FsAclPermission.java | 77 -
.../hdfs/protocol/FsPermissionExtension.java| 89
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 +-
.../hdfs/server/namenode/FSDirectory.java | 36 +---
.../org/apache/hadoop/hdfs/web/JsonUtil.java| 16 +++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 88 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 5 +-
11 files changed, 280 insertions(+), 94 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
index b261f7f..da3807d 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileStatus.java
@@ -200,6 +200,15 @@ public class FileStatus implements Writable, Comparable {
public FsPermission getPermission() {
return permission;
}
+
+ /**
+ * Tell whether the underlying file or directory is encrypted or not.
+ *
+ * @return true if the underlying file is encrypted.
+ */
+ public boolean isEncrypted() {
+return permission.getEncryptedBit();
+ }
/**
* Get the owner of the file.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index ee84437..264a095 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -294,6 +294,13 @@ public class FsPermission implements Writable {
return false;
}
+ /**
+ * Returns true if the file is encrypted or directory is in an encryption
zone
+ */
+ public boolean getEncryptedBit() {
+return false;
+ }
+
/** Set the user file creation mask (umask) */
public static void setUMask(Configuration conf, FsPermission umask) {
conf.set(UMASK_LABEL, String.format(%1$03o, umask.toShort()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e3803d00/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
--
diff --git
a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 70796cc..e59fa1b 100644
---
a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++
b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -64,6 +64,33 @@ all operations on a valid FileSystem MUST result in a new
FileSystem that is als
def isSymlink(FS, p) = p in symlinks(FS)
+### 'boolean inEncryptionZone(Path p)'
+
+Return True if the data for p is encrypted. The nature of the encryption and
the
+mechanism for creating an encryption zone are implementation details not
covered
+in this specification. No guarantees are made about the quality of the
+encryption. The metadata is not encrypted.
+
+ Preconditions
+
+if not exists(FS, p) : raise FileNotFoundException
+
+ Postconditions
+
+ Invariants
+
+All files and directories
[02/11] git commit: HDFS-6843. Add to CHANGES.txt
HDFS-6843. Add to CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f24ac429 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f24ac429 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f24ac429 Branch: refs/heads/HDFS-6584 Commit: f24ac429d102777fe021e9852cfff38312643512 Parents: e3803d0 Author: Colin Patrick Mccabe cmcc...@cloudera.com Authored: Wed Sep 17 13:38:11 2014 -0700 Committer: Colin Patrick Mccabe cmcc...@cloudera.com Committed: Wed Sep 17 13:38:11 2014 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24ac429/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index a1dca66..8cb6c8d 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -530,6 +530,8 @@ Release 2.6.0 - UNRELEASED HADOOP-10922. User documentation for CredentialShell. (Larry McCay via wang) +HDFS-6843. Create FileStatus isEncrypted() method (clamb via cmccabe) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
[11/11] git commit: Merge branch 'trunk' into HDFS-6584
Merge branch 'trunk' into HDFS-6584 Conflicts: hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d2b0009 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d2b0009 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d2b0009 Branch: refs/heads/HDFS-6584 Commit: 2d2b0009e662db75cf22e2ce8d618ed0a8e61c2f Parents: 911979c ee21b13 Author: Tsz-Wo Nicholas Sze szets...@hortonworks.com Authored: Thu Sep 18 13:00:29 2014 +0800 Committer: Tsz-Wo Nicholas Sze szets...@hortonworks.com Committed: Thu Sep 18 13:00:29 2014 +0800 -- hadoop-common-project/hadoop-common/CHANGES.txt | 7 +- .../apache/hadoop/crypto/CryptoInputStream.java | 11 +- .../hadoop/crypto/key/KeyProviderFactory.java | 3 +- .../java/org/apache/hadoop/fs/FileStatus.java | 9 + .../hadoop/fs/permission/FsPermission.java | 7 + .../src/site/markdown/filesystem/filesystem.md | 31 .../hadoop/crypto/CryptoStreamsTestBase.java| 18 +- .../fs/contract/AbstractContractOpenTest.java | 12 ++ hadoop-common-project/hadoop-kms/pom.xml| 5 + .../hadoop-kms/src/main/conf/kms-site.xml | 59 +- .../key/kms/server/KMSAuthenticationFilter.java | 7 +- .../crypto/key/kms/server/KMSConfiguration.java | 4 + .../hadoop/crypto/key/kms/server/KMSWebApp.java | 14 +- .../hadoop-kms/src/site/apt/index.apt.vm| 163 - .../hadoop/crypto/key/kms/server/MiniKMS.java | 2 +- .../hadoop/crypto/key/kms/server/TestKMS.java | 7 +- .../crypto/key/kms/server/TestKMSWithZK.java| 179 +++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++ .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 1 + .../java/org/apache/hadoop/hdfs/DFSUtil.java| 41 +++-- .../hadoop/hdfs/protocol/FsAclPermission.java | 77 .../hdfs/protocol/FsPermissionExtension.java| 89 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 +- .../server/namenode/EncryptionZoneManager.java | 17 +- .../hdfs/server/namenode/FSDirectory.java | 35 +++- .../org/apache/hadoop/hdfs/web/JsonUtil.java| 16 +- .../src/main/resources/hdfs-default.xml | 8 + .../src/site/apt/TransparentEncryption.apt.vm | 6 + .../apache/hadoop/cli/TestCryptoAdminCLI.java | 2 +- .../apache/hadoop/hdfs/TestEncryptionZones.java | 136 +- .../hadoop/hdfs/TestEncryptionZonesWithHA.java | 3 +- .../hadoop/hdfs/TestReservedRawPaths.java | 3 +- .../hdfs/server/namenode/FSAclBaseTest.java | 5 +- hadoop-yarn-project/CHANGES.txt | 6 + .../yarn/security/ContainerTokenIdentifier.java | 4 +- .../metrics/SystemMetricsPublisher.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 6 +- .../metrics/TestSystemMetricsPublisher.java | 8 +- .../attempt/TestRMAppAttemptTransitions.java| 5 +- .../server/TestContainerManagerSecurity.java| 92 ++ 40 files changed, 917 insertions(+), 205 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2b0009/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2b0009/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2b0009/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d2b0009/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java -- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java index 3426bf2,56105d9..9346ea5 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java @@@ -2438,10 -2391,10 +2451,10 @@@ public class FSDirectory implements Clo new HdfsLocatedFileStatus(size, node.isDirectory(), replication, blocksize, node.getModificationTime(snapshot), node.getAccessTime(snapshot), - getPermissionForFileStatus(node, snapshot), + getPermissionForFileStatus(node, snapshot, isEncrypted), node.getUserName(snapshot), node.getGroupName(snapshot), node.isSymlink()
