hadoop git commit: HDFS-8175. Provide information on snapshotDiff for supporting the comparison between snapshot and current status (Contributed by J.Andreina)
Repository: hadoop Updated Branches: refs/heads/branch-2 10999ffd7 - 595b92e2c HDFS-8175. Provide information on snapshotDiff for supporting the comparison between snapshot and current status (Contributed by J.Andreina) (cherry picked from commit 6232235df3ae14db66181922c8343c986a6e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/595b92e2 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/595b92e2 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/595b92e2 Branch: refs/heads/branch-2 Commit: 595b92e2c29a4c0d010bd29ceb79e6c7e0d4eeb4 Parents: 10999ff Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 14:37:50 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 14:38:53 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml | 4 2 files changed, 7 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/595b92e2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index bcb08c8..40934c0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -351,6 +351,9 @@ Release 2.8.0 - UNRELEASED HDFS-8187. Remove usage of -setStoragePolicy and -getStoragePolicy using dfsadmin cmd (as it is not been supported) (J.Andreina via vinayakumarb) +HDFS-8175. Provide information on snapshotDiff for supporting the comparison +between snapshot and current status (J.Andreina via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/595b92e2/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml index eba1d80..330d00f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsSnapshots.xml @@ -257,6 +257,10 @@ trtdfromSnapshot/tdtdThe name of the starting snapshot./td/tr trtdtoSnapshot/tdtdThe name of the ending snapshot./td/tr /table/li +p + Note that snapshotDiff can be used to get the difference report between two snapshots, or between + a snapshot and the current status of a directory.Users can use . to represent the current status. +/p liResults: table trtd+/tdtdThe file/directory has been created./td/tr
hadoop git commit: HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing federated cluster is wrong (Contributed by Ajith S)
Repository: hadoop Updated Branches: refs/heads/branch-2 b7a5ebd56 - 1f7850241 HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing federated cluster is wrong (Contributed by Ajith S) (cherry picked from commit 26da2ddf38a2caaee375852bdf0d6975d151476a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f785024 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f785024 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f785024 Branch: refs/heads/branch-2 Commit: 1f785024161a38212c69a3c4f75bca4450298e10 Parents: b7a5ebd Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 12:31:59 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 13:47:06 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f785024/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8549d90..7243fba 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -339,6 +339,9 @@ Release 2.8.0 - UNRELEASED HDFS-8332. DFS client API calls should check filesystem closed (Rakesh R via umamahesh) +HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing +federated cluster is wrong (Ajith S via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f785024/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md index 6996fac..200e2e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md @@ -187,7 +187,7 @@ Perform the following steps: * Refresh the Datanodes to pickup the newly added Namenode by running the following command against all the Datanodes in the cluster: -[hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNameNode datanode_host_name:datanode_rpc_port +[hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNamenodes datanode_host_name:datanode_rpc_port Managing the cluster
hadoop git commit: HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing federated cluster is wrong (Contributed by Ajith S)
Repository: hadoop Updated Branches: refs/heads/trunk d18f10ad1 - 26da2ddf3 HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing federated cluster is wrong (Contributed by Ajith S) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26da2ddf Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26da2ddf Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26da2ddf Branch: refs/heads/trunk Commit: 26da2ddf38a2caaee375852bdf0d6975d151476a Parents: d18f10a Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 12:31:59 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 12:33:49 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/26da2ddf/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 6032692..6951494 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -666,6 +666,9 @@ Release 2.8.0 - UNRELEASED HDFS-8332. DFS client API calls should check filesystem closed (Rakesh R via umamahesh) +HDFS-7998. HDFS Federation : Command mentioned to add a NN to existing +federated cluster is wrong (Ajith S via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/26da2ddf/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md index 09dea0a..c00e544 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/Federation.md @@ -187,7 +187,7 @@ Perform the following steps: * Refresh the Datanodes to pickup the newly added Namenode by running the following command against all the Datanodes in the cluster: -[hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNameNode datanode_host_name:datanode_rpc_port +[hdfs]$ $HADOOP_PREFIX/bin/hdfs dfsadmin -refreshNamenodes datanode_host_name:datanode_rpc_port Managing the cluster
hadoop git commit: HDFS-8187. Remove usage of '-setStoragePolicy' and '-getStoragePolicy' using dfsadmin cmd (as it is not been supported) (Contributed by J.Andreina)
Repository: hadoop Updated Branches: refs/heads/branch-2 a56f57b42 - 10999ffd7 HDFS-8187. Remove usage of '-setStoragePolicy' and '-getStoragePolicy' using dfsadmin cmd (as it is not been supported) (Contributed by J.Andreina) (cherry picked from commit e642eff0e0065d1c504e41bc87993948b4bc073c) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10999ffd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10999ffd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10999ffd Branch: refs/heads/branch-2 Commit: 10999ffd7ab030524c2846248926e1d89457 Parents: a56f57b Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 14:16:30 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 14:26:58 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 3 +++ .../hadoop-hdfs/src/site/markdown/ArchivalStorage.md | 6 +++--- .../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 6 +- 3 files changed, 7 insertions(+), 8 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/10999ffd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 41157b7..bcb08c8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -348,6 +348,9 @@ Release 2.8.0 - UNRELEASED HDFS-8108. Fsck should provide the info on mandatory option to be used along with -blocks , -locations and -racks (J.Andreina via umamahesh) +HDFS-8187. Remove usage of -setStoragePolicy and -getStoragePolicy using +dfsadmin cmd (as it is not been supported) (J.Andreina via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/10999ffd/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md index d924f80..bde4e7b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ArchivalStorage.md @@ -1,4 +1,4 @@ -!--- +!--- Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -84,13 +84,13 @@ Note that the Lasy\_Persist policy is useful only for single replica blocks. For ### Storage Policy Resolution -When a file or directory is created, its storage policy is *unspecified*. The storage policy can be specified using the [`dfsadmin -setStoragePolicy`](#Set_Storage_Policy) command. The effective storage policy of a file or directory is resolved by the following rules. +When a file or directory is created, its storage policy is *unspecified*. The storage policy can be specified using the [`storagepolicies -setStoragePolicy`](#Set_Storage_Policy) command. The effective storage policy of a file or directory is resolved by the following rules. 1. If the file or directory is specificed with a storage policy, return it. 2. For an unspecified file or directory, if it is the root directory, return the *default storage policy*. Otherwise, return its parent's effective storage policy. -The effective storage policy can be retrieved by the [`dfsadmin -getStoragePolicy`](#Get_Storage_Policy) command. +The effective storage policy can be retrieved by the [`storagepolicies -getStoragePolicy`](#Get_Storage_Policy) command. ### Configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/10999ffd/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index c1101a2..1153de7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -1,4 +1,4 @@ -!--- +!--- Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. You may obtain a copy of the License at @@ -309,8 +309,6 @@ Usage: [-clrQuota dirname...dirname] [-setSpaceQuota quota [-storageType storagetype] dirname...dirname] [-clrSpaceQuota [-storageType storagetype] dirname...dirname
hadoop git commit: HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina)
Repository: hadoop Updated Branches: refs/heads/trunk 2401ff76b - e01c1a979 HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e01c1a97 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e01c1a97 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e01c1a97 Branch: refs/heads/trunk Commit: e01c1a979cbc8fb398d5b7a46693657bc2668a58 Parents: 2401ff7 Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 15:24:36 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 15:24:36 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++ hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh | 7 --- .../hadoop-hdfs/src/site/markdown/HdfsUserGuide.md| 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e01c1a97/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 82e0799..e1717e8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -755,6 +755,8 @@ Release 2.7.1 - UNRELEASED HDFS-7980. Incremental BlockReport will dramatically slow down namenode startup. (Walter Su via szetszwo) +HDFS-8226. Non-HA rollback compatibility broken (J.Andreina via vinayakumarb) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e01c1a97/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh index 5799dec..275c1c0 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh @@ -48,13 +48,14 @@ fi # get arguments if [[ $# -ge 1 ]]; then - nameStartOpt=$1 + startOpt=$1 shift - case $nameStartOpt in + case $startOpt in -upgrade) + nameStartOpt=$startOpt ;; -rollback) - dataStartOpt=$nameStartOpt + dataStartOpt=$startOpt ;; *) hadoop_exit_with_usage 1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/e01c1a97/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md index a9692a8..24af67b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md @@ -318,6 +318,8 @@ When Hadoop is upgraded on an existing cluster, as with any software upgrade, it * stop the cluster and distribute earlier version of Hadoop. +* run the rollback command on the namenode (`bin/hdfs namenode -rollback`). + * start the cluster with rollback option. (`sbin/start-dfs.sh -rollback`). When upgrading to a new version of HDFS, it is necessary to rename or delete any paths that are reserved in the new version of HDFS. If the NameNode encounters a reserved path during upgrade, it will print an error like the following:
hadoop git commit: HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 ade5165ec - 3a894b456
HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina)
(cherry picked from commit e01c1a979cbc8fb398d5b7a46693657bc2668a58)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a894b45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a894b45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a894b45
Branch: refs/heads/branch-2
Commit: 3a894b45635d6528bb4c29953f57bd418a7c8a80
Parents: ade5165
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 15:24:36 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 15:34:17 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 28 ++--
.../src/site/markdown/HdfsUserGuide.md | 4 ++-
3 files changed, 19 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a894b45/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f1eb24d..2b6f476 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -428,6 +428,8 @@ Release 2.7.1 - UNRELEASED
HDFS-7980. Incremental BlockReport will dramatically slow down namenode
startup. (Walter Su via szetszwo)
+HDFS-8226. Non-HA rollback compatibility broken (J.Andreina via
vinayakumarb)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a894b45/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
index 8cbea16..c46e625 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
@@ -30,20 +30,20 @@
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
# get arguments
-if [ $# -ge 1 ]; then
- nameStartOpt=$1
- shift
- case $nameStartOpt in
- (-upgrade)
- ;;
- (-rollback)
- dataStartOpt=$nameStartOpt
- ;;
- (*)
- echo $usage
- exit 1
- ;;
- esac
+if [[ $# -ge 1 ]]; then
+ startOpt=$1
+ shift
+ case $startOpt in
+-upgrade)
+ nameStartOpt=$startOpt
+;;
+-rollback)
+ dataStartOpt=$startOpt
+;;
+*)
+ hadoop_exit_with_usage 1
+;;
+ esac
fi
#Add other possible options
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a894b45/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index f80c1f6..9c6bd02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -318,7 +318,9 @@ When Hadoop is upgraded on an existing cluster, as with any
software upgrade, it
* stop the cluster and distribute earlier version of Hadoop.
-* start the cluster with rollback option. (`bin/start-dfs.sh -rollback`).
+* run the rollback command on the namenode (`bin/hdfs namenode -rollback`).
+
+* start the cluster with rollback option. (`sbin/start-dfs.sh -rollback`).
When upgrading to a new version of HDFS, it is necessary to rename or delete
any paths that are reserved in the new version of HDFS. If the NameNode
encounters a reserved path during upgrade, it will print an error like the
following:
hadoop git commit: HADOOP-11877. SnappyDecompressor's Logger class name is wrong ( Contributed by surendra singh lilhore)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a87ab77fe - 2c3d79023
HADOOP-11877. SnappyDecompressor's Logger class name is wrong ( Contributed by
surendra singh lilhore)
(cherry picked from commit 1ffb7fa42e98e1b0c9ec46a32e682f92c660308a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c3d7902
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c3d7902
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c3d7902
Branch: refs/heads/branch-2
Commit: 2c3d79023bb940a2e0db85fa6fc6849db3d9cb75
Parents: a87ab77
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 16:52:14 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 16:53:16 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c3d7902/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ba7e3be..2c358bb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -173,6 +173,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11922. Misspelling of threshold in log4j.properties for tests
in hadoop-tools (Gabor Liptak via vinayakumarb)
+HADOOP-11877. SnappyDecompressor's Logger class name is wrong
+(surendra singh lilhore via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c3d7902/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
index dbffba8..8712431 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.util.NativeCodeLoader;
*/
public class SnappyDecompressor implements Decompressor {
private static final Log LOG =
- LogFactory.getLog(SnappyCompressor.class.getName());
+ LogFactory.getLog(SnappyDecompressor.class.getName());
private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024;
private int directBufferSize;
hadoop git commit: HADOOP-11877. SnappyDecompressor's Logger class name is wrong ( Contributed by surendra singh lilhore)
Repository: hadoop
Updated Branches:
refs/heads/trunk 46509617b - 1ffb7fa42
HADOOP-11877. SnappyDecompressor's Logger class name is wrong ( Contributed by
surendra singh lilhore)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ffb7fa4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ffb7fa4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ffb7fa4
Branch: refs/heads/trunk
Commit: 1ffb7fa42e98e1b0c9ec46a32e682f92c660308a
Parents: 4650961
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 16:52:14 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 16:52:14 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ffb7fa4/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 691ddef..f4b9246 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11922. Misspelling of threshold in log4j.properties for tests
in hadoop-tools (Gabor Liptak via vinayakumarb)
+HADOOP-11877. SnappyDecompressor's Logger class name is wrong
+(surendra singh lilhore via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ffb7fa4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
index dbffba8..8712431 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/compress/snappy/SnappyDecompressor.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.util.NativeCodeLoader;
*/
public class SnappyDecompressor implements Decompressor {
private static final Log LOG =
- LogFactory.getLog(SnappyCompressor.class.getName());
+ LogFactory.getLog(SnappyDecompressor.class.getName());
private static final int DEFAULT_DIRECT_BUFFER_SIZE = 64 * 1024;
private int directBufferSize;
hadoop git commit: HDFS-8209. Support different number of datanode directories in MiniDFSCluster. (Contributed by surendra singh lilhore)
Repository: hadoop
Updated Branches:
refs/heads/trunk ecfa05227 - 4c6816faf
HDFS-8209. Support different number of datanode directories in MiniDFSCluster.
(Contributed by surendra singh lilhore)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c6816fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c6816fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c6816fa
Branch: refs/heads/trunk
Commit: 4c6816faf83726c71c9c3981e7280b461f824a4f
Parents: ecfa052
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 15:03:44 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 15:03:44 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 4 +-
.../apache/hadoop/hdfs/TestMiniDFSCluster.java | 43
3 files changed, 49 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6816fa/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 195a946..82e0799 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -684,6 +684,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8175. Provide information on snapshotDiff for supporting the
comparison
between snapshot and current status (J.Andreina via vinayakumarb)
+HDFS-8209. Support different number of datanode directories in
MiniDFSCluster.
+(surendra singh lilhore via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6816fa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 8aeaef8..12ad23e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -1211,8 +1211,10 @@ public class MiniDFSCluster {
String makeDataNodeDirs(int dnIndex, StorageType[] storageTypes) throws
IOException {
StringBuilder sb = new StringBuilder();
-assert storageTypes == null || storageTypes.length == storagesPerDatanode;
for (int j = 0; j storagesPerDatanode; ++j) {
+ if ((storageTypes != null) (j = storageTypes.length)) {
+break;
+ }
File dir = getInstanceStorageDir(dnIndex, j);
dir.mkdirs();
if (!dir.isDirectory()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c6816fa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
index cf29d97..3fa852e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestMiniDFSCluster.java
@@ -23,9 +23,13 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.test.PathUtils;
import org.junit.Before;
import org.junit.Test;
@@ -139,4 +143,43 @@ public class TestMiniDFSCluster {
MiniDFSCluster.shutdownCluster(cluster5);
}
}
+
+ @Test
+ public void testClusterSetDatanodeDifferentStorageType() throws IOException {
+final Configuration conf = new HdfsConfiguration();
+StorageType[][] storageType = new StorageType[][] {
+{StorageType.DISK, StorageType.ARCHIVE}, {StorageType.DISK},
+{StorageType.ARCHIVE}};
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(3).storageTypes(storageType).build();
+try {
+ cluster.waitActive();
+ ArrayListDataNode dataNodes = cluster.getDataNodes();
+ // Check the number of directory in DN's
+ for (int i = 0; i storageType.length; i++) {
+assertEquals
hadoop git commit: HDFS-6576. Datanode log is generating at root directory in security mode (Contributed by surendra singh lilhore)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5ddc3adb0 - a87ab77fe
HDFS-6576. Datanode log is generating at root directory in security mode
(Contributed by surendra singh lilhore)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a87ab77f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a87ab77f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a87ab77f
Branch: refs/heads/branch-2
Commit: a87ab77fec375c02ba507bd7c7ca1e0082f623c2
Parents: 5ddc3ad
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 16:37:36 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 16:37:36 2015 +0530
--
hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a87ab77f/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
--
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index f60b65a..930b10c 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -72,7 +72,7 @@ export HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER}
#export HADOOP_LOG_DIR=${HADOOP_LOG_DIR}/$USER
# Where log files are stored in the secure data environment.
-export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
+#export HADOOP_SECURE_DN_LOG_DIR=${HADOOP_LOG_DIR}/${HADOOP_HDFS_USER}
###
# HDFS Mover specific parameters
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a87ab77f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2b6f476..56f419c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -360,6 +360,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8209. Support different number of datanode directories in
MiniDFSCluster.
(surendra singh lilhore via vinayakumarb)
+HDFS-6576. Datanode log is generating at root directory in security mode
+(surendra singh lilhore via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
[1/2] hadoop git commit: HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 ada2f74b1 - 1fbefe586
HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina)
(cherry picked from commit e01c1a979cbc8fb398d5b7a46693657bc2668a58)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
(cherry picked from commit 3a894b45635d6528bb4c29953f57bd418a7c8a80)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99309a9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99309a9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99309a9f
Branch: refs/heads/branch-2.7
Commit: 99309a9fecbd2d17e525f6989c936ee6d0bbaeb9
Parents: ada2f74
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 15:24:36 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 15:42:02 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../hadoop-hdfs/src/main/bin/start-dfs.sh | 28 ++--
.../src/site/markdown/HdfsUserGuide.md | 4 ++-
3 files changed, 19 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99309a9f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eaade89..bdb3032 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -68,6 +68,8 @@ Release 2.7.1 - UNRELEASED
HDFS-7980. Incremental BlockReport will dramatically slow down namenode
startup. (Walter Su via szetszwo)
+HDFS-8226. Non-HA rollback compatibility broken (J.Andreina via
vinayakumarb)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99309a9f/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
index 8cbea16..c46e625 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh
@@ -30,20 +30,20 @@
HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:-$DEFAULT_LIBEXEC_DIR}
. $HADOOP_LIBEXEC_DIR/hdfs-config.sh
# get arguments
-if [ $# -ge 1 ]; then
- nameStartOpt=$1
- shift
- case $nameStartOpt in
- (-upgrade)
- ;;
- (-rollback)
- dataStartOpt=$nameStartOpt
- ;;
- (*)
- echo $usage
- exit 1
- ;;
- esac
+if [[ $# -ge 1 ]]; then
+ startOpt=$1
+ shift
+ case $startOpt in
+-upgrade)
+ nameStartOpt=$startOpt
+;;
+-rollback)
+ dataStartOpt=$startOpt
+;;
+*)
+ hadoop_exit_with_usage 1
+;;
+ esac
fi
#Add other possible options
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99309a9f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
index 37fa4be..c43c63c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsUserGuide.md
@@ -319,7 +319,9 @@ When Hadoop is upgraded on an existing cluster, as with any
software upgrade, it
* stop the cluster and distribute earlier version of Hadoop.
-* start the cluster with rollback option. (`bin/start-dfs.sh -rollback`).
+* run the rollback command on the namenode (`bin/hdfs namenode -rollback`).
+
+* start the cluster with rollback option. (`sbin/start-dfs.sh -rollback`).
When upgrading to a new version of HDFS, it is necessary to rename or delete
any paths that are reserved in the new version of HDFS. If the NameNode
encounters a reserved path during upgrade, it will print an error like the
following:
[2/2] hadoop git commit: HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina) Added one missed change, missed while resolving conflict
HDFS-8226. Non-HA rollback compatibility broken (Contributed by J.Andreina) Added one missed change, missed while resolving conflict (cherry picked from commit 3ee1b50dad9604bd959adbae472813f4bdf2fec7) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1fbefe58 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1fbefe58 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1fbefe58 Branch: refs/heads/branch-2.7 Commit: 1fbefe586b35e581f2c4610cd8e010e27e782bc3 Parents: 99309a9 Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 15:39:47 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 15:42:17 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1fbefe58/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh index c46e625..a8c2b98 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-dfs.sh @@ -41,7 +41,8 @@ if [[ $# -ge 1 ]]; then dataStartOpt=$startOpt ;; *) - hadoop_exit_with_usage 1 + echo $usage + exit 1 ;; esac fi
hadoop git commit: HDFS-6576. Datanode log is generating at root directory in security mode (Contributed by surendra singh lilhore) Just merging CHANGES.txt changes from branch-2 to keep in sync.
Repository: hadoop Updated Branches: refs/heads/trunk b419c1b2e - 46509617b HDFS-6576. Datanode log is generating at root directory in security mode (Contributed by surendra singh lilhore) Just merging CHANGES.txt changes from branch-2 to keep in sync. (cherry picked from commit a87ab77fec375c02ba507bd7c7ca1e0082f623c2) Conflicts: hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46509617 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46509617 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46509617 Branch: refs/heads/trunk Commit: 46509617b6296eeeabf9d99e8ba365040bec05ad Parents: b419c1b Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 16:37:36 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 16:44:03 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 1 file changed, 3 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/46509617/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e1717e8..c9e5b87 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -687,6 +687,9 @@ Release 2.8.0 - UNRELEASED HDFS-8209. Support different number of datanode directories in MiniDFSCluster. (surendra singh lilhore via vinayakumarb) +HDFS-6576. Datanode log is generating at root directory in security mode +(surendra singh lilhore via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES
hadoop git commit: HDFS-3384. DataStreamer thread should be closed immediatly when failed to setup a PipelineForAppendOrRecovery (Contributed by Uma Maheswara Rao G)
Repository: hadoop
Updated Branches:
refs/heads/trunk 1ffb7fa42 - c648317a6
HDFS-3384. DataStreamer thread should be closed immediatly when failed to setup
a PipelineForAppendOrRecovery (Contributed by Uma Maheswara Rao G)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c648317a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c648317a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c648317a
Branch: refs/heads/trunk
Commit: c648317a68891e1c900f04b7a9c98ba40c5faddb
Parents: 1ffb7fa
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 17:18:14 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 17:18:14 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DataStreamer.java| 4
.../org/apache/hadoop/hdfs/TestFileAppend.java | 22
3 files changed, 29 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c648317a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c9e5b87..4decfd8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -690,6 +690,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6576. Datanode log is generating at root directory in security mode
(surendra singh lilhore via vinayakumarb)
+HDFS-3384. DataStreamer thread should be closed immediatly when failed to
+setup a PipelineForAppendOrRecovery (Uma Maheswara Rao G via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c648317a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 96bf212..697ee11 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -460,6 +460,9 @@ class DataStreamer extends Daemon {
LOG.debug(Append to block + block);
}
setupPipelineForAppendOrRecovery();
+ if (true == streamerClosed) {
+continue;
+ }
initDataStreaming();
}
@@ -571,6 +574,7 @@ class DataStreamer extends Daemon {
}
}
lastException.set(e);
+assert !(e instanceof NullPointerException);
hasError = true;
if (errorIndex == -1 restartingNodeIndex.get() == -1) {
// Not a datanode issue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c648317a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index 6a7c3ea..402c944 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -602,4 +602,26 @@ public class TestFileAppend{
cluster.shutdown();
}
}
+
+ @Test(timeout = 1)
+ public void testAppendCorruptedBlock() throws Exception {
+Configuration conf = new HdfsConfiguration();
+conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
+conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+conf.setInt(dfs.min.replication, 1);
+MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+.build();
+try {
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path fileName = new Path(/appendCorruptBlock);
+ DFSTestUtil.createFile(fs, fileName, 512, (short) 1, 0);
+ DFSTestUtil.waitReplication(fs, fileName, (short) 1);
+ Assert.assertTrue(File not created, fs.exists(fileName));
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
+ cluster.corruptBlockOnDataNodes(block);
+ DFSTestUtil.appendFile(fs, fileName, appendCorruptBlock);
+} finally {
+ cluster.shutdown();
+}
+ }
}
hadoop git commit: HDFS-3384. DataStreamer thread should be closed immediatly when failed to setup a PipelineForAppendOrRecovery (Contributed by Uma Maheswara Rao G)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2c3d79023 - d1293
HDFS-3384. DataStreamer thread should be closed immediatly when failed to setup
a PipelineForAppendOrRecovery (Contributed by Uma Maheswara Rao G)
(cherry picked from commit c648317a68891e1c900f04b7a9c98ba40c5faddb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d129
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d129
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d129
Branch: refs/heads/branch-2
Commit: d12937eadc0d096c0ca5f2c86cb57e148b75
Parents: 2c3d790
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 17:18:14 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 17:18:59 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DataStreamer.java| 4
.../org/apache/hadoop/hdfs/TestFileAppend.java | 22
3 files changed, 29 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d129/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 56f419c..0cc149c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -363,6 +363,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6576. Datanode log is generating at root directory in security mode
(surendra singh lilhore via vinayakumarb)
+HDFS-3384. DataStreamer thread should be closed immediatly when failed to
+setup a PipelineForAppendOrRecovery (Uma Maheswara Rao G via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d129/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 96bf212..697ee11 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -460,6 +460,9 @@ class DataStreamer extends Daemon {
LOG.debug(Append to block + block);
}
setupPipelineForAppendOrRecovery();
+ if (true == streamerClosed) {
+continue;
+ }
initDataStreaming();
}
@@ -571,6 +574,7 @@ class DataStreamer extends Daemon {
}
}
lastException.set(e);
+assert !(e instanceof NullPointerException);
hasError = true;
if (errorIndex == -1 restartingNodeIndex.get() == -1) {
// Not a datanode issue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d129/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
index 04f523e..622b349 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestFileAppend.java
@@ -603,4 +603,26 @@ public class TestFileAppend{
cluster.shutdown();
}
}
+
+ @Test(timeout = 1)
+ public void testAppendCorruptedBlock() throws Exception {
+Configuration conf = new HdfsConfiguration();
+conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
+conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 1);
+conf.setInt(dfs.min.replication, 1);
+MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+.build();
+try {
+ DistributedFileSystem fs = cluster.getFileSystem();
+ Path fileName = new Path(/appendCorruptBlock);
+ DFSTestUtil.createFile(fs, fileName, 512, (short) 1, 0);
+ DFSTestUtil.waitReplication(fs, fileName, (short) 1);
+ Assert.assertTrue(File not created, fs.exists(fileName));
+ ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, fileName);
+ cluster.corruptBlockOnDataNodes(block);
+ DFSTestUtil.appendFile(fs, fileName, appendCorruptBlock);
+} finally {
+ cluster.shutdown();
+}
+ }
}
hadoop git commit: HDFS-8067. haadmin prints out stale help messages (Contributed by Ajith S)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 00954714a - d0fff5d32
HDFS-8067. haadmin prints out stale help messages (Contributed by Ajith S)
(cherry picked from commit 66988476d09a6d04c0b81a663db1e6e5a28c37fb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0fff5d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0fff5d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0fff5d3
Branch: refs/heads/branch-2
Commit: d0fff5d3282a6a823a29b2ed651adbe6cec3caff
Parents: 0095471
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 10:45:21 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 10:47:23 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 2 ++
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0fff5d3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fd2684c..17f9144 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -330,6 +330,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8257. Namenode rollingUpgrade option is incorrect in document
(J.Andreina via vinayakumarb)
+HDFS-8067. haadmin prints out stale help messages (Ajith S via
vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0fff5d3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
index aa69dca..4c0ddb2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
@@ -90,7 +90,7 @@ public class DFSHAAdmin extends HAAdmin {
@Override
protected String getUsageString() {
-return Usage: haadmin;
+return Usage: haadmin [-ns nameserviceId];
}
@Override
hadoop git commit: HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun() ( Contributed by Sanghyun Yun)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7fe1d7400 - 089d420a8
HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun() (
Contributed by Sanghyun Yun)
(cherry picked from commit e5e492a9631ff78302fccedcb64d7b64b9407991)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/089d420a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/089d420a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/089d420a
Branch: refs/heads/branch-2
Commit: 089d420a824ced46b806d0e5156999f0abad66aa
Parents: 7fe1d74
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 11:25:24 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 11:26:56 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/089d420a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a51968d..ab00727 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -334,6 +334,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8174. Update replication count to live rep count in fsck report.
(J.Andreina)
+HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun()
+(Sanghyun Yun via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/089d420a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index 5981b4f..7038dc0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -320,8 +320,9 @@ public class BootstrapStandby implements Tool, Configurable
{
image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
hash);
} catch (IOException ioe) {
- image.close();
throw ioe;
+} finally {
+ image.close();
}
return 0;
}
hadoop git commit: HADOOP-11922. Misspelling of threshold in log4j.properties for tests in hadoop-tools (Contributed by Gabor Liptak)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c7f3dd1eb - 14d88a85e
HADOOP-11922. Misspelling of threshold in log4j.properties for tests in
hadoop-tools (Contributed by Gabor Liptak)
(cherry picked from commit f33efbc82709e75d9b4c4ed9835629668dbade7d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14d88a85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14d88a85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14d88a85
Branch: refs/heads/branch-2
Commit: 14d88a85e490cd844995b9d8daeff78752dca9c1
Parents: c7f3dd1
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 10:22:11 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 10:23:27 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-tools/hadoop-aws/src/test/resources/log4j.properties | 2 +-
hadoop-tools/hadoop-azure/src/test/resources/log4j.properties | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14d88a85/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index dc6705e..ba7e3be 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -170,6 +170,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11936. Dockerfile references a removed image (aw)
+HADOOP-11922. Misspelling of threshold in log4j.properties for tests
+in hadoop-tools (Gabor Liptak via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14d88a85/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
--
diff --git a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
index 1a6baae..ced0687 100644
--- a/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-aws/src/test/resources/log4j.properties
@@ -12,7 +12,7 @@
# log4j configuration used during build and unit tests
log4j.rootLogger=info,stdout
-log4j.threshhold=ALL
+log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}
(%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14d88a85/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
--
diff --git a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
index 81b935b..bb5cbe5 100644
--- a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
+++ b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties
@@ -17,7 +17,7 @@
# log4j configuration used during build and unit tests
log4j.rootLogger=INFO,stdout
-log4j.threshhold=ALL
+log4j.threshold=ALL
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2}
(%F:%M(%L)) - %m%n
hadoop git commit: HDFS-8257. Namenode rollingUpgrade option is incorrect in document (Contributed by J.Andreina)
Repository: hadoop Updated Branches: refs/heads/trunk f33efbc82 - c7c26a1e4 HDFS-8257. Namenode rollingUpgrade option is incorrect in document (Contributed by J.Andreina) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7c26a1e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7c26a1e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7c26a1e Branch: refs/heads/trunk Commit: c7c26a1e4aff0b89016ec838d06ba2b628a6808e Parents: f33efbc Author: Vinayakumar B vinayakum...@apache.org Authored: Fri May 8 10:35:25 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Fri May 8 10:35:25 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c26a1e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index b4f2042..4fcf524 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -651,6 +651,9 @@ Release 2.8.0 - UNRELEASED HDFS-8037. CheckAccess in WebHDFS silently accepts malformed FsActions parameters. (wheat9) +HDFS-8257. Namenode rollingUpgrade option is incorrect in document +(J.Andreina via vinayakumarb) + Release 2.7.1 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7c26a1e/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md index bdb051b..254a77c4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md @@ -419,7 +419,7 @@ Usage: [-upgrade [-clusterid cid] [-renameReservedk-v pairs] ] | [-upgradeOnly [-clusterid cid] [-renameReservedk-v pairs] ] | [-rollback] | - [-rollingUpgrade downgrade |rollback ] | + [-rollingUpgrade rollback |started ] | [-finalize] | [-importCheckpoint] | [-initializeSharedEdits] |
hadoop git commit: HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun() ( Contributed by Sanghyun Yun)
Repository: hadoop
Updated Branches:
refs/heads/trunk 8bafafadf - e5e492a96
HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun() (
Contributed by Sanghyun Yun)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5e492a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5e492a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5e492a9
Branch: refs/heads/trunk
Commit: e5e492a9631ff78302fccedcb64d7b64b9407991
Parents: 8bafafa
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 11:25:24 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 11:25:24 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e492a9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 916e327..76cd03b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -658,6 +658,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8174. Update replication count to live rep count in fsck report.
(J.Andreina)
+HDFS-6291. FSImage may be left unclosed in BootstrapStandby#doRun()
+(Sanghyun Yun via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5e492a9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
index 0accf53..88d9a6a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/BootstrapStandby.java
@@ -320,8 +320,9 @@ public class BootstrapStandby implements Tool, Configurable
{
image.saveDigestAndRenameCheckpointImage(NameNodeFile.IMAGE, imageTxId,
hash);
} catch (IOException ioe) {
- image.close();
throw ioe;
+} finally {
+ image.close();
}
return 0;
}
hadoop git commit: HDFS-8067. haadmin prints out stale help messages (Contributed by Ajith S)
Repository: hadoop
Updated Branches:
refs/heads/trunk c7c26a1e4 - 66988476d
HDFS-8067. haadmin prints out stale help messages (Contributed by Ajith S)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/66988476
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/66988476
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/66988476
Branch: refs/heads/trunk
Commit: 66988476d09a6d04c0b81a663db1e6e5a28c37fb
Parents: c7c26a1
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri May 8 10:45:21 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri May 8 10:45:21 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 2 ++
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java | 2 +-
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66988476/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4fcf524..b9443b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -654,6 +654,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8257. Namenode rollingUpgrade option is incorrect in document
(J.Andreina via vinayakumarb)
+HDFS-8067. haadmin prints out stale help messages (Ajith S via
vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/66988476/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
index aa69dca..4c0ddb2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSHAAdmin.java
@@ -90,7 +90,7 @@ public class DFSHAAdmin extends HAAdmin {
@Override
protected String getUsageString() {
-return Usage: haadmin;
+return Usage: haadmin [-ns nameserviceId];
}
@Override
hadoop git commit: HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands (Contributed by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 858800e89 - edfaff3c7
HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands
(Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edfaff3c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edfaff3c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edfaff3c
Branch: refs/heads/HDFS-7285
Commit: edfaff3c74a1bfc2dfea1d936a348ea140df04b0
Parents: 858800e
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue May 5 11:54:30 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue May 5 11:54:30 2015 +0530
--
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 +
.../hdfs/tools/erasurecode/ECCommand.java | 9 +-
.../hadoop/cli/CLITestCmdErasureCoding.java | 38 +++
.../apache/hadoop/cli/TestErasureCodingCLI.java | 114 +++
.../cli/util/CLICommandErasureCodingCli.java| 21 ++
.../cli/util/ErasureCodingCliCmdExecutor.java | 37 ++
.../test/resources/testErasureCodingConf.xml| 342 +++
7 files changed, 561 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edfaff3c/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index faec023..ef760fc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -166,3 +166,6 @@
(jing9)
HDFS-8137. Send the EC schema to DataNode via EC encoding/recovering
command(umamahesh)
+
+HDFS-8242. Erasure Coding: XML based end-to-end test for ECCli commands
+(Rakesh R via vinayakumarb)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edfaff3c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
index 84c2275..802a46d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/erasurecode/ECCommand.java
@@ -17,7 +17,9 @@
package org.apache.hadoop.hdfs.tools.erasurecode;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.LinkedList;
+import java.util.List;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -120,11 +122,12 @@ public abstract class ECCommand extends Command {
sb.append(Schema ');
sb.append(schemaName);
sb.append(' does not match any of the supported schemas.);
-sb.append(Please select any one of [);
+sb.append( Please select any one of );
+ListString schemaNames = new ArrayListString();
for (ECSchema ecSchema : ecSchemas) {
- sb.append(ecSchema.getSchemaName());
- sb.append(, );
+ schemaNames.add(ecSchema.getSchemaName());
}
+sb.append(schemaNames);
throw new HadoopIllegalArgumentException(sb.toString());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edfaff3c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
new file mode 100644
index 000..6c06a8d
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/CLITestCmdErasureCoding.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * p/
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * p/
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Reverted earlier commit
Repository: hadoop
Updated Branches:
refs/heads/trunk 338e88a19 - 318081ccd
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes
for infinite loop (Contributed by Vinayakumar B)
Reverted earlier commit
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/318081cc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/318081cc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/318081cc
Branch: refs/heads/trunk
Commit: 318081ccd7af1ec02ec18f35ea95c579326be728
Parents: 338e88a
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue May 5 11:05:37 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue May 5 11:05:37 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ---
.../apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java | 4
2 files changed, 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/318081cc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 365b005..cd9b7b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -664,9 +664,6 @@ Release 2.7.1 - UNRELEASED
HDFS-8179. DFSClient#getServerDefaults returns null within 1
hour of system start. (Xiaoyu Yao via Arpit Agarwal)
-HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
-goes for infinite loop (vinayakumarb)
-
HDFS-8163. Using monotonicNow for block report scheduling causes
test failures on recently restarted systems. (Arpit Agarwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/318081cc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index 991b56d..7155eae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -60,9 +59,6 @@ public class ReportBadBlockAction implements
BPServiceActorAction {
try {
bpNamenode.reportBadBlocks(locatedBlock);
-} catch (StandbyException e) {
- DataNode.LOG.warn(Failed to report bad block + block
- + to standby namenode);
} catch (IOException e) {
throw new BPServiceActorActionException(Failed to report bad block
+ block + to namenode: );
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Reverted earlier commit
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 e7b6a076d - 69d837428
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes
for infinite loop (Contributed by Vinayakumar B)
Reverted earlier commit
(cherry picked from commit 318081ccd7af1ec02ec18f35ea95c579326be728)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69d83742
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69d83742
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69d83742
Branch: refs/heads/branch-2.7
Commit: 69d837428ca96a445cb662c8be06bf36e7034640
Parents: e7b6a07
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue May 5 11:05:37 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue May 5 11:07:59 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ---
.../apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java | 4
2 files changed, 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d83742/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 44571b4..2c209cd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -44,9 +44,6 @@ Release 2.7.1 - UNRELEASED
HDFS-8179. DFSClient#getServerDefaults returns null within 1
hour of system start. (Xiaoyu Yao via Arpit Agarwal)
-HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
-goes for infinite loop (vinayakumarb)
-
HDFS-8163. Using monotonicNow for block report scheduling causes
test failures on recently restarted systems. (Arpit Agarwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d83742/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index 991b56d..7155eae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -60,9 +59,6 @@ public class ReportBadBlockAction implements
BPServiceActorAction {
try {
bpNamenode.reportBadBlocks(locatedBlock);
-} catch (StandbyException e) {
- DataNode.LOG.warn(Failed to report bad block + block
- + to standby namenode);
} catch (IOException e) {
throw new BPServiceActorActionException(Failed to report bad block
+ block + to namenode: );
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Reverted earlier commit
Repository: hadoop
Updated Branches:
refs/heads/branch-2 aefadb838 - 01bdfd794
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes
for infinite loop (Contributed by Vinayakumar B)
Reverted earlier commit
(cherry picked from commit 318081ccd7af1ec02ec18f35ea95c579326be728)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01bdfd79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01bdfd79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01bdfd79
Branch: refs/heads/branch-2
Commit: 01bdfd794cf460ae0a399649eaae54676d101214
Parents: aefadb8
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue May 5 11:05:37 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue May 5 11:06:44 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ---
.../apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java | 4
2 files changed, 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01bdfd79/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a699a59..7b18365 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -340,9 +340,6 @@ Release 2.7.1 - UNRELEASED
HDFS-8179. DFSClient#getServerDefaults returns null within 1
hour of system start. (Xiaoyu Yao via Arpit Agarwal)
-HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
-goes for infinite loop (vinayakumarb)
-
HDFS-8163. Using monotonicNow for block report scheduling causes
test failures on recently restarted systems. (Arpit Agarwal)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01bdfd79/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index 991b56d..7155eae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
-import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -60,9 +59,6 @@ public class ReportBadBlockAction implements
BPServiceActorAction {
try {
bpNamenode.reportBadBlocks(locatedBlock);
-} catch (StandbyException e) {
- DataNode.LOG.warn(Failed to report bad block + block
- + to standby namenode);
} catch (IOException e) {
throw new BPServiceActorActionException(Failed to report bad block
+ block + to namenode: );
hadoop git commit: HDFS-8230. Erasure Coding: Ignore DatanodeProtocol#DNA_ERASURE_CODING_RECOVERY commands from standbynode if any (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 bc3091b18 - 89a4f3d65
HDFS-8230. Erasure Coding: Ignore DatanodeProtocol#DNA_ERASURE_CODING_RECOVERY
commands from standbynode if any (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89a4f3d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89a4f3d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89a4f3d6
Branch: refs/heads/HDFS-7285
Commit: 89a4f3d655f881470b651af74236ca7edf9e5bdd
Parents: bc3091b
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 28 14:14:33 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 28 14:14:33 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/BPOfferService.java| 1 +
2 files changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a4f3d6/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index e8db485..c28473b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -134,3 +134,6 @@
HDFS-8033. Erasure coding: stateful (non-positional) read from files in
striped layout (Zhe Zhang)
+
+HDFS-8230. Erasure Coding: Ignore
DatanodeProtocol#DNA_ERASURE_CODING_RECOVERY
+commands from standbynode if any (vinayakumarb)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89a4f3d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 69baac7..6606d0b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -757,6 +757,7 @@ class BPOfferService {
case DatanodeProtocol.DNA_BALANCERBANDWIDTHUPDATE:
case DatanodeProtocol.DNA_CACHE:
case DatanodeProtocol.DNA_UNCACHE:
+case DatanodeProtocol.DNA_ERASURE_CODING_RECOVERY:
LOG.warn(Got a command from standby NN - ignoring command: +
cmd.getAction());
break;
default:
hadoop git commit: HDFS-8189. ClientProtocol#createErasureCodingZone API was wrongly annotated as Idempotent (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 89a4f3d65 - cb77ab52b
HDFS-8189. ClientProtocol#createErasureCodingZone API was wrongly annotated as
Idempotent (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb77ab52
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb77ab52
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb77ab52
Branch: refs/heads/HDFS-7285
Commit: cb77ab52b968e9867cca55385792a6b6f9707267
Parents: 89a4f3d
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 28 14:24:17 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 28 14:24:17 2015 +0530
--
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 5 -
.../apache/hadoop/hdfs/protocol/ClientProtocol.java | 16
2 files changed, 12 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb77ab52/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index c28473b..6c5d7ce 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -136,4 +136,7 @@
striped layout (Zhe Zhang)
HDFS-8230. Erasure Coding: Ignore
DatanodeProtocol#DNA_ERASURE_CODING_RECOVERY
-commands from standbynode if any (vinayakumarb)
\ No newline at end of file
+commands from standbynode if any (vinayakumarb)
+
+HDFS-8189. ClientProtocol#createErasureCodingZone API was wrongly annotated
+as Idempotent (vinayakumarb)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb77ab52/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index c257cc1..e32ac22 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1364,14 +1364,6 @@ public interface ClientProtocol {
long prevId) throws IOException;
/**
- * Create an erasure coding zone with specified schema, if any, otherwise
- * default
- */
- @Idempotent
- public void createErasureCodingZone(String src, ECSchema schema)
- throws IOException;
-
- /**
* Set xattr of a file or directory.
* The name must be prefixed with the namespace followed by .. For example,
* user.attr.
@@ -1467,6 +1459,14 @@ public interface ClientProtocol {
public EventBatchList getEditsFromTxid(long txid) throws IOException;
/**
+ * Create an erasure coding zone with specified schema, if any, otherwise
+ * default
+ */
+ @AtMostOnce
+ public void createErasureCodingZone(String src, ECSchema schema)
+ throws IOException;
+
+ /**
* Gets the ECInfo for the specified file/directory
*
* @param src
hadoop git commit: HDFS-8217. During block recovery for truncate Log new Block Id in case of copy-on-truncate is true. (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk a8898445d - 262c1bc33
HDFS-8217. During block recovery for truncate Log new Block Id in case of
copy-on-truncate is true. (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/262c1bc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/262c1bc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/262c1bc3
Branch: refs/heads/trunk
Commit: 262c1bc3398ce2ede03f9d86fc97c35ca7a8e9db
Parents: a889844
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Apr 24 12:16:41 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Apr 24 12:16:41 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/datanode/DataNode.java| 4 +++-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 +++-
3 files changed, 13 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/262c1bc3/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b0a0a50..0e00025 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -545,6 +545,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7993. Provide each Replica details in fsck (J.Andreina via
vinayakumarb)
+HDFS-8217. During block recovery for truncate Log new Block Id in case of
+copy-on-truncate is true. (vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/262c1bc3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index e81da52..23ab43a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2840,7 +2840,9 @@ public class DataNode extends ReconfigurableBase
LOG.info(who + calls recoverBlock( + block
+ , targets=[ + Joiner.on(, ).join(targets) + ]
-+ , newGenerationStamp= + rb.getNewGenerationStamp() + ));
++ ((rb.getNewBlock() == null) ? , newGenerationStamp=
++ rb.getNewGenerationStamp() : , newBlock= + rb.getNewBlock())
++ ));
}
@Override // ClientDataNodeProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/262c1bc3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3599fad..4477dc4 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4229,6 +4229,8 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
String src = ;
waitForLoadingFSImage();
writeLock();
+boolean copyTruncate = false;
+BlockInfoContiguousUnderConstruction truncatedBlock = null;
try {
checkOperation(OperationCategory.WRITE);
// If a DN tries to commit to the standby, the recovery will
@@ -4285,11 +4287,10 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
return;
}
- BlockInfoContiguousUnderConstruction truncatedBlock =
- (BlockInfoContiguousUnderConstruction) iFile.getLastBlock();
+ truncatedBlock = (BlockInfoContiguousUnderConstruction) iFile
+ .getLastBlock();
long recoveryId = truncatedBlock.getBlockRecoveryId();
- boolean copyTruncate =
- truncatedBlock.getBlockId() != storedBlock.getBlockId();
+ copyTruncate = truncatedBlock.getBlockId() != storedBlock.getBlockId();
if(recoveryId != newgenerationstamp) {
throw new IOException(The recovery id + newgenerationstamp
+ does not match current recovery id
@@ -4382,7 +4383,8 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
if (closeFile
hadoop git commit: HDFS-8217. During block recovery for truncate Log new Block Id in case of copy-on-truncate is true. (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 3d0385c3c - 68063cac3
HDFS-8217. During block recovery for truncate Log new Block Id in case of
copy-on-truncate is true. (Contributed by Vinayakumar B)
(cherry picked from commit 262c1bc3398ce2ede03f9d86fc97c35ca7a8e9db)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68063cac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68063cac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68063cac
Branch: refs/heads/branch-2
Commit: 68063cac3e701d84217cfae8d15ed214af398803
Parents: 3d0385c
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Apr 24 12:16:41 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Apr 24 12:18:04 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/datanode/DataNode.java| 4 +++-
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 12 +++-
3 files changed, 13 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68063cac/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b3b0607..913040f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -227,6 +227,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7993. Provide each Replica details in fsck (J.Andreina via
vinayakumarb)
+HDFS-8217. During block recovery for truncate Log new Block Id in case of
+copy-on-truncate is true. (vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68063cac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index a13a31f..ba02be2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2847,7 +2847,9 @@ public class DataNode extends ReconfigurableBase
LOG.info(who + calls recoverBlock( + block
+ , targets=[ + Joiner.on(, ).join(targets) + ]
-+ , newGenerationStamp= + rb.getNewGenerationStamp() + ));
++ ((rb.getNewBlock() == null) ? , newGenerationStamp=
++ rb.getNewGenerationStamp() : , newBlock= + rb.getNewBlock())
++ ));
}
@Override // ClientDataNodeProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68063cac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 4249fec..f175301 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4225,6 +4225,8 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
String src = ;
waitForLoadingFSImage();
writeLock();
+boolean copyTruncate = false;
+BlockInfoContiguousUnderConstruction truncatedBlock = null;
try {
checkOperation(OperationCategory.WRITE);
// If a DN tries to commit to the standby, the recovery will
@@ -4281,11 +4283,10 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
return;
}
- BlockInfoContiguousUnderConstruction truncatedBlock =
- (BlockInfoContiguousUnderConstruction) iFile.getLastBlock();
+ truncatedBlock = (BlockInfoContiguousUnderConstruction) iFile
+ .getLastBlock();
long recoveryId = truncatedBlock.getBlockRecoveryId();
- boolean copyTruncate =
- truncatedBlock.getBlockId() != storedBlock.getBlockId();
+ copyTruncate = truncatedBlock.getBlockId() != storedBlock.getBlockId();
if(recoveryId != newgenerationstamp) {
throw new IOException(The recovery id + newgenerationstamp
+ does not match current recovery id
@@ -4378,7 +4379,8 @@ public class
hadoop git commit: HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds (Contributed by J.Andreina and Xiaoyu Yao)
Repository: hadoop
Updated Branches:
refs/heads/trunk 262c1bc33 - c8d72907f
HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds
(Contributed by J.Andreina and Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c8d72907
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c8d72907
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c8d72907
Branch: refs/heads/trunk
Commit: c8d72907ff5a4cb9ce1effca8ad9b69689d11d1d
Parents: 262c1bc
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Apr 24 12:51:04 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Apr 24 12:51:04 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 7 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 ++
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java| 2 ++
.../apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 2 ++
5 files changed, 16 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8d72907/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0e00025..b442bad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8217. During block recovery for truncate Log new Block Id in case of
copy-on-truncate is true. (vinayakumarb)
+HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds
+(J.Andreina and Xiaoyu Yao via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c8d72907/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 63145b0..8fc9e77 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1425,6 +1425,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
ParentNotDirectoryException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
@@ -1467,6 +1468,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
FileNotFoundException.class,
SafeModeException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnsupportedOperationException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
@@ -1542,6 +1544,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
FileNotFoundException.class,
SafeModeException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
@@ -1598,6 +1601,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
throw re.unwrapRemoteException(AccessControlException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
@@ -1635,6 +1639,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
} catch
hadoop git commit: HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds (Contributed by J.Andreina and Xiaoyu Yao)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 68063cac3 - 2ec356fcd
HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds
(Contributed by J.Andreina and Xiaoyu Yao)
(cherry picked from commit c8d72907ff5a4cb9ce1effca8ad9b69689d11d1d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ec356fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ec356fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ec356fc
Branch: refs/heads/branch-2
Commit: 2ec356fcdb6d8e8f8167030d38c86e230dbbddcf
Parents: 68063ca
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Apr 24 12:51:04 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Apr 24 12:51:56 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 7 +++
.../src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java | 2 ++
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java| 2 ++
.../apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 2 ++
5 files changed, 16 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ec356fc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 913040f..faf2320 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -230,6 +230,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8217. During block recovery for truncate Log new Block Id in case of
copy-on-truncate is true. (vinayakumarb)
+HDFS-8231. StackTrace displayed at client while QuotaByStorageType exceeds
+(J.Andreina and Xiaoyu Yao via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ec356fc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index b241815..22ed86f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1427,6 +1427,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
ParentNotDirectoryException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
@@ -1469,6 +1470,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
FileNotFoundException.class,
SafeModeException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnsupportedOperationException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
@@ -1544,6 +1546,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
FileNotFoundException.class,
SafeModeException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
@@ -1600,6 +1603,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
throw re.unwrapRemoteException(AccessControlException.class,
NSQuotaExceededException.class,
DSQuotaExceededException.class,
+ QuotaByStorageTypeExceededException.class,
UnresolvedPathException.class,
SnapshotAccessControlException.class);
} finally {
@@ -1637,6 +1641,7 @@ public class
hadoop git commit: HDFS-7993. Provide each Replica details in fsck (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/trunk d52de6154 - 8ddbb8dd4
HDFS-7993. Provide each Replica details in fsck (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ddbb8dd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ddbb8dd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ddbb8dd
Branch: refs/heads/trunk
Commit: 8ddbb8dd433862509bd9b222dddafe2c3a74778a
Parents: d52de61
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 21 15:24:49 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 21 15:24:49 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../blockmanagement/DatanodeStorageInfo.java| 6 +-
.../hdfs/server/namenode/NamenodeFsck.java | 61 ++---
.../org/apache/hadoop/hdfs/tools/DFSck.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 71
5 files changed, 132 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ddbb8dd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1aa9ce4..6951a08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -526,6 +526,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to
create (surendra singh lilhore via vinayakumarb)
+HDFS-7993. Provide each Replica details in fsck (J.Andreina via
vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ddbb8dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 8c752ac..c6c9001 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -155,7 +155,7 @@ public class DatanodeStorageInfo {
this.blockReportCount = blockReportCount;
}
- boolean areBlockContentsStale() {
+ public boolean areBlockContentsStale() {
return blockContentsStale;
}
@@ -205,11 +205,11 @@ public class DatanodeStorageInfo {
return getState() == State.FAILED numBlocks != 0;
}
- String getStorageID() {
+ public String getStorageID() {
return storageID;
}
- StorageType getStorageType() {
+ public StorageType getStorageType() {
return storageType;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ddbb8dd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index a8586dd..afaec87 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -68,9 +69,11 @@ import
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import
hadoop git commit: HDFS-7993. Provide each Replica details in fsck (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 27fc4fb99 - e7bb0fc92
HDFS-7993. Provide each Replica details in fsck (Contributed by J.Andreina)
(cherry picked from commit 8ddbb8dd433862509bd9b222dddafe2c3a74778a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7bb0fc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7bb0fc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7bb0fc9
Branch: refs/heads/branch-2
Commit: e7bb0fc922295fe8086797b521fb721a06ac6137
Parents: 27fc4fb
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 21 15:24:49 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 21 15:27:15 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../blockmanagement/DatanodeStorageInfo.java| 6 +-
.../hdfs/server/namenode/NamenodeFsck.java | 61 ++---
.../org/apache/hadoop/hdfs/tools/DFSck.java | 6 +-
.../hadoop/hdfs/server/namenode/TestFsck.java | 71
5 files changed, 132 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7bb0fc9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5c5e59f..d8558e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -208,6 +208,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to
create (surendra singh lilhore via vinayakumarb)
+HDFS-7993. Provide each Replica details in fsck (J.Andreina via
vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7bb0fc9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 8c752ac..c6c9001 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -155,7 +155,7 @@ public class DatanodeStorageInfo {
this.blockReportCount = blockReportCount;
}
- boolean areBlockContentsStale() {
+ public boolean areBlockContentsStale() {
return blockContentsStale;
}
@@ -205,11 +205,11 @@ public class DatanodeStorageInfo {
return getState() == State.FAILED numBlocks != 0;
}
- String getStorageID() {
+ public String getStorageID() {
return storageID;
}
- StorageType getStorageType() {
+ public StorageType getStorageType() {
return storageType;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7bb0fc9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
index a1ad4d3..52fc105 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.hdfs.net.TcpPeerServer;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -68,9 +69,11 @@ import
org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.NumberReplicas;
import
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Updated CHANGES.TXT for correct version
Repository: hadoop Updated Branches: refs/heads/trunk 44872b76f - ed4137ceb HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Updated CHANGES.TXT for correct version Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed4137ce Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed4137ce Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed4137ce Branch: refs/heads/trunk Commit: ed4137cebf27717e9c79eae515b0b83ab6676465 Parents: 44872b7 Author: Vinayakumar B vinayakum...@apache.org Authored: Tue Apr 21 07:59:43 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Tue Apr 21 07:59:43 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed4137ce/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 2d20812..2291855 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -488,9 +488,6 @@ Release 2.8.0 - UNRELEASED HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe) -HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor -goes for infinite loop (vinayakumarb) - HDFS-5215. dfs.datanode.du.reserved is not considered while computing available space ( Brahma Reddy Battula via Yongjun Zhang) @@ -564,6 +561,9 @@ Release 2.7.1 - UNRELEASED HDFS-8179. DFSClient#getServerDefaults returns null within 1 hour of system start. (Xiaoyu Yao via Arpit Agarwal) +HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor +goes for infinite loop (vinayakumarb) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Updated CHANGES.TXT for correct version
Repository: hadoop Updated Branches: refs/heads/branch-2 d0ea982e6 - 961051e56 HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B) Updated CHANGES.TXT for correct version (cherry picked from commit ed4137cebf27717e9c79eae515b0b83ab6676465) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/961051e5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/961051e5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/961051e5 Branch: refs/heads/branch-2 Commit: 961051e569151d68d90f91055c8678c034c20207 Parents: d0ea982 Author: Vinayakumar B vinayakum...@apache.org Authored: Tue Apr 21 07:59:43 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Tue Apr 21 08:00:37 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/961051e5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 0da2516..122c23a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -173,9 +173,6 @@ Release 2.8.0 - UNRELEASED HDFS-7922. ShortCircuitCache#close is not releasing ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe) -HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor -goes for infinite loop (vinayakumarb) - HDFS-5215. dfs.datanode.du.reserved is not considered while computing available space ( Brahma Reddy Battula via Yongjun Zhang) @@ -246,6 +243,9 @@ Release 2.7.1 - UNRELEASED HDFS-8179. DFSClient#getServerDefaults returns null within 1 hour of system start. (Xiaoyu Yao via Arpit Agarwal) +HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor +goes for infinite loop (vinayakumarb) + Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 cc42ad03d - 1c8d54bc0
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes
for infinite loop (Contributed by Vinayakumar B)
(cherry picked from commit 867d5d2675b8fb73c40fac1e581b02b005459d95)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c8d54bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c8d54bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c8d54bc
Branch: refs/heads/branch-2.7
Commit: 1c8d54bc04cf12a404a8d53e87eb198c16720ac8
Parents: cc42ad0
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Apr 2 08:12:00 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 21 07:45:59 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/ReportBadBlockAction.java| 8 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8d54bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b2d2aff..a169e87 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -35,6 +35,9 @@ Release 2.7.1 - UNRELEASED
HDFS-8179. DFSClient#getServerDefaults returns null within 1
hour of system start. (Xiaoyu Yao via Arpit Agarwal)
+HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
+goes for infinite loop (vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c8d54bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index fd01a01..991b56d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -58,8 +59,11 @@ public class ReportBadBlockAction implements
BPServiceActorAction {
dnArr, uuids, types) };
try {
- bpNamenode.reportBadBlocks(locatedBlock);
-} catch (IOException e){
+ bpNamenode.reportBadBlocks(locatedBlock);
+} catch (StandbyException e) {
+ DataNode.LOG.warn(Failed to report bad block + block
+ + to standby namenode);
+} catch (IOException e) {
throw new BPServiceActorActionException(Failed to report bad block
+ block + to namenode: );
}
hadoop git commit: HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to create (Contributed by surendra singh lilhore)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5d3a4d51b - c562e1b19
HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to
create (Contributed by surendra singh lilhore)
(cherry picked from commit c17cd4f7c163cf97d6e42865df25709f29d39e3d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c562e1b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c562e1b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c562e1b1
Branch: refs/heads/branch-2
Commit: c562e1b19be435733d405f80f0f5b27e11dc2e7b
Parents: 5d3a4d5
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Apr 20 14:58:04 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Apr 20 14:58:59 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 10 ++
2 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c562e1b1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7da514a..1acb906 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -208,6 +208,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8043. NPE in MiniDFSCluster teardown. (Brahma Reddy Battula via ozawa)
+HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to
+create (surendra singh lilhore via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c562e1b1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 723b322..a13a31f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1728,10 +1728,12 @@ public class DataNode extends ReconfigurableBase
LOG.warn(Exception shutting down DataNode, e);
}
}
-try {
- httpServer.close();
-} catch (Exception e) {
- LOG.warn(Exception shutting down DataNode HttpServer, e);
+if (httpServer != null) {
+ try {
+httpServer.close();
+ } catch (Exception e) {
+LOG.warn(Exception shutting down DataNode HttpServer, e);
+ }
}
if (pauseMonitor != null) {
hadoop git commit: HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to create (Contributed by surendra singh lilhore)
Repository: hadoop
Updated Branches:
refs/heads/trunk 5c97db07f - c17cd4f7c
HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to
create (Contributed by surendra singh lilhore)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c17cd4f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c17cd4f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c17cd4f7
Branch: refs/heads/trunk
Commit: c17cd4f7c163cf97d6e42865df25709f29d39e3d
Parents: 5c97db0
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Apr 20 14:58:04 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Apr 20 14:58:04 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/DataNode.java | 10 ++
2 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c17cd4f7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9bb1fd4..8dec32e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -526,6 +526,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8043. NPE in MiniDFSCluster teardown. (Brahma Reddy Battula via ozawa)
+HDFS-8173. NPE thrown at DataNode shutdown when HTTP server was not able to
+create (surendra singh lilhore via vinayakumarb)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c17cd4f7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 0ddb99c..e81da52 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1721,10 +1721,12 @@ public class DataNode extends ReconfigurableBase
LOG.warn(Exception shutting down DataNode, e);
}
}
-try {
- httpServer.close();
-} catch (Exception e) {
- LOG.warn(Exception shutting down DataNode HttpServer, e);
+if (httpServer != null) {
+ try {
+httpServer.close();
+ } catch (Exception e) {
+LOG.warn(Exception shutting down DataNode HttpServer, e);
+ }
}
if (pauseMonitor != null) {
hadoop git commit: HDFS-8181. createErasureCodingZone sets retryCache state as false always (Contributed by Uma Maheswara Rao G)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 c6114e64a - bcf4c205e
HDFS-8181. createErasureCodingZone sets retryCache state as false always
(Contributed by Uma Maheswara Rao G)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bcf4c205
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bcf4c205
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bcf4c205
Branch: refs/heads/HDFS-7285
Commit: bcf4c205e1a4764391b9d8fb8e16c3080b8b9537
Parents: c6114e6
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Apr 20 15:04:49 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Apr 20 15:04:49 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +++
.../org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 1 +
2 files changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcf4c205/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 40517e7..87c6b1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -90,3 +90,6 @@
HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for
making it ready for transfer to DN (Uma Maheswara Rao G via vinayakumarb)
+
+HDFS-8181. createErasureCodingZone sets retryCache state as false always
+(Uma Maheswara Rao G via vinayakumarb)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bcf4c205/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 06aee59..487705a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -1847,6 +1847,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
boolean success = false;
try {
namesystem.createErasureCodingZone(src, schema, cacheEntry != null);
+ success = true;
} finally {
RetryCache.setState(cacheEntry, success);
}
hadoop git commit: HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B) Updated again
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 bcf4c205e - b0fdfb054 HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B) Updated again Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0fdfb05 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0fdfb05 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0fdfb05 Branch: refs/heads/HDFS-7285 Commit: b0fdfb0548572dab229055813def7e76306564f8 Parents: bcf4c20 Author: Vinayakumar B vinayakum...@apache.org Authored: Mon Apr 20 15:12:18 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Mon Apr 20 15:12:18 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 11 +++ 1 file changed, 11 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0fdfb05/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 87c6b1d..c8dbf08 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -83,11 +83,22 @@ HDFS-7349. Support DFS command for the EC encoding (vinayakumarb) +HDFS-8120. Erasure coding: created util class to analyze striped block groups. +(Contributed by Zhe Zhang and Li Bo via Jing Zhao) + HDFS-7994. Detect if resevered EC Block ID is already used during namenode startup. (Hui Zheng via szetszwo) HDFS-8167. BlockManager.addBlockCollectionWithCheck should check if the block is a striped block. (Hui Zheng via zhz). +HDFS-8166. DFSStripedOutputStream should not create empty blocks. (Jing Zhao) + +HDFS-7937. Erasure Coding: INodeFile quota computation unit tests. +(Kai Sasaki via Jing Zhao) + +HDFS-8145. Fix the editlog corruption exposed by failed TestAddStripedBlocks. +(Jing Zhao) + HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for making it ready for transfer to DN (Uma Maheswara Rao G via vinayakumarb)
hadoop git commit: HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for making it ready for transfer to DN (Contributed by Uma Maheswara Rao G)
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 e01dab0a9 - c6114e64a HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for making it ready for transfer to DN (Contributed by Uma Maheswara Rao G) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6114e64 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6114e64 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6114e64 Branch: refs/heads/HDFS-7285 Commit: c6114e64a4ba38f489d45c7c34234c17d30a35f1 Parents: e01dab0 Author: Vinayakumar B vinayakum...@apache.org Authored: Sat Apr 18 23:20:45 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Sat Apr 18 23:20:45 2015 +0530 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 3 + .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 137 ++- .../blockmanagement/DatanodeDescriptor.java | 31 + .../server/blockmanagement/DatanodeManager.java | 4 +- .../server/protocol/BlockECRecoveryCommand.java | 80 ++- .../hdfs/server/protocol/DatanodeProtocol.java | 2 +- .../src/main/proto/DatanodeProtocol.proto | 8 ++ .../src/main/proto/erasurecoding.proto | 13 ++ .../hadoop/hdfs/protocolPB/TestPBHelper.java| 88 .../namenode/TestRecoverStripedBlocks.java | 10 +- 10 files changed, 335 insertions(+), 41 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6114e64/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 0ed61cd..40517e7 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -87,3 +87,6 @@ startup. (Hui Zheng via szetszwo) HDFS-8167. BlockManager.addBlockCollectionWithCheck should check if the block is a striped block. (Hui Zheng via zhz). + +HDFS-8146. Protobuf changes for BlockECRecoveryCommand and its fields for +making it ready for transfer to DN (Uma Maheswara Rao G via vinayakumarb) http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6114e64/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 33b6a2a..5cd256f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -28,6 +28,7 @@ import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.EnumSet; import java.util.HashMap; import java.util.List; @@ -99,7 +100,7 @@ import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTyp import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.FsActionProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclStatusProto; import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto; -import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos; +import org.apache.hadoop.hdfs.protocol.proto.*; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveEntryProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoExpirationProto; import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto; @@ -120,6 +121,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmI import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BalancerBandwidthCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockCommandProto; +import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockECRecoveryCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockIdCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.BlockRecoveryCommandProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.DatanodeCommandProto; @@ -131,11 +133,11 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDele import
hadoop git commit: HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B)
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 a6daad049 - 91e7bf81d HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91e7bf81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91e7bf81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91e7bf81 Branch: refs/heads/HDFS-7285 Commit: 91e7bf81dc966d37e3fbebd9ff8aed2ed546f76d Parents: a6daad0 Author: Vinayakumar B vinayakum...@apache.org Authored: Wed Apr 15 12:23:07 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Wed Apr 15 12:23:07 2015 +0530 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 15 +++ 1 file changed, 15 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/91e7bf81/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 07bbd4a..9fdac98 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -51,11 +51,20 @@ HDFS-7839. Erasure coding: implement facilities in NameNode to create and manage EC zones (Zhe Zhang) +HDFS-7969. Erasure coding: NameNode support for lease recovery of striped +block groups. (Zhe Zhang) + +HDFS-7782. Erasure coding: pread from files in striped layout. +(Zhe Zhang and Jing Zhao via Zhe Zhang) + HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from NameNode (vinayakumarb) HDFS-8074. Define a system-wide default EC schema. (Kai Zheng) +HDFS-8077. Erasure coding: fix bugs in EC zone and symlinks. +(Jing Zhao and Zhe Zhang via Jing Zhao) + HDFS-8104. Make hard-coded values consistent with the system default schema first before remove them. (Kai Zheng) HDFS-7889. Subclass DFSOutputStream to support writing striping layout files. (Li Bo via Kai Zheng) @@ -63,5 +72,11 @@ HDFS-8090. Erasure Coding: Add RPC to client-namenode to list all ECSchemas loaded in Namenode. (vinayakumarb) +HDFS-8122. Erasure Coding: Support specifying ECSchema during creation of ECZone. +(Vinayakumar B via Zhe Zhang) + +HDFS-8114. Erasure coding: Add auditlog FSNamesystem#createErasureCodingZone if this +operation fails. (Rakesh R via Zhe Zhang) + HDFS-8123. Erasure Coding: Better to move EC related proto messages to a separate erasurecoding proto file (Rakesh R via vinayakumarb) \ No newline at end of file
hadoop git commit: HDFS-8123. Erasure Coding: Better to move EC related proto messages to a separate erasurecoding proto file (Contrubuted by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 6e202bac1 - a6daad049
HDFS-8123. Erasure Coding: Better to move EC related proto messages to a
separate erasurecoding proto file (Contrubuted by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6daad04
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6daad04
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6daad04
Branch: refs/heads/HDFS-7285
Commit: a6daad049b2ff3ea2fb1cd85fcc3edd8857d5472
Parents: 6e202ba
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Apr 15 12:09:16 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Apr 15 12:09:16 2015 +0530
--
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 5 +-
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 +
...tNamenodeProtocolServerSideTranslatorPB.java | 12 ++--
.../ClientNamenodeProtocolTranslatorPB.java | 13 ++--
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 6 +-
.../namenode/ErasureCodingZoneManager.java | 2 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 24 +--
.../src/main/proto/erasurecoding.proto | 74
.../hadoop-hdfs/src/main/proto/hdfs.proto | 27 ---
9 files changed, 96 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6daad04/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 5250dfa..07bbd4a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -61,4 +61,7 @@
HDFS-7889. Subclass DFSOutputStream to support writing striping layout
files. (Li Bo via Kai Zheng)
HDFS-8090. Erasure Coding: Add RPC to client-namenode to list all
-ECSchemas loaded in Namenode. (vinayakumarb)
\ No newline at end of file
+ECSchemas loaded in Namenode. (vinayakumarb)
+
+HDFS-8123. Erasure Coding: Better to move EC related proto messages to a
+separate erasurecoding proto file (Rakesh R via vinayakumarb)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6daad04/hadoop-hdfs-project/hadoop-hdfs/pom.xml
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index c11b963..a13a2bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -343,6 +343,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd;
includehdfs.proto/include
includeencryption.proto/include
includeinotify.proto/include
+ includeerasurecoding.proto/include
/includes
/source
output${project.build.directory}/generated-sources/java/output
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6daad04/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index cc5ca55..e3073ae 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -106,12 +106,8 @@ import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDat
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeReportResponseProto;
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportRequestProto;
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetDatanodeStorageReportResponseProto;
-import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetECSchemasRequestProto;
-import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetECSchemasResponseProto;
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidRequestProto;
import
org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.GetEditsFromTxidResponseProto;
-import
hadoop git commit: HDFS-7349. Support DFS command for the EC encoding (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 91e7bf81d - 295ccbca7
HDFS-7349. Support DFS command for the EC encoding (Contributed by Vinayakumar
B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/295ccbca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/295ccbca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/295ccbca
Branch: refs/heads/HDFS-7285
Commit: 295ccbca74a66d2dab89dd704908deb7071298e9
Parents: 91e7bf8
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Apr 15 16:38:22 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Apr 15 16:38:22 2015 +0530
--
.../main/java/org/apache/hadoop/fs/FsShell.java | 8 +-
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 4 +-
.../hadoop-hdfs/src/main/bin/hdfs | 5 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++
.../hadoop/hdfs/DistributedFileSystem.java | 32 +++
.../hadoop/hdfs/protocol/ClientProtocol.java| 9 +
.../apache/hadoop/hdfs/protocol/ECZoneInfo.java | 56 +
...tNamenodeProtocolServerSideTranslatorPB.java | 18 ++
.../ClientNamenodeProtocolTranslatorPB.java | 19 ++
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 12 ++
.../namenode/ErasureCodingZoneManager.java | 11 +-
.../hdfs/server/namenode/FSDirectory.java | 10 +
.../hdfs/server/namenode/FSNamesystem.java | 24 +++
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +
.../hadoop/hdfs/tools/erasurecode/ECCli.java| 48 +
.../hdfs/tools/erasurecode/ECCommand.java | 209 +++
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../src/main/proto/erasurecoding.proto | 15 ++
18 files changed, 502 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295ccbca/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index db73f6d..f873a01 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -111,6 +111,10 @@ public class FsShell extends Configured implements Tool {
return getTrash().getCurrentTrashDir();
}
+ protected String getUsagePrefix() {
+return usagePrefix;
+ }
+
// NOTE: Usage/Help are inner classes to allow access to outer methods
// that access commandFactory
@@ -194,7 +198,7 @@ public class FsShell extends Configured implements Tool {
}
} else {
// display help or usage for all commands
- out.println(usagePrefix);
+ out.println(getUsagePrefix());
// display list of short usages
ArrayListCommand instances = new ArrayListCommand();
@@ -218,7 +222,7 @@ public class FsShell extends Configured implements Tool {
}
private void printInstanceUsage(PrintStream out, Command instance) {
-out.println(usagePrefix + + instance.getUsage());
+out.println(getUsagePrefix() + + instance.getUsage());
}
private void printInstanceHelp(PrintStream out, Command instance) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295ccbca/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 9fdac98..b9fc6fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -79,4 +79,6 @@
operation fails. (Rakesh R via Zhe Zhang)
HDFS-8123. Erasure Coding: Better to move EC related proto messages to a
-separate erasurecoding proto file (Rakesh R via vinayakumarb)
\ No newline at end of file
+separate erasurecoding proto file (Rakesh R via vinayakumarb)
+
+HDFS-7349. Support DFS command for the EC encoding (vinayakumarb)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/295ccbca/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index f464261..84c79b8 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -134,6 +134,11 @@ case ${COMMAND} in
hadoop_debug Appending HADOOP_CLIENT_OPTS onto
hadoop git commit: HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app directory if it does not exist. (Contributed by Rohith)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d77705c98 - a75679cdb
HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app
directory if it does not exist. (Contributed by Rohith)
(cherry picked from commit b5a0b24643688270e77f85aaf1af24fee2c556cc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a75679cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a75679cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a75679cd
Branch: refs/heads/branch-2
Commit: a75679cdb33e07fb982bac02fbe295192a0eb78a
Parents: d77705c
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 14 11:58:12 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 14 11:59:07 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/http/HttpServerFunctionalTest.java| 12 +++-
2 files changed, 10 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a75679cd/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 742a054..c77bbb7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -50,6 +50,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
via aw)
+HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web
+app directory if it does not exist. (Rohith via vinayakumarb)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a75679cd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
index 4a4de41..faa27a7 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
@@ -123,11 +123,13 @@ public class HttpServerFunctionalTest extends Assert {
File testWebappDir = new File(webapps +
File.separatorChar + TEST);
try {
-if (!testWebappDir.exists()) {
- fail(Test webapp dir + testWebappDir.getCanonicalPath() + missing);
-}
-}
-catch (IOException e) {
+ if (!testWebappDir.exists()) {
+if (!testWebappDir.mkdirs()) {
+ fail(Test webapp dir + testWebappDir.getCanonicalPath()
+ + can not be created);
+}
+ }
+} catch (IOException e) {
}
}
hadoop git commit: HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app directory if it does not exist. (Contributed by Rohith)
Repository: hadoop
Updated Branches:
refs/heads/trunk d4e8c0959 - b5a0b2464
HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app
directory if it does not exist. (Contributed by Rohith)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5a0b246
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5a0b246
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5a0b246
Branch: refs/heads/trunk
Commit: b5a0b24643688270e77f85aaf1af24fee2c556cc
Parents: d4e8c09
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 14 11:58:12 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 14 11:58:12 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/http/HttpServerFunctionalTest.java| 12 +++-
2 files changed, 10 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5a0b246/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 68913bc..bb2eca5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -494,6 +494,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
via aw)
+HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web
+app directory if it does not exist. (Rohith via vinayakumarb)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5a0b246/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
index 4a4de41..faa27a7 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
@@ -123,11 +123,13 @@ public class HttpServerFunctionalTest extends Assert {
File testWebappDir = new File(webapps +
File.separatorChar + TEST);
try {
-if (!testWebappDir.exists()) {
- fail(Test webapp dir + testWebappDir.getCanonicalPath() + missing);
-}
-}
-catch (IOException e) {
+ if (!testWebappDir.exists()) {
+if (!testWebappDir.mkdirs()) {
+ fail(Test webapp dir + testWebappDir.getCanonicalPath()
+ + can not be created);
+}
+ }
+} catch (IOException e) {
}
}
hadoop git commit: HDFS-8111. NPE thrown when invalid FSImage filename given for 'hdfs oiv_legacy' cmd ( Contributed by surendra singh lilhore )
Repository: hadoop
Updated Branches:
refs/heads/branch-2 293c763e8 - 1071a57b2
HDFS-8111. NPE thrown when invalid FSImage filename given for 'hdfs oiv_legacy'
cmd ( Contributed by surendra singh lilhore )
(cherry picked from commit 14384f5b5142a98a10ce4bffadeb13e89bda9365)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1071a57b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1071a57b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1071a57b
Branch: refs/heads/branch-2
Commit: 1071a57b20a0e71d3e8525a84cdf46707cd3a7f2
Parents: 293c763
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Apr 13 17:16:05 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Apr 13 17:19:18 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 3 +++
.../hdfs/tools/offlineImageViewer/OfflineImageViewer.java | 6 +-
2 files changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1071a57b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 695b67d..d8a3673d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -164,6 +164,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7939. Two fsimage_rollback_* files are created which are not deleted
after rollback. (J.Andreina via vinayakumarb)
+HDFS-8111. NPE thrown when invalid FSImage filename given for
+'hdfs oiv_legacy' cmd ( surendra singh lilhore via vinayakumarb )
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1071a57b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index 08985d6..f1dd959 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -137,7 +137,11 @@ public class OfflineImageViewer {
done = true;
} finally {
if (!done) {
-LOG.error(image loading failed at offset + tracker.getPos());
+if (tracker != null) {
+ LOG.error(image loading failed at offset + tracker.getPos());
+} else {
+ LOG.error(Failed to load image file.);
+}
}
IOUtils.cleanup(LOG, in, tracker);
}
hadoop git commit: HDFS-8111. NPE thrown when invalid FSImage filename given for 'hdfs oiv_legacy' cmd ( Contributed by surendra singh lilhore )
Repository: hadoop
Updated Branches:
refs/heads/trunk f8f588720 - 14384f5b5
HDFS-8111. NPE thrown when invalid FSImage filename given for 'hdfs oiv_legacy'
cmd ( Contributed by surendra singh lilhore )
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14384f5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14384f5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14384f5b
Branch: refs/heads/trunk
Commit: 14384f5b5142a98a10ce4bffadeb13e89bda9365
Parents: f8f5887
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Apr 13 17:16:05 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Apr 13 17:16:05 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 3 +++
.../hdfs/tools/offlineImageViewer/OfflineImageViewer.java | 6 +-
2 files changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14384f5b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 134bba0..abbfe6a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -482,6 +482,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7939. Two fsimage_rollback_* files are created which are not deleted
after rollback. (J.Andreina via vinayakumarb)
+HDFS-8111. NPE thrown when invalid FSImage filename given for
+'hdfs oiv_legacy' cmd ( surendra singh lilhore via vinayakumarb )
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14384f5b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index 316ebca..7f81ba8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -137,7 +137,11 @@ public class OfflineImageViewer {
done = true;
} finally {
if (!done) {
-LOG.error(image loading failed at offset + tracker.getPos());
+if (tracker != null) {
+ LOG.error(image loading failed at offset + tracker.getPos());
+} else {
+ LOG.error(Failed to load image file.);
+}
}
IOUtils.cleanup(LOG, in, tracker);
}
hadoop git commit: HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and even for failed transfers (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk b1e059089 - 9d8952f97
HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and even
for failed transfers (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d8952f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d8952f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d8952f9
Branch: refs/heads/trunk
Commit: 9d8952f97f638ede27e4336b9601507d7bb1de7b
Parents: b1e0590
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Apr 9 11:58:00 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Apr 9 11:58:00 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/datanode/BPOfferService.java| 1 -
.../hadoop/hdfs/server/datanode/DataNode.java | 2 ++
.../datanode/metrics/DataNodeMetrics.java | 4 +--
.../server/datanode/TestDataNodeMetrics.java| 37 +---
5 files changed, 40 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d8952f9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 19f264a..74ed624 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -451,6 +451,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7725. Incorrect nodes in service metrics caused all writes to fail.
(Ming Ma via wang)
+HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and
+even for failed transfers (vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d8952f9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index da9642a..1b42b19 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -656,7 +656,6 @@ class BPOfferService {
// Send a copy of a block to another datanode
dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(),
bcmd.getTargets(), bcmd.getTargetStorageTypes());
- dn.metrics.incrBlocksReplicated(bcmd.getBlocks().length);
break;
case DatanodeProtocol.DNA_INVALIDATE:
//
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d8952f9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 50dccb8..8c08871 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2174,6 +2174,8 @@ public class DataNode extends ReconfigurableBase
+ Arrays.asList(targets));
}
}
+} else {
+ metrics.incrBlocksReplicated();
}
} catch (IOException ie) {
LOG.warn(bpReg + :Failed to transfer + b + to +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d8952f9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 2e8eb22..2e62b3c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -210,8 +210,8 @@ public class DataNodeMetrics {
cacheReports.add(latency);
}
- public void incrBlocksReplicated(int delta) {
-blocksReplicated.incr(delta
hadoop git commit: HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and even for failed transfers (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 351fac25a - 4e099c113
HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and even
for failed transfers (Contributed by Vinayakumar B)
(cherry picked from commit 9d8952f97f638ede27e4336b9601507d7bb1de7b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4e099c11
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4e099c11
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4e099c11
Branch: refs/heads/branch-2
Commit: 4e099c113b5823f9bce92a014d469083a7863c1a
Parents: 351fac2
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Apr 9 11:58:00 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Apr 9 11:59:05 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hdfs/server/datanode/BPOfferService.java| 1 -
.../hadoop/hdfs/server/datanode/DataNode.java | 2 ++
.../datanode/metrics/DataNodeMetrics.java | 4 +--
.../server/datanode/TestDataNodeMetrics.java| 37 +---
5 files changed, 40 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5a75f383..53b88dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -136,6 +136,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7725. Incorrect nodes in service metrics caused all writes to fail.
(Ming Ma via wang)
+HDFS-8096. DatanodeMetrics#blocksReplicated will get incremented early and
+even for failed transfers (vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
index 8efad83..67979f3 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPOfferService.java
@@ -657,7 +657,6 @@ class BPOfferService {
// Send a copy of a block to another datanode
dn.transferBlocks(bcmd.getBlockPoolId(), bcmd.getBlocks(),
bcmd.getTargets(), bcmd.getTargetStorageTypes());
- dn.metrics.incrBlocksReplicated(bcmd.getBlocks().length);
break;
case DatanodeProtocol.DNA_INVALIDATE:
//
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 6f70168..1285aaf 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -2181,6 +2181,8 @@ public class DataNode extends ReconfigurableBase
+ Arrays.asList(targets));
}
}
+} else {
+ metrics.incrBlocksReplicated();
}
} catch (IOException ie) {
LOG.warn(bpReg + :Failed to transfer + b + to +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4e099c11/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
index 2e8eb22..2e62b3c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetrics.java
@@ -210,8 +210,8 @@ public class DataNodeMetrics {
cacheReports.add(latency
hadoop git commit: HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from NameNode (Contributed by Vinayakumar B) Added missed file
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 d63800e84 - ead48867a
HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from
NameNode (Contributed by Vinayakumar B)
Added missed file
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ead48867
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ead48867
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ead48867
Branch: refs/heads/HDFS-7285
Commit: ead48867a5aa9ad608c93fa45e2c51022f62ced5
Parents: d63800e
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Apr 8 14:23:03 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Apr 8 14:23:03 2015 +0530
--
.../org/apache/hadoop/hdfs/protocol/ECInfo.java | 41
1 file changed, 41 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ead48867/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ECInfo.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ECInfo.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ECInfo.java
new file mode 100644
index 000..ca642c2
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ECInfo.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.io.erasurecode.ECSchema;
+
+/**
+ * Class to provide information, such as ECSchema, for a file/block.
+ */
+public class ECInfo {
+ private final String src;
+ private final ECSchema schema;
+
+ public ECInfo(String src, ECSchema schema) {
+this.src = src;
+this.schema = schema;
+ }
+
+ public String getSrc() {
+return src;
+ }
+
+ public ECSchema getSchema() {
+return schema;
+ }
+}
hadoop git commit: HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from NameNode (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 196774350 - d63800e84
HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from
NameNode (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d63800e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d63800e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d63800e8
Branch: refs/heads/HDFS-7285
Commit: d63800e84e64d4ce91953efe5604c3129d9f47c0
Parents: 1967743
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Apr 8 12:48:59 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Apr 8 12:48:59 2015 +0530
--
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 5 ++-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 14 ++
.../hadoop/hdfs/protocol/ClientProtocol.java| 10 +
...tNamenodeProtocolServerSideTranslatorPB.java | 19
.../ClientNamenodeProtocolTranslatorPB.java | 18
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 47 +++-
.../hdfs/server/namenode/FSNamesystem.java | 31 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 7 +++
.../src/main/proto/ClientNamenodeProtocol.proto | 10 +
.../hadoop-hdfs/src/main/proto/hdfs.proto | 28
.../hadoop/hdfs/TestErasureCodingZones.java | 38 +++-
11 files changed, 223 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d63800e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 9927ccf..7423033 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -49,4 +49,7 @@
(Hui Zheng via Zhe Zhang)
HDFS-7839. Erasure coding: implement facilities in NameNode to create and
-manage EC zones (Zhe Zhang)
\ No newline at end of file
+manage EC zones (Zhe Zhang)
+
+HDFS-8023. Erasure Coding: retrieve eraure coding schema for a file from
+NameNode (vinayakumarb)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d63800e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index bfb1022..a4cfc55 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -157,6 +157,7 @@ import
org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
+import org.apache.hadoop.hdfs.protocol.ECInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.EncryptionZoneIterator;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -3460,6 +3461,19 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
}
}
+ public ECInfo getErasureCodingInfo(String src) throws IOException {
+checkOpen();
+TraceScope scope = getPathTraceScope(getErasureCodingInfo, src);
+try {
+ return namenode.getErasureCodingInfo(src);
+} catch (RemoteException re) {
+ throw re.unwrapRemoteException(AccessControlException.class,
+ FileNotFoundException.class, UnresolvedPathException.class);
+} finally {
+ scope.close();
+}
+ }
+
public DFSInotifyEventInputStream getInotifyEventStream() throws IOException
{
return new DFSInotifyEventInputStream(traceSampler, namenode);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d63800e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 8efe344..45d92f3 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1464,4 +1464,14 @@ public interface
[2/2] hadoop git commit: HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang. Updated CHANGES-HDFS-EC-7285.txt
HADOOP-11740. Combine erasure encoder and decoder interfaces. Contributed by Zhe Zhang. Updated CHANGES-HDFS-EC-7285.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b839dd3b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b839dd3b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b839dd3b Branch: refs/heads/HDFS-7285 Commit: b839dd3b094b8842f2da3346859b6cb642c90a48 Parents: b494c30 Author: Vinayakumar B vinayakum...@apache.org Authored: Tue Apr 7 15:35:18 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Tue Apr 7 15:35:18 2015 +0530 -- hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt | 2 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 5 + 2 files changed, 3 insertions(+), 4 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b839dd3b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt index 68d1d32..7716728 100644 --- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt @@ -33,5 +33,7 @@ HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin ( Xinwei Qin via Kai Zheng ) +HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang) + HADOOP-11805 Better to rename some raw erasure coders. Contributed by Kai Zheng ( Kai Zheng ) http://git-wip-us.apache.org/repos/asf/hadoop/blob/b839dd3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 3874cb4..9927ccf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -49,7 +49,4 @@ (Hui Zheng via Zhe Zhang) HDFS-7839. Erasure coding: implement facilities in NameNode to create and -manage EC zones (Zhe Zhang) - -HADOOP-11740. Combine erasure encoder and decoder interfaces (Zhe Zhang) - +manage EC zones (Zhe Zhang) \ No newline at end of file
[1/2] hadoop git commit: HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin Updated CHANGES-HDFS-EC-7285.txt
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 1ceda4394 - b839dd3b0 HADOOP-11782 Correct two thrown messages in ECSchema class. Contributed by Xinwei Qin Updated CHANGES-HDFS-EC-7285.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b494c304 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b494c304 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b494c304 Branch: refs/heads/HDFS-7285 Commit: b494c304dfbafa330c8ca7ba29e3ca1d7600ecb0 Parents: 1ceda43 Author: Vinayakumar B vinayakum...@apache.org Authored: Tue Apr 7 15:34:37 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Tue Apr 7 15:34:37 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 --- 1 file changed, 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b494c304/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 4e60a7c..3874cb4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -48,9 +48,6 @@ HDFS-7617. Add unit tests for editlog transactions for EC (Hui Zheng via Zhe Zhang) -HADOOP-11782. Correct two thrown messages in ECSchema class -(Xinwei Qin via Kai Zheng) - HDFS-7839. Erasure coding: implement facilities in NameNode to create and manage EC zones (Zhe Zhang)
hadoop git commit: HADOOP-11645. Erasure Codec API covering the essential aspects for an erasure code ( Contributed by Kai Zheng)
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 b839dd3b0 - bd3f89108
HADOOP-11645. Erasure Codec API covering the essential aspects for an erasure
code ( Contributed by Kai Zheng)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bd3f8910
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bd3f8910
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bd3f8910
Branch: refs/heads/HDFS-7285
Commit: bd3f891085b70ecffa26b64a87a2d0c5ced98ceb
Parents: b839dd3
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 7 16:05:22 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 7 16:05:22 2015 +0530
--
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 3 +
.../hadoop/io/erasurecode/ECBlockGroup.java | 18
.../erasurecode/codec/AbstractErasureCodec.java | 88 +++
.../io/erasurecode/codec/ErasureCodec.java | 56
.../io/erasurecode/codec/RSErasureCodec.java| 38 +
.../io/erasurecode/codec/XORErasureCodec.java | 45 ++
.../erasurecode/coder/AbstractErasureCoder.java | 7 ++
.../io/erasurecode/coder/ErasureCoder.java | 7 ++
.../io/erasurecode/grouper/BlockGrouper.java| 90
9 files changed, 352 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd3f8910/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index 7716728..c72394e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -37,3 +37,6 @@
HADOOP-11805 Better to rename some raw erasure coders. Contributed by Kai
Zheng
( Kai Zheng )
+
+HADOOP-11645. Erasure Codec API covering the essential aspects for an
erasure code
+( Kai Zheng via vinayakumarb )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd3f8910/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
index 2c851a5..0a86907 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlockGroup.java
@@ -79,4 +79,22 @@ public class ECBlockGroup {
return false;
}
+ /**
+ * Get erased blocks count
+ * @return
+ */
+ public int getErasedCount() {
+int erasedCount = 0;
+
+for (ECBlock dataBlock : dataBlocks) {
+ if (dataBlock.isErased()) erasedCount++;
+}
+
+for (ECBlock parityBlock : parityBlocks) {
+ if (parityBlock.isErased()) erasedCount++;
+}
+
+return erasedCount;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bd3f8910/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
new file mode 100644
index 000..9993786
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/codec/AbstractErasureCodec.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode.codec;
+
+import org.apache.hadoop.conf.Configured;
+import
hadoop git commit: HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. (Contributed by Walter Su)
Repository: hadoop
Updated Branches:
refs/heads/trunk 4be648b55 - d505c8acd
HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be
easily overrided. (Contributed by Walter Su)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d505c8ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d505c8ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d505c8ac
Branch: refs/heads/trunk
Commit: d505c8acd30d6f40d0632fe9c93c886a4499a9fc
Parents: 4be648b
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Apr 8 09:56:37 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Apr 8 09:56:37 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../BlockPlacementPolicyDefault.java| 87
2 files changed, 54 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d505c8ac/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f194bd7..ac508cb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -385,6 +385,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8079. Move dfs.client.retry.* confs from DFSConfigKeys to
HdfsClientConfigKeys.Retry. (szetszwo)
+HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it
+can be easily overrided. (Walter Su via vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d505c8ac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 3262772..09db986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -333,41 +333,8 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
+ unavailableStorages= + unavailableStorages
+ , storagePolicy= + storagePolicy);
}
-
- if (numOfResults == 0) {
-writer = chooseLocalStorage(writer, excludedNodes, blocksize,
-maxNodesPerRack, results, avoidStaleNodes, storageTypes, true)
-.getDatanodeDescriptor();
-if (--numOfReplicas == 0) {
- return writer;
-}
- }
- final DatanodeDescriptor dn0 = results.get(0).getDatanodeDescriptor();
- if (numOfResults = 1) {
-chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
-results, avoidStaleNodes, storageTypes);
-if (--numOfReplicas == 0) {
- return writer;
-}
- }
- if (numOfResults = 2) {
-final DatanodeDescriptor dn1 = results.get(1).getDatanodeDescriptor();
-if (clusterMap.isOnSameRack(dn0, dn1)) {
- chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
-} else if (newBlock){
- chooseLocalRack(dn1, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
-} else {
- chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
-}
-if (--numOfReplicas == 0) {
- return writer;
-}
- }
- chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ writer = chooseTargetInOrder(numOfReplicas, writer, excludedNodes,
blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, newBlock, storageTypes);
} catch (NotEnoughReplicasException e) {
final String message = Failed to place enough replicas, still in need
of
+ (totalReplicasExpected - results.size()) + to reach
@@ -422,7 +389,55 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
}
return writer;
}
-
+
+ protected Node chooseTargetInOrder(int numOfReplicas,
+ Node writer
hadoop git commit: HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. (Contributed by Walter Su)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 eb3ca514f - 3cf7ac181
HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be
easily overrided. (Contributed by Walter Su)
(cherry picked from commit d505c8acd30d6f40d0632fe9c93c886a4499a9fc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cf7ac18
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cf7ac18
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cf7ac18
Branch: refs/heads/branch-2
Commit: 3cf7ac181ba9c6a31d4452d05d796901c5ed5517
Parents: eb3ca51
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Apr 8 09:56:37 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Apr 8 09:57:49 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../BlockPlacementPolicyDefault.java| 87
2 files changed, 54 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cf7ac18/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 687d7d5..e969644 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -70,6 +70,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8079. Move dfs.client.retry.* confs from DFSConfigKeys to
HdfsClientConfigKeys.Retry. (szetszwo)
+HDFS-8073. Split BlockPlacementPolicyDefault.chooseTarget(..) so it
+can be easily overrided. (Walter Su via vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cf7ac18/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 3262772..09db986 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -333,41 +333,8 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
+ unavailableStorages= + unavailableStorages
+ , storagePolicy= + storagePolicy);
}
-
- if (numOfResults == 0) {
-writer = chooseLocalStorage(writer, excludedNodes, blocksize,
-maxNodesPerRack, results, avoidStaleNodes, storageTypes, true)
-.getDatanodeDescriptor();
-if (--numOfReplicas == 0) {
- return writer;
-}
- }
- final DatanodeDescriptor dn0 = results.get(0).getDatanodeDescriptor();
- if (numOfResults = 1) {
-chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
-results, avoidStaleNodes, storageTypes);
-if (--numOfReplicas == 0) {
- return writer;
-}
- }
- if (numOfResults = 2) {
-final DatanodeDescriptor dn1 = results.get(1).getDatanodeDescriptor();
-if (clusterMap.isOnSameRack(dn0, dn1)) {
- chooseRemoteRack(1, dn0, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
-} else if (newBlock){
- chooseLocalRack(dn1, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
-} else {
- chooseLocalRack(writer, excludedNodes, blocksize, maxNodesPerRack,
- results, avoidStaleNodes, storageTypes);
-}
-if (--numOfReplicas == 0) {
- return writer;
-}
- }
- chooseRandom(numOfReplicas, NodeBase.ROOT, excludedNodes, blocksize,
- maxNodesPerRack, results, avoidStaleNodes, storageTypes);
+ writer = chooseTargetInOrder(numOfReplicas, writer, excludedNodes,
blocksize,
+ maxNodesPerRack, results, avoidStaleNodes, newBlock, storageTypes);
} catch (NotEnoughReplicasException e) {
final String message = Failed to place enough replicas, still in need
of
+ (totalReplicasExpected - results.size()) + to reach
@@ -422,7 +389,55 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
}
return writer;
}
-
+
+ protected Node chooseTargetInOrder(int numOfReplicas
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 27970b3f4 - beb0fd0d6
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes
for infinite loop (Contributed by Vinayakumar B)
(cherry picked from commit 867d5d2675b8fb73c40fac1e581b02b005459d95)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/beb0fd0d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/beb0fd0d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/beb0fd0d
Branch: refs/heads/branch-2
Commit: beb0fd0d601aff0ba993c2d48b83fe52edfb9065
Parents: 27970b3
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Apr 2 08:12:00 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Apr 2 08:12:51 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/ReportBadBlockAction.java| 8 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beb0fd0d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fc81ae0..bae6148 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -97,6 +97,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7922. ShortCircuitCache#close is not releasing
ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
+HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
+goes for infinite loop (vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/beb0fd0d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index fd01a01..991b56d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -58,8 +59,11 @@ public class ReportBadBlockAction implements
BPServiceActorAction {
dnArr, uuids, types) };
try {
- bpNamenode.reportBadBlocks(locatedBlock);
-} catch (IOException e){
+ bpNamenode.reportBadBlocks(locatedBlock);
+} catch (StandbyException e) {
+ DataNode.LOG.warn(Failed to report bad block + block
+ + to standby namenode);
+} catch (IOException e) {
throw new BPServiceActorActionException(Failed to report bad block
+ block + to namenode: );
}
hadoop git commit: HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes for infinite loop (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk f383fd9b6 - 867d5d267
HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor goes
for infinite loop (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/867d5d26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/867d5d26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/867d5d26
Branch: refs/heads/trunk
Commit: 867d5d2675b8fb73c40fac1e581b02b005459d95
Parents: f383fd9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Apr 2 08:12:00 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Apr 2 08:12:00 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/ReportBadBlockAction.java| 8 ++--
2 files changed, 9 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/867d5d26/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 518df9f..80d958d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -412,6 +412,9 @@ Release 2.8.0 - UNRELEASED
HDFS-7922. ShortCircuitCache#close is not releasing
ScheduledThreadPoolExecutors (Rakesh R via Colin P. McCabe)
+HDFS-7916. 'reportBadBlocks' from datanodes to standby Node BPServiceActor
+goes for infinite loop (vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/867d5d26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
index fd01a01..991b56d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReportBadBlockAction.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import
org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.ipc.StandbyException;
/**
* ReportBadBlockAction is an instruction issued by {{BPOfferService}} to
@@ -58,8 +59,11 @@ public class ReportBadBlockAction implements
BPServiceActorAction {
dnArr, uuids, types) };
try {
- bpNamenode.reportBadBlocks(locatedBlock);
-} catch (IOException e){
+ bpNamenode.reportBadBlocks(locatedBlock);
+} catch (StandbyException e) {
+ DataNode.LOG.warn(Failed to report bad block + block
+ + to standby namenode);
+} catch (IOException e) {
throw new BPServiceActorActionException(Failed to report bad block
+ block + to namenode: );
}
hadoop git commit: HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B)
Repository: hadoop Updated Branches: refs/heads/HDFS-7285 5ef6204c0 - 86a9b65df HDFS-8027. Erasure Coding: Update CHANGES-HDFS-7285.txt with branch commits (Vinayakumar B) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/86a9b65d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/86a9b65d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/86a9b65d Branch: refs/heads/HDFS-7285 Commit: 86a9b65df1d71c12eaf9cafb2a39e1dfb5f5f2c9 Parents: 5ef6204 Author: Vinayakumar B vinayakuma...@intel.com Authored: Tue Mar 31 15:12:09 2015 +0530 Committer: Vinayakumar B vinayakuma...@intel.com Committed: Tue Mar 31 15:12:09 2015 +0530 -- .../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt| 40 +++- 1 file changed, 39 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/86a9b65d/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt index 21e4c03..a686315 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt @@ -3,6 +3,44 @@ HDFS-7347. Configurable erasure coding policy for individual files and directories ( Zhe Zhang via vinayakumarb ) -HDFS-7716. Add a test for BlockGroup support in FSImage. +HDFS-7339. Representing striped block groups in NameNode with hierarchical +naming protocol ( Zhe Zhang ) + +HDFS-7652. Process block reports for erasure coded blocks (Zhe Zhang) + +HDFS-7716. Erasure Coding: extend BlockInfo to handle EC info (Jing Zhao) + +HDFS-7749. Erasure Coding: Add striped block support in INodeFile (Jing Zhao) + +HDFS-7837. Erasure Coding: allocate and persist striped blocks in NameNode +(Jing Zhao via Zhe Zhang) + +HDFS-7872. Erasure Coding: INodeFile.dumpTreeRecursively() supports to print +striped blocks (Takuya Fukudome via jing9) + +HDFS-7853. Erasure coding: extend LocatedBlocks to support reading from +striped files (Jing Zhao) + +HDFS-7826. Erasure Coding: Update INodeFile quota computation for striped +blocks ( Kai Sasaki via jing9 ) + +HDFS-7912. Erasure Coding: track BlockInfo instead of Block in +UnderReplicatedBlocks and PendingReplicationBlocks (Jing Zhao) + +HDFS-7369. Erasure coding: distribute recovery work for striped blocks to +DataNode (Zhe Zhang) + +HDFS-7864. Erasure Coding: Update safemode calculation for striped blocks +(GAO Rui via jing9) + +HDFS-7827. Erasure Coding: support striped blocks in non-protobuf fsimage +( Hui Zheng via jing9 ) + +HDFS-7616. Add a test for BlockGroup support in FSImage. (Takuya Fukudome via szetszwo) +HDFS-7907. Erasure Coding: track invalid, corrupt, and under-recovery striped +blocks in NameNode (Jing Zhao) + +HDFS-8005. Erasure Coding: simplify striped block recovery work computation +and add tests (Jing Zhao) \ No newline at end of file
hadoop git commit: HDFS-7890. Improve information on Top users for metrics in RollingWindowsManager and lower log level (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/trunk 232eca944 - 1ed9fb766
HDFS-7890. Improve information on Top users for metrics in
RollingWindowsManager and lower log level (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ed9fb76
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ed9fb76
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ed9fb76
Branch: refs/heads/trunk
Commit: 1ed9fb76645ecd195afe0067497dca10a3fb997d
Parents: 232eca9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Mar 30 10:02:48 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Mar 30 10:02:48 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/namenode/top/window/RollingWindowManager.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ed9fb76/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f4991da..9b1cc3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -353,6 +353,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6408. Remove redundant definitions in log4j.properties.
(Abhiraj Butala via aajisaka)
+HDFS-7890. Improve information on Top users for metrics in
+RollingWindowsManager and lower log level (J.Andreina via vinayakumarb)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ed9fb76/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
index 00e7087..4759cc8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
@@ -245,7 +245,7 @@ public class RollingWindowManager {
metricName, userName, windowSum);
topN.offer(new NameValuePair(userName, windowSum));
}
-LOG.info(topN size for command {} is: {}, metricName, topN.size());
+LOG.debug(topN users size for command {} is: {}, metricName,
topN.size());
return topN;
}
hadoop git commit: HDFS-7890. Improve information on Top users for metrics in RollingWindowsManager and lower log level (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7aded71f3 - 9f49b3e93
HDFS-7890. Improve information on Top users for metrics in
RollingWindowsManager and lower log level (Contributed by J.Andreina)
(cherry picked from commit 1ed9fb76645ecd195afe0067497dca10a3fb997d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f49b3e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f49b3e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f49b3e9
Branch: refs/heads/branch-2
Commit: 9f49b3e93d2d50c474a66a5dc8d6bcba1804ed1a
Parents: 7aded71
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Mar 30 10:02:48 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Mar 30 10:04:11 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/namenode/top/window/RollingWindowManager.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f49b3e9/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4e93a1d..151f71b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -38,6 +38,9 @@ Release 2.8.0 - UNRELEASED
HDFS-6408. Remove redundant definitions in log4j.properties.
(Abhiraj Butala via aajisaka)
+HDFS-7890. Improve information on Top users for metrics in
+RollingWindowsManager and lower log level (J.Andreina via vinayakumarb)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f49b3e9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
index 00e7087..4759cc8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/top/window/RollingWindowManager.java
@@ -245,7 +245,7 @@ public class RollingWindowManager {
metricName, userName, windowSum);
topN.offer(new NameValuePair(userName, windowSum));
}
-LOG.info(topN size for command {} is: {}, metricName, topN.size());
+LOG.debug(topN users size for command {} is: {}, metricName,
topN.size());
return topN;
}
hadoop git commit: HDFS-3325. When configuring 'dfs.namenode.safemode.threshold-pct' to a value greater or equal to 1 there is mismatch in the UI report (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 e7702ac6a - c3e2c20c9
HDFS-3325. When configuring 'dfs.namenode.safemode.threshold-pct' to a value
greater or equal to 1 there is mismatch in the UI report (Contributed by
J.Andreina)
(cherry picked from commit c6c396fcd69514ba93583268b2633557c3d74a47)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3e2c20c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3e2c20c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3e2c20c
Branch: refs/heads/branch-2
Commit: c3e2c20c9907d106e1cb9da0562c06de4975bc09
Parents: e7702ac
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 24 12:12:01 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 24 12:13:21 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +-
.../src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java | 2 +-
.../apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java| 2 +-
4 files changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3e2c20c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 17adae8..8977904 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -25,6 +25,10 @@ Release 2.8.0 - UNRELEASED
HDFS-7867. Update action param from start to prepare in rolling upgrade
javadoc (J.Andreina via vinayakumarb)
+HDFS-3325. When configuring dfs.namenode.safemode.threshold-pct to a
value
+greater or equal to 1 there is mismatch in the UI report
+(J.Andreina via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3e2c20c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index ad0c344..697b975 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5417,7 +5417,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
msg += String.format(
The reported blocks %d needs additional %d
+ blocks to reach the threshold %.4f of total blocks %d.%n,
- blockSafe, (blockThreshold - blockSafe) + 1, threshold, blockTotal);
+blockSafe, (blockThreshold - blockSafe), threshold,
blockTotal);
thresholdsMet = false;
} else {
msg += String.format(The reported blocks %d has reached the threshold
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3e2c20c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index 2d5bef2..80fe9ee 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -198,7 +198,7 @@ public class TestSafeMode {
String status = nn.getNamesystem().getSafemode();
assertEquals(Safe mode is ON. The reported blocks 0 needs additional +
-15 blocks to reach the threshold 0.9990 of total blocks 15. +
NEWLINE +
+14 blocks to reach the threshold 0.9990 of total blocks 15. +
NEWLINE +
The number of live datanodes 0 has reached the minimum number 0. +
Safe mode will be turned off automatically once the thresholds +
have been reached., status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3e2c20c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache
hadoop git commit: HDFS-3325. When configuring 'dfs.namenode.safemode.threshold-pct' to a value greater or equal to 1 there is mismatch in the UI report (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/trunk fbceb3b41 - c6c396fcd
HDFS-3325. When configuring 'dfs.namenode.safemode.threshold-pct' to a value
greater or equal to 1 there is mismatch in the UI report (Contributed by
J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6c396fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6c396fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6c396fc
Branch: refs/heads/trunk
Commit: c6c396fcd69514ba93583268b2633557c3d74a47
Parents: fbceb3b
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 24 12:12:01 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 24 12:12:01 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 2 +-
.../src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java | 2 +-
.../apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java| 2 +-
4 files changed, 7 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6c396fc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3ea1346..ee9a5db 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -338,6 +338,10 @@ Release 2.8.0 - UNRELEASED
HDFS-7867. Update action param from start to prepare in rolling upgrade
javadoc (J.Andreina via vinayakumarb)
+HDFS-3325. When configuring dfs.namenode.safemode.threshold-pct to a
value
+greater or equal to 1 there is mismatch in the UI report
+(J.Andreina via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6c396fc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 71c84b1..34b5e95 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5417,7 +5417,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
msg += String.format(
The reported blocks %d needs additional %d
+ blocks to reach the threshold %.4f of total blocks %d.%n,
- blockSafe, (blockThreshold - blockSafe) + 1, threshold, blockTotal);
+blockSafe, (blockThreshold - blockSafe), threshold,
blockTotal);
thresholdsMet = false;
} else {
msg += String.format(The reported blocks %d has reached the threshold
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6c396fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
index 2d5bef2..80fe9ee 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
@@ -198,7 +198,7 @@ public class TestSafeMode {
String status = nn.getNamesystem().getSafemode();
assertEquals(Safe mode is ON. The reported blocks 0 needs additional +
-15 blocks to reach the threshold 0.9990 of total blocks 15. +
NEWLINE +
+14 blocks to reach the threshold 0.9990 of total blocks 15. +
NEWLINE +
The number of live datanodes 0 has reached the minimum number 0. +
Safe mode will be turned off automatically once the thresholds +
have been reached., status);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6c396fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestHASafeMode.java
index c5aad9c..86f3e7b
hadoop git commit: HDFS-7869. Update action param from 'start' to 'prepare' in rolling upgrade ( Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/trunk 19b298f61 - 338b477ff
HDFS-7869. Update action param from 'start' to 'prepare' in rolling upgrade (
Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/338b477f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/338b477f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/338b477f
Branch: refs/heads/trunk
Commit: 338b477ff736cb6c8fb8204ab83fb038594dadf0
Parents: 19b298f
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Mar 19 11:59:31 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Mar 19 11:59:31 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/338b477f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ad34527..f78c9a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -330,6 +330,9 @@ Release 2.8.0 - UNRELEASED
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Rakesh R via vinayakumarb)
+HDFS-7869. Update action param from start to prepare in rolling upgrade
+javadoc (J.Andreina via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/338b477f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index e6bbd92..c750e79 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1226,7 +1226,7 @@ public class DistributedFileSystem extends FileSystem {
}
/**
- * Rolling upgrade: start/finalize/query.
+ * Rolling upgrade: prepare/finalize/query.
*/
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/338b477f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index d4fe903..2b07789 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -847,7 +847,7 @@ public interface ClientProtocol {
/**
* Rolling upgrade operations.
- * @param action either query, start or finailze.
+ * @param action either query, prepare or finalize.
* @return rolling upgrade information.
*/
@Idempotent
hadoop git commit: HDFS-7867. Update action param from 'start' to 'prepare' in rolling upgrade ( Contributed by J.Andreina) Updated JIRA id.
Repository: hadoop Updated Branches: refs/heads/branch-2 4527aa6f7 - 25515d142 HDFS-7867. Update action param from 'start' to 'prepare' in rolling upgrade ( Contributed by J.Andreina) Updated JIRA id. (cherry picked from commit 355e008ffbb0a33cb7d5782fc291abd62d684aaa) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/25515d14 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/25515d14 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/25515d14 Branch: refs/heads/branch-2 Commit: 25515d1421e617e40c8f2ec8f7f0c408bb2a7694 Parents: 4527aa6 Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Mar 19 12:10:39 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Thu Mar 19 12:11:34 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/25515d14/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 317f233..15b529a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -17,7 +17,7 @@ Release 2.8.0 - UNRELEASED HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Rakesh R via vinayakumarb) -HDFS-7869. Update action param from start to prepare in rolling upgrade +HDFS-7867. Update action param from start to prepare in rolling upgrade javadoc (J.Andreina via vinayakumarb) Release 2.7.0 - UNRELEASED
hadoop git commit: HDFS-7867. Update action param from 'start' to 'prepare' in rolling upgrade ( Contributed by J.Andreina) Updated JIRA id.
Repository: hadoop Updated Branches: refs/heads/trunk 338b477ff - 355e008ff HDFS-7867. Update action param from 'start' to 'prepare' in rolling upgrade ( Contributed by J.Andreina) Updated JIRA id. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/355e008f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/355e008f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/355e008f Branch: refs/heads/trunk Commit: 355e008ffbb0a33cb7d5782fc291abd62d684aaa Parents: 338b477 Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Mar 19 12:10:39 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Thu Mar 19 12:10:39 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/355e008f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index f78c9a3..b0331b5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -330,7 +330,7 @@ Release 2.8.0 - UNRELEASED HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Rakesh R via vinayakumarb) -HDFS-7869. Update action param from start to prepare in rolling upgrade +HDFS-7867. Update action param from start to prepare in rolling upgrade javadoc (J.Andreina via vinayakumarb) Release 2.7.0 - UNRELEASED
hadoop git commit: HDFS-7869. Update action param from 'start' to 'prepare' in rolling upgrade ( Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b36fc6bc9 - 4527aa6f7
HDFS-7869. Update action param from 'start' to 'prepare' in rolling upgrade (
Contributed by J.Andreina)
(cherry picked from commit 338b477ff736cb6c8fb8204ab83fb038594dadf0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4527aa6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4527aa6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4527aa6f
Branch: refs/heads/branch-2
Commit: 4527aa6f77ec817219951ddb6992d66fdf64fe91
Parents: b36fc6b
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Mar 19 11:59:31 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Mar 19 12:01:05 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 2 +-
.../main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4527aa6f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6faeb82..317f233 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -17,6 +17,9 @@ Release 2.8.0 - UNRELEASED
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Rakesh R via vinayakumarb)
+HDFS-7869. Update action param from start to prepare in rolling upgrade
+javadoc (J.Andreina via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4527aa6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 9443eaf..46b8a2a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -1270,7 +1270,7 @@ public class DistributedFileSystem extends FileSystem {
}
/**
- * Rolling upgrade: start/finalize/query.
+ * Rolling upgrade: prepare/finalize/query.
*/
public RollingUpgradeInfo rollingUpgrade(RollingUpgradeAction action)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4527aa6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index d4fe903..2b07789 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -847,7 +847,7 @@ public interface ClientProtocol {
/**
* Rolling upgrade operations.
- * @param action either query, start or finailze.
+ * @param action either query, prepare or finalize.
* @return rolling upgrade information.
*/
@Idempotent
hadoop git commit: HDFS.7849. Update documentation for enabling a new feature in rolling upgrade ( Contributed by J.Andreina )
Repository: hadoop Updated Branches: refs/heads/branch-2 ceb39c1cc - 786ba9784 HDFS.7849. Update documentation for enabling a new feature in rolling upgrade ( Contributed by J.Andreina ) (cherry picked from commit 5be5129d93481e9caae6a05b2361c463211a6d55) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/786ba978 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/786ba978 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/786ba978 Branch: refs/heads/branch-2 Commit: 786ba9784dac768c6270933c55b47e66fec0c28f Parents: ceb39c1 Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Mar 19 19:05:42 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Thu Mar 19 19:06:47 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml| 9 + 2 files changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/786ba978/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 9be5da8..2e5e8d1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -459,6 +459,9 @@ Release 2.7.0 - UNRELEASED HDFS-7054. Make DFSOutputStream tracing more fine-grained (cmccabe) +HDFS.7849. Update documentation for enabling a new feature in rolling +upgrade ( J.Andreina via vinayakumarb ) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/786ba978/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml index e7d8698..a62198f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml @@ -48,6 +48,15 @@ In order to upgrade a HDFS cluster without downtime, the cluster must be setup with HA. /p p +If there is any new feature which is enabled in new software release, may not work with old software release after upgrade. +In such cases upgrade should be done by following steps. + /p + ol +liDisable new feature./li +liUpgrade the cluster./li +liEnable the new feature./li + /ol + p Note that rolling upgrade is supported only from Hadoop-2.4.0 onwards. /p subsection name=Upgrade without Downtime id=UpgradeWithoutDowntime
hadoop git commit: HDFS.7849. Update documentation for enabling a new feature in rolling upgrade ( Contributed by J.Andreina )
Repository: hadoop Updated Branches: refs/heads/branch-2.7 b8f269af9 - 76ce2e8ea HDFS.7849. Update documentation for enabling a new feature in rolling upgrade ( Contributed by J.Andreina ) (cherry picked from commit 5be5129d93481e9caae6a05b2361c463211a6d55) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ce2e8e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ce2e8e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ce2e8e Branch: refs/heads/branch-2.7 Commit: 76ce2e8ea0a0c737992ca1ea3bd9aea33db98343 Parents: b8f269a Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Mar 19 19:05:42 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Thu Mar 19 19:10:23 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml| 9 + 2 files changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ce2e8e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 3dd5f1f..ac83ec8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -439,6 +439,9 @@ Release 2.7.0 - UNRELEASED HDFS-7054. Make DFSOutputStream tracing more fine-grained (cmccabe) +HDFS.7849. Update documentation for enabling a new feature in rolling +upgrade ( J.Andreina via vinayakumarb ) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ce2e8e/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml index e7d8698..a62198f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml @@ -48,6 +48,15 @@ In order to upgrade a HDFS cluster without downtime, the cluster must be setup with HA. /p p +If there is any new feature which is enabled in new software release, may not work with old software release after upgrade. +In such cases upgrade should be done by following steps. + /p + ol +liDisable new feature./li +liUpgrade the cluster./li +liEnable the new feature./li + /ol + p Note that rolling upgrade is supported only from Hadoop-2.4.0 onwards. /p subsection name=Upgrade without Downtime id=UpgradeWithoutDowntime
hadoop git commit: HDFS.7849. Update documentation for enabling a new feature in rolling upgrade ( Contributed by J.Andreina )
Repository: hadoop Updated Branches: refs/heads/trunk e79be0ee1 - 5be5129d9 HDFS.7849. Update documentation for enabling a new feature in rolling upgrade ( Contributed by J.Andreina ) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5be5129d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5be5129d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5be5129d Branch: refs/heads/trunk Commit: 5be5129d93481e9caae6a05b2361c463211a6d55 Parents: e79be0e Author: Vinayakumar B vinayakum...@apache.org Authored: Thu Mar 19 19:05:42 2015 +0530 Committer: Vinayakumar B vinayakum...@apache.org Committed: Thu Mar 19 19:05:42 2015 +0530 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ .../hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml| 9 + 2 files changed, 12 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be5129d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index d9e8b9e..ac58c39 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -764,6 +764,9 @@ Release 2.7.0 - UNRELEASED HDFS-7054. Make DFSOutputStream tracing more fine-grained (cmccabe) +HDFS.7849. Update documentation for enabling a new feature in rolling +upgrade ( J.Andreina via vinayakumarb ) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. http://git-wip-us.apache.org/repos/asf/hadoop/blob/5be5129d/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml index 28649a8..2ad28e1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/xdoc/HdfsRollingUpgrade.xml @@ -48,6 +48,15 @@ In order to upgrade a HDFS cluster without downtime, the cluster must be setup with HA. /p p +If there is any new feature which is enabled in new software release, may not work with old software release after upgrade. +In such cases upgrade should be done by following steps. + /p + ol +liDisable new feature./li +liUpgrade the cluster./li +liEnable the new feature./li + /ol + p Note that rolling upgrade is supported only from Hadoop-2.4.0 onwards. /p subsection name=Upgrade without Downtime id=UpgradeWithoutDowntime
hadoop git commit: HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Contributed by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c58786794 - 6ddb1bc85
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Contributed by Rakesh R)
(cherry picked from commit 018893e81ec1c43e6c79c77adec92c2edfb20cab)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ddb1bc8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ddb1bc8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ddb1bc8
Branch: refs/heads/branch-2
Commit: 6ddb1bc857b2ab85748171d1084882569f760c69
Parents: c587867
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 15:32:34 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 15:34:48 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 32 +---
.../apache/hadoop/hdfs/TestFileCreation.java| 4 +--
.../snapshot/TestRenameWithSnapshots.java | 4 +--
4 files changed, 35 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ddb1bc8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8e1a696..f9d2d32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -14,6 +14,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
+(Rakesh R via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ddb1bc8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 1076938..2113268 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -61,6 +61,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -120,6 +121,7 @@ import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* This class creates a single-process DFS cluster for junit testing.
@@ -525,7 +527,8 @@ public class MiniDFSCluster {
private boolean federation;
private boolean checkExitOnShutdown = true;
protected final int storagesPerDatanode;
-
+ private SetFileSystem fileSystems = Sets.newHashSet();
+
/**
* A unique instance identifier for the cluster. This
* is used to disambiguate HA filesystems in the case where
@@ -1709,6 +1712,13 @@ public class MiniDFSCluster {
* Shutdown all the nodes in the cluster.
*/
public void shutdown(boolean deleteDfsDir) {
+shutdown(deleteDfsDir, true);
+ }
+
+ /**
+ * Shutdown all the nodes in the cluster.
+ */
+ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
LOG.info(Shutting down the Mini HDFS Cluster);
if (checkExitOnShutdown) {
if (ExitUtil.terminateCalled()) {
@@ -1718,6 +1728,16 @@ public class MiniDFSCluster {
throw new AssertionError(Test resulted in an unexpected exit);
}
}
+if (closeFileSystem) {
+ for (FileSystem fs : fileSystems) {
+try {
+ fs.close();
+} catch (IOException ioe) {
+ LOG.warn(Exception while closing file system, ioe);
+}
+ }
+ fileSystems.clear();
+}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
if (nnInfo == null) continue;
@@ -2138,8 +2158,10 @@ public class MiniDFSCluster {
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
-return (DistributedFileSystem)FileSystem.get(getURI(nnIndex),
-nameNodes[nnIndex].conf);
+DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+getURI(nnIndex), nameNodes[nnIndex].conf);
+fileSystems.add(dfs);
+return dfs;
}
/**
@@ -2147,7 +2169,9 @@ public class MiniDFSCluster {
* This simulating different threads working
hadoop git commit: HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown() (Contributed by Rakesh R)
Repository: hadoop
Updated Branches:
refs/heads/trunk e5370477c - 018893e81
HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
(Contributed by Rakesh R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/018893e8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/018893e8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/018893e8
Branch: refs/heads/trunk
Commit: 018893e81ec1c43e6c79c77adec92c2edfb20cab
Parents: e537047
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 15:32:34 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 15:32:34 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 32 +---
.../apache/hadoop/hdfs/TestFileCreation.java| 4 +--
.../snapshot/TestRenameWithSnapshots.java | 4 +--
4 files changed, 35 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ad3e880..bbe1f02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -327,6 +327,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HDFS-5356. MiniDFSCluster should close all open FileSystems when shutdown()
+(Rakesh R via vinayakumarb)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/018893e8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 9208ed2..a6cc71f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -60,6 +60,7 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Random;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -118,6 +119,7 @@ import org.apache.hadoop.util.ToolRunner;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**
* This class creates a single-process DFS cluster for junit testing.
@@ -523,7 +525,8 @@ public class MiniDFSCluster {
private boolean federation;
private boolean checkExitOnShutdown = true;
protected final int storagesPerDatanode;
-
+ private SetFileSystem fileSystems = Sets.newHashSet();
+
/**
* A unique instance identifier for the cluster. This
* is used to disambiguate HA filesystems in the case where
@@ -1705,6 +1708,13 @@ public class MiniDFSCluster {
* Shutdown all the nodes in the cluster.
*/
public void shutdown(boolean deleteDfsDir) {
+shutdown(deleteDfsDir, true);
+ }
+
+ /**
+ * Shutdown all the nodes in the cluster.
+ */
+ public void shutdown(boolean deleteDfsDir, boolean closeFileSystem) {
LOG.info(Shutting down the Mini HDFS Cluster);
if (checkExitOnShutdown) {
if (ExitUtil.terminateCalled()) {
@@ -1714,6 +1724,16 @@ public class MiniDFSCluster {
throw new AssertionError(Test resulted in an unexpected exit);
}
}
+if (closeFileSystem) {
+ for (FileSystem fs : fileSystems) {
+try {
+ fs.close();
+} catch (IOException ioe) {
+ LOG.warn(Exception while closing file system, ioe);
+}
+ }
+ fileSystems.clear();
+}
shutdownDataNodes();
for (NameNodeInfo nnInfo : nameNodes) {
if (nnInfo == null) continue;
@@ -2144,8 +2164,10 @@ public class MiniDFSCluster {
* Get a client handle to the DFS cluster for the namenode at given index.
*/
public DistributedFileSystem getFileSystem(int nnIndex) throws IOException {
-return (DistributedFileSystem)FileSystem.get(getURI(nnIndex),
-nameNodes[nnIndex].conf);
+DistributedFileSystem dfs = (DistributedFileSystem) FileSystem.get(
+getURI(nnIndex), nameNodes[nnIndex].conf);
+fileSystems.add(dfs);
+return dfs;
}
/**
@@ -2153,7 +2175,9 @@ public class MiniDFSCluster {
* This simulating different threads working on different FileSystem
instances.
*/
public FileSystem
hadoop git commit: HADOOP-11721. switch jenkins patch tester to use git clean instead of mvn clean (temp commit)
Repository: hadoop
Updated Branches:
refs/heads/trunk 7179f94f9 - a89b087c4
HADOOP-11721. switch jenkins patch tester to use git clean instead of mvn clean
(temp commit)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a89b087c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a89b087c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a89b087c
Branch: refs/heads/trunk
Commit: a89b087c45e549e1f5b5fc953de4657fcbb97195
Parents: 7179f94
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 17 21:39:14 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 17 21:39:14 2015 +0530
--
dev-support/test-patch.sh | 4
1 file changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a89b087c/dev-support/test-patch.sh
--
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index b0fbb80..574a4fd 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -292,6 +292,10 @@ prebuildWithoutPatch () {
cd -
fi
echo Compiling $(pwd)
+ if [[ -d $(pwd)/hadoop-hdfs-project/hadoop-hdfs/target/test/data/dfs ]];
then
+echo Changing permission
$(pwd)/hadoop-hdfs-project/hadoop-hdfs/target/test/data/dfs to avoid broken
builds
+chmod +x -R $(pwd)/hadoop-hdfs-project/hadoop-hdfs/target/test/data/dfs
+ fi
echo $MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
$PATCH_DIR/trunkJavacWarnings.txt 21
$MVN clean test -DskipTests -D${PROJECT_NAME}PatchProcess -Ptest-patch
$PATCH_DIR/trunkJavacWarnings.txt 21
if [[ $? != 0 ]] ; then
hadoop git commit: HADOOP-11568. Description on usage of classpath in hadoop command is incomplete. ( Contributed by Archana T )
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5efee5efd - 2f902a823
HADOOP-11568. Description on usage of classpath in hadoop command is
incomplete. ( Contributed by Archana T )
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f902a82
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f902a82
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f902a82
Branch: refs/heads/branch-2
Commit: 2f902a823cc72b120e9a19983958f9f0873feeb4
Parents: 5efee5e
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Mar 10 13:02:56 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Mar 10 13:02:56 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
hadoop-common-project/hadoop-common/src/main/bin/hadoop | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f902a82/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d5b3418..cfbe92f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -15,6 +15,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+HADOOP-11568. Description on usage of classpath in hadoop command is
+incomplete. ( Archana T via vinayakumarb )
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f902a82/hadoop-common-project/hadoop-common/src/main/bin/hadoop
--
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index b71aa8c..980cd0c 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -45,8 +45,8 @@ function print_usage(){
echo distcp srcurl desturl copy file or directories recursively
echo archive -archiveName NAME -p parent path src* dest create a
hadoop archive
echo classpathprints the class path needed to get the
- echo credential interact with credential providers
echoHadoop jar and the required libraries
+ echo credential interact with credential providers
echo daemonlogget/set the log level for each daemon
echo traceview and modify Hadoop tracing settings
echo
hadoop git commit: HADOOP-11646. Erasure Coder API for encoding and decoding of block group ( Contributed by Kai Zheng )
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7285 80401779f - 720901acf
HADOOP-11646. Erasure Coder API for encoding and decoding of block group (
Contributed by Kai Zheng )
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/720901ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/720901ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/720901ac
Branch: refs/heads/HDFS-7285
Commit: 720901acfc9085e35dbd2ab0f4eaa8c1cb249e81
Parents: 8040177
Author: Vinayakumar B vinayakum...@apache.org
Authored: Mon Mar 9 12:32:26 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Mon Mar 9 12:32:26 2015 +0530
--
.../hadoop-common/CHANGES-HDFS-EC-7285.txt | 2 +
.../apache/hadoop/io/erasurecode/ECBlock.java | 80 ++
.../hadoop/io/erasurecode/ECBlockGroup.java | 82 ++
.../erasurecode/coder/AbstractErasureCoder.java | 63 +
.../coder/AbstractErasureCodingStep.java| 59
.../coder/AbstractErasureDecoder.java | 152 +++
.../coder/AbstractErasureEncoder.java | 50
.../io/erasurecode/coder/ErasureCoder.java | 77 ++
.../io/erasurecode/coder/ErasureCodingStep.java | 55
.../io/erasurecode/coder/ErasureDecoder.java| 41 +++
.../erasurecode/coder/ErasureDecodingStep.java | 52
.../io/erasurecode/coder/ErasureEncoder.java| 39 +++
.../erasurecode/coder/ErasureEncodingStep.java | 49
.../io/erasurecode/coder/XorErasureDecoder.java | 78 ++
.../io/erasurecode/coder/XorErasureEncoder.java | 45
.../erasurecode/rawcoder/RawErasureCoder.java | 2 +-
.../erasurecode/coder/TestErasureCoderBase.java | 266 +++
.../io/erasurecode/coder/TestXorCoder.java | 50
18 files changed, 1241 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/720901ac/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
index ee42c84..c17a1bd 100644
--- a/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES-HDFS-EC-7285.txt
@@ -15,4 +15,6 @@
HADOOP-11643. Define EC schema API for ErasureCodec. Contributed by Kai
Zheng
( Kai Zheng )
+HADOOP-11646. Erasure Coder API for encoding and decoding of block group
+( Kai Zheng via vinayakumarb )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/720901ac/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
new file mode 100644
index 000..956954a
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECBlock.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.io.erasurecode;
+
+/**
+ * A wrapper of block level data source/output that {@link ECChunk}s can be
+ * extracted from. For HDFS, it can be an HDFS block (250MB). Note it only
cares
+ * about erasure coding specific logic thus avoids coupling with any HDFS block
+ * details. We can have something like HdfsBlock extend it.
+ */
+public class ECBlock {
+
+ private boolean isParity;
+ private boolean isErased;
+
+ /**
+ * A default constructor. isParity and isErased are false by default.
+ */
+ public ECBlock() {
+this(false, false);
+ }
+
+ /**
+ * A constructor specifying isParity and isErased.
+ * @param isParity
+ * @param isErased
+ */
+ public ECBlock(boolean isParity, boolean isErased) {
+this.isParity = isParity;
+this.isErased
hadoop git commit: HDFS-7869. Inconsistency in the return information while performing rolling upgrade ( Contributed by J.Andreina )
Repository: hadoop
Updated Branches:
refs/heads/trunk 29bb68986 - 3560180b6
HDFS-7869. Inconsistency in the return information while performing rolling
upgrade ( Contributed by J.Andreina )
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3560180b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3560180b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3560180b
Branch: refs/heads/trunk
Commit: 3560180b6e9926aa3ee1357da59b28a4b4689a0d
Parents: 29bb689
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Mar 4 14:38:38 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Mar 4 14:38:38 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 3 +++
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 6 +++---
.../apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 3 +--
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java | 4 ++--
5 files changed, 10 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3560180b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7ff3c78..2037973 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1083,6 +1083,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file
includes non-snapshotted content. (Charles Lamb via atm)
+HDFS-7869. Inconsistency in the return information while performing rolling
+upgrade ( J.Andreina via vinayakumarb )
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3560180b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index d2b48f3..77b4a27 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7475,7 +7475,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
}
}
- void finalizeRollingUpgrade() throws IOException {
+ RollingUpgradeInfo finalizeRollingUpgrade() throws IOException {
checkSuperuserPrivilege();
checkOperation(OperationCategory.WRITE);
writeLock();
@@ -7483,7 +7483,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
try {
checkOperation(OperationCategory.WRITE);
if (!isRollingUpgrade()) {
-return;
+return null;
}
checkNameNodeSafeMode(Failed to finalize rolling upgrade);
@@ -7508,7 +7508,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
if (auditLog.isInfoEnabled() isExternalInvocation()) {
logAuditEvent(true, finalizeRollingUpgrade, null, null, null);
}
-return;
+return returnInfo;
}
RollingUpgradeInfo finalizeRollingUpgradeInternal(long finalizeTime)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3560180b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 9ccdb40..f20fb35 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -1145,8 +1145,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
case PREPARE:
return namesystem.startRollingUpgrade();
case FINALIZE:
- namesystem.finalizeRollingUpgrade();
- return null;
+ return namesystem.finalizeRollingUpgrade();
default:
throw new UnsupportedActionException(action + is not yet supported.);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3560180b
hadoop git commit: HDFS-7869. Inconsistency in the return information while performing rolling upgrade ( Contributed by J.Andreina )
Repository: hadoop
Updated Branches:
refs/heads/branch-2 31b3f8460 - c24448c27
HDFS-7869. Inconsistency in the return information while performing rolling
upgrade ( Contributed by J.Andreina )
(cherry picked from commit 3560180b6e9926aa3ee1357da59b28a4b4689a0d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c24448c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c24448c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c24448c2
Branch: refs/heads/branch-2
Commit: c24448c27b91d0398fb42a0a9ab8b8cdd4887647
Parents: 31b3f84
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Mar 4 14:38:38 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Mar 4 14:39:51 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 3 +++
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 6 +++---
.../apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 3 +--
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 2 +-
.../test/java/org/apache/hadoop/hdfs/TestRollingUpgrade.java | 4 ++--
5 files changed, 10 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24448c2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bff45bb..6506451 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -777,6 +777,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7682. {{DistributedFileSystem#getFileChecksum}} of a snapshotted file
includes non-snapshotted content. (Charles Lamb via atm)
+HDFS-7869. Inconsistency in the return information while performing rolling
+upgrade ( J.Andreina via vinayakumarb )
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24448c2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 17f313d..81d2b88 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7475,7 +7475,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
}
}
- void finalizeRollingUpgrade() throws IOException {
+ RollingUpgradeInfo finalizeRollingUpgrade() throws IOException {
checkSuperuserPrivilege();
checkOperation(OperationCategory.WRITE);
writeLock();
@@ -7483,7 +7483,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
try {
checkOperation(OperationCategory.WRITE);
if (!isRollingUpgrade()) {
-return;
+return null;
}
checkNameNodeSafeMode(Failed to finalize rolling upgrade);
@@ -7508,7 +7508,7 @@ public class FSNamesystem implements Namesystem,
FSNamesystemMBean,
if (auditLog.isInfoEnabled() isExternalInvocation()) {
logAuditEvent(true, finalizeRollingUpgrade, null, null, null);
}
-return;
+return returnInfo;
}
RollingUpgradeInfo finalizeRollingUpgradeInternal(long finalizeTime)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c24448c2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index a746e8d..b1ae348 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -1139,8 +1139,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
case PREPARE:
return namesystem.startRollingUpgrade();
case FINALIZE:
- namesystem.finalizeRollingUpgrade();
- return null;
+ return namesystem.finalizeRollingUpgrade();
default:
throw new UnsupportedActionException(action + is not yet supported
hadoop git commit: HDFS-6753. Initialize checkDisk when DirectoryScanner not able to get files list for scanning (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/trunk 2954e6546 - 4f75b1562
HDFS-6753. Initialize checkDisk when DirectoryScanner not able to get files
list for scanning (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f75b156
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f75b156
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f75b156
Branch: refs/heads/trunk
Commit: 4f75b15628a76881efc39054612dc128e23d27be
Parents: 2954e65
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Feb 27 16:36:28 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Feb 27 16:36:28 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/datanode/DataNode.java| 2 +-
.../hadoop/hdfs/server/datanode/DirectoryScanner.java | 12 +---
.../hdfs/server/datanode/TestDirectoryScanner.java | 9 ++---
4 files changed, 19 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f75b156/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ba553dc..8556afd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1040,6 +1040,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7774. Unresolved symbols error while compiling HDFS on Windows 7/32
bit.
(Kiran Kumar M R via cnauroth)
+HDFS-6753. Initialize checkDisk when DirectoryScanner not able to get
+files list for scanning (J.Andreina via vinayakumarb)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f75b156/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index f233e02..92ddb7b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -815,7 +815,7 @@ public class DataNode extends ReconfigurableBase
reason = verifcation is not supported by SimulatedFSDataset;
}
if (reason == null) {
- directoryScanner = new DirectoryScanner(data, conf);
+ directoryScanner = new DirectoryScanner(this, data, conf);
directoryScanner.start();
} else {
LOG.info(Periodic Directory Tree Verification scan is disabled because
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f75b156/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 09c2914..c7ee21e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -63,6 +63,7 @@ public class DirectoryScanner implements Runnable {
private final long scanPeriodMsecs;
private volatile boolean shouldRun = false;
private boolean retainDiffs = false;
+ private final DataNode datanode;
final ScanInfoPerBlockPool diffs = new ScanInfoPerBlockPool();
final MapString, Stats stats = new HashMapString, Stats();
@@ -308,7 +309,8 @@ public class DirectoryScanner implements Runnable {
}
}
- DirectoryScanner(FsDatasetSpi? dataset, Configuration conf) {
+ DirectoryScanner(DataNode datanode, FsDatasetSpi? dataset, Configuration
conf) {
+this.datanode = datanode;
this.dataset = dataset;
int interval =
conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT);
@@ -547,7 +549,7 @@ public class DirectoryScanner implements Runnable {
for (int i = 0; i volumes.size(); i++) {
if (isValid(dataset, volumes.get(i))) {
ReportCompiler reportCompiler =
- new ReportCompiler(volumes.get(i));
+ new ReportCompiler
hadoop git commit: HDFS-6753. Initialize checkDisk when DirectoryScanner not able to get files list for scanning (Contributed by J.Andreina)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d223a4a59 - bc60404ea
HDFS-6753. Initialize checkDisk when DirectoryScanner not able to get files
list for scanning (Contributed by J.Andreina)
(cherry picked from commit 4f75b15628a76881efc39054612dc128e23d27be)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bc60404e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bc60404e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bc60404e
Branch: refs/heads/branch-2
Commit: bc60404eaf6e5298c20a552685ec0b6c59b6fd0b
Parents: d223a4a
Author: Vinayakumar B vinayakum...@apache.org
Authored: Fri Feb 27 16:36:28 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Fri Feb 27 16:37:03 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../apache/hadoop/hdfs/server/datanode/DataNode.java| 2 +-
.../hadoop/hdfs/server/datanode/DirectoryScanner.java | 12 +---
.../hdfs/server/datanode/TestDirectoryScanner.java | 9 ++---
4 files changed, 19 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc60404e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8dd26d4..998715e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -741,6 +741,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7774. Unresolved symbols error while compiling HDFS on Windows 7/32
bit.
(Kiran Kumar M R via cnauroth)
+HDFS-6753. Initialize checkDisk when DirectoryScanner not able to get
+files list for scanning (J.Andreina via vinayakumarb)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc60404e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index d25e58b..5c516d3 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -822,7 +822,7 @@ public class DataNode extends ReconfigurableBase
reason = verifcation is not supported by SimulatedFSDataset;
}
if (reason == null) {
- directoryScanner = new DirectoryScanner(data, conf);
+ directoryScanner = new DirectoryScanner(this, data, conf);
directoryScanner.start();
} else {
LOG.info(Periodic Directory Tree Verification scan is disabled because
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bc60404e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 09c2914..c7ee21e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -63,6 +63,7 @@ public class DirectoryScanner implements Runnable {
private final long scanPeriodMsecs;
private volatile boolean shouldRun = false;
private boolean retainDiffs = false;
+ private final DataNode datanode;
final ScanInfoPerBlockPool diffs = new ScanInfoPerBlockPool();
final MapString, Stats stats = new HashMapString, Stats();
@@ -308,7 +309,8 @@ public class DirectoryScanner implements Runnable {
}
}
- DirectoryScanner(FsDatasetSpi? dataset, Configuration conf) {
+ DirectoryScanner(DataNode datanode, FsDatasetSpi? dataset, Configuration
conf) {
+this.datanode = datanode;
this.dataset = dataset;
int interval =
conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT);
@@ -547,7 +549,7 @@ public class DirectoryScanner implements Runnable {
for (int i = 0; i volumes.size(); i++) {
if (isValid(dataset, volumes.get(i))) {
ReportCompiler reportCompiler
hadoop git commit: HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem' (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0512e50d6 - f0f3d1937
HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem' (Contributed
by Vinayakumar B)
(cherry picked from commit 166eecf687765f369cd5bf210cc6d7bf493121ea)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0f3d193
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0f3d193
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0f3d193
Branch: refs/heads/branch-2
Commit: f0f3d1937db2f43479b3daa6890dce1f86c1aa23
Parents: 0512e50
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Feb 26 14:36:09 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Feb 26 14:37:03 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.html | 2 ++
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 9 -
3 files changed, 13 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0f3d193/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cda4b7d..f59bb71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -378,6 +378,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7537. Add UNDER MIN REPL'D BLOCKS count to fsck. (GAO Rui via
szetszwo)
+HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem'
+(vinayakumarb)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0f3d193/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index e1fdfa3..7b34044 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -112,6 +112,7 @@
thOwner/th
thGroup/th
thSize/th
+thLast Modified/th
thReplication/th
thBlock Size/th
thName/th
@@ -124,6 +125,7 @@
td{owner}/td
td{group}/td
td{length|fmt_bytes}/td
+td{#helper_date_tostring value={modificationTime}/}/td
td{replication}/td
td{blockSize|fmt_bytes}/td
tda style=cursor:pointer inode-type={type}
class=explorer-browse-links inode-path={pathSuffix}{pathSuffix}/a/td
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0f3d193/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index 87d47fa..0a53dcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -143,6 +143,12 @@
}
function browse_directory(dir) {
+var HELPERS = {
+ 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
+var value = dust.helpers.tap(params.value, chunk, ctx);
+return chunk.write('' + new Date(Number(value)).toLocaleString());
+ }
+};
var url = '/webhdfs/v1' + dir + '?op=LISTSTATUS';
$.get(url, function(data) {
var d = get_response(data, FileStatuses);
@@ -154,7 +160,8 @@
current_directory = dir;
$('#directory').val(dir);
window.location.hash = dir;
- dust.render('explorer', d, function(err, out) {
+ var base = dust.makeBase(HELPERS);
+ dust.render('explorer', base.push(d), function(err, out) {
$('#panel').html(out);
$('.explorer-browse-links').click(function() {
hadoop git commit: HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem' (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 71385f9b7 - 166eecf68
HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem' (Contributed
by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/166eecf6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/166eecf6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/166eecf6
Branch: refs/heads/trunk
Commit: 166eecf687765f369cd5bf210cc6d7bf493121ea
Parents: 71385f9
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Feb 26 14:36:09 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Thu Feb 26 14:36:09 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.html | 2 ++
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 9 -
3 files changed, 13 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166eecf6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4523bf4..e09714f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -676,6 +676,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7537. Add UNDER MIN REPL'D BLOCKS count to fsck. (GAO Rui via
szetszwo)
+HDFS-7832. Show 'Last Modified' in Namenode's 'Browse Filesystem'
+(vinayakumarb)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166eecf6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
index e1fdfa3..7b34044 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.html
@@ -112,6 +112,7 @@
thOwner/th
thGroup/th
thSize/th
+thLast Modified/th
thReplication/th
thBlock Size/th
thName/th
@@ -124,6 +125,7 @@
td{owner}/td
td{group}/td
td{length|fmt_bytes}/td
+td{#helper_date_tostring value={modificationTime}/}/td
td{replication}/td
td{blockSize|fmt_bytes}/td
tda style=cursor:pointer inode-type={type}
class=explorer-browse-links inode-path={pathSuffix}{pathSuffix}/a/td
http://git-wip-us.apache.org/repos/asf/hadoop/blob/166eecf6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index 87d47fa..0a53dcd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -143,6 +143,12 @@
}
function browse_directory(dir) {
+var HELPERS = {
+ 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
+var value = dust.helpers.tap(params.value, chunk, ctx);
+return chunk.write('' + new Date(Number(value)).toLocaleString());
+ }
+};
var url = '/webhdfs/v1' + dir + '?op=LISTSTATUS';
$.get(url, function(data) {
var d = get_response(data, FileStatuses);
@@ -154,7 +160,8 @@
current_directory = dir;
$('#directory').val(dir);
window.location.hash = dir;
- dust.render('explorer', d, function(err, out) {
+ var base = dust.makeBase(HELPERS);
+ dust.render('explorer', base.push(d), function(err, out) {
$('#panel').html(out);
$('.explorer-browse-links').click(function() {
hadoop git commit: HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned to SERVICE_NOT_RESPONDING (Contributed by Ming Ma)
Repository: hadoop
Updated Branches:
refs/heads/trunk 500e6a0f4 - cf4b7f506
HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned to
SERVICE_NOT_RESPONDING (Contributed by Ming Ma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf4b7f50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf4b7f50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf4b7f50
Branch: refs/heads/trunk
Commit: cf4b7f506dd338ecf2ed4c643b6a6a334e070fca
Parents: 500e6a0
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Feb 17 14:55:56 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Feb 17 14:55:56 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/ha/HealthMonitor.java | 35 ++
.../org/apache/hadoop/ha/DummyHAService.java| 73 +---
.../org/apache/hadoop/ha/TestHealthMonitor.java | 4 +-
4 files changed, 94 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 522ec47..51305bb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -934,6 +934,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11467. KerberosAuthenticator can connect to a non-secure cluster.
(yzhangal via rkanter)
+HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned
+to SERVICE_NOT_RESPONDING (Ming Ma via vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
index 0d1..8c87629 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
@@ -30,6 +30,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.util.Daemon;
@@ -201,18 +202,20 @@ public class HealthMonitor {
status = proxy.getServiceStatus();
proxy.monitorHealth();
healthy = true;
- } catch (HealthCheckFailedException e) {
-LOG.warn(Service health check failed for + targetToMonitor
-+ : + e.getMessage());
-enterState(State.SERVICE_UNHEALTHY);
} catch (Throwable t) {
-LOG.warn(Transport-level exception trying to monitor health of +
-targetToMonitor + : + t.getLocalizedMessage());
-RPC.stopProxy(proxy);
-proxy = null;
-enterState(State.SERVICE_NOT_RESPONDING);
-Thread.sleep(sleepAfterDisconnectMillis);
-return;
+if (isHealthCheckFailedException(t)) {
+ LOG.warn(Service health check failed for + targetToMonitor
+ + : + t.getMessage());
+ enterState(State.SERVICE_UNHEALTHY);
+} else {
+ LOG.warn(Transport-level exception trying to monitor health of +
+ targetToMonitor + : + t.getCause() + +
t.getLocalizedMessage());
+ RPC.stopProxy(proxy);
+ proxy = null;
+ enterState(State.SERVICE_NOT_RESPONDING);
+ Thread.sleep(sleepAfterDisconnectMillis);
+ return;
+}
}
if (status != null) {
@@ -225,7 +228,15 @@ public class HealthMonitor {
Thread.sleep(checkIntervalMillis);
}
}
-
+
+ private boolean isHealthCheckFailedException(Throwable t) {
+return ((t instanceof HealthCheckFailedException) ||
+(t instanceof RemoteException
+((RemoteException)t).unwrapRemoteException(
+HealthCheckFailedException.class) instanceof
+HealthCheckFailedException));
+ }
+
private synchronized void setLastServiceStatus(HAServiceStatus status) {
this.lastServiceState = status;
for (ServiceStateCallback cb : serviceStateCallbacks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop
[1/2] hadoop git commit: HDFS-7703. Support favouredNodes for the append for new blocks ( Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 54e33baaf - 005e1df54
HDFS-7703. Support favouredNodes for the append for new blocks ( Contributed by
Vinayakumar B)
(cherry picked from commit 89a544928083501625bc69f96b530040228f0a5f)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/91a5d929
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/91a5d929
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/91a5d929
Branch: refs/heads/branch-2
Commit: 91a5d92916c6c1b0475d5794c3855b53b020d4ec
Parents: 54e33ba
Author: Vinayakumar B vinayakum...@apache.org
Authored: Thu Feb 12 12:38:44 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Feb 17 15:19:03 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 53 +++-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 7 ++-
.../hadoop/hdfs/DistributedFileSystem.java | 43
.../namenode/TestFavoredNodesEndToEnd.java | 29 +++
5 files changed, 121 insertions(+), 14 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 6bfa34c..a1b2053 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -324,6 +324,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7761. cleanup unnecssary code logic in LocatedBlock. (yliu)
+HDFS-7703. Support favouredNodes for the append for new blocks
+(vinayakumarb)
+
HDFS-7694. FSDataInputStream should support unbuffer (cmccabe)
HDFS-7684. The host:port settings of the daemons should be trimmed before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/91a5d929/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index d27197f..3c0ec99 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -1693,6 +1693,15 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
if(LOG.isDebugEnabled()) {
LOG.debug(src + : masked= + masked);
}
+final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
+src, masked, flag, createParent, replication, blockSize, progress,
+buffersize, dfsClientConf.createChecksum(checksumOpt),
+getFavoredNodesStr(favoredNodes));
+beginFileLease(result.getFileId(), result);
+return result;
+ }
+
+ private String[] getFavoredNodesStr(InetSocketAddress[] favoredNodes) {
String[] favoredNodeStrs = null;
if (favoredNodes != null) {
favoredNodeStrs = new String[favoredNodes.length];
@@ -1702,12 +1711,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
+ favoredNodes[i].getPort();
}
}
-final DFSOutputStream result = DFSOutputStream.newStreamForCreate(this,
-src, masked, flag, createParent, replication, blockSize, progress,
-buffersize, dfsClientConf.createChecksum(checksumOpt),
-favoredNodeStrs);
-beginFileLease(result.getFileId(), result);
-return result;
+return favoredNodeStrs;
}
/**
@@ -1725,7 +1729,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
}
return null;
}
- return callAppend(src, buffersize, flag, progress);
+ return callAppend(src, buffersize, flag, progress, null);
}
return null;
}
@@ -1804,7 +1808,8 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
/** Method to get stream returned by append call */
private DFSOutputStream callAppend(String src, int buffersize,
- EnumSetCreateFlag flag, Progressable progress) throws IOException {
+ EnumSetCreateFlag flag, Progressable progress, String[] favoredNodes)
+ throws IOException {
CreateFlag.validateForAppend(flag);
try {
LastBlockWithStatus blkWithStatus = namenode.append(src, clientName,
@@ -1812,7 +1817,7 @@ public class DFSClient implements java.io.Closeable,
RemotePeerFactory,
return DFSOutputStream.newStreamForAppend(this, src
[2/2] hadoop git commit: HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned to SERVICE_NOT_RESPONDING (Contributed by Ming Ma)
HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned to
SERVICE_NOT_RESPONDING (Contributed by Ming Ma)
(cherry picked from commit cf4b7f506dd338ecf2ed4c643b6a6a334e070fca)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/005e1df5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/005e1df5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/005e1df5
Branch: refs/heads/branch-2
Commit: 005e1df540aeccbeb371f3bab834b140d85f0ec5
Parents: 91a5d92
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Feb 17 14:55:56 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Feb 17 15:19:44 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/ha/HealthMonitor.java | 35 ++
.../org/apache/hadoop/ha/DummyHAService.java| 73 +---
.../org/apache/hadoop/ha/TestHealthMonitor.java | 4 +-
4 files changed, 94 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ff71162..f3ab6f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -537,6 +537,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11467. KerberosAuthenticator can connect to a non-secure cluster.
(yzhangal via rkanter)
+HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned
+to SERVICE_NOT_RESPONDING (Ming Ma via vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
index 0d1..8c87629 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
@@ -30,6 +30,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.util.Daemon;
@@ -201,18 +202,20 @@ public class HealthMonitor {
status = proxy.getServiceStatus();
proxy.monitorHealth();
healthy = true;
- } catch (HealthCheckFailedException e) {
-LOG.warn(Service health check failed for + targetToMonitor
-+ : + e.getMessage());
-enterState(State.SERVICE_UNHEALTHY);
} catch (Throwable t) {
-LOG.warn(Transport-level exception trying to monitor health of +
-targetToMonitor + : + t.getLocalizedMessage());
-RPC.stopProxy(proxy);
-proxy = null;
-enterState(State.SERVICE_NOT_RESPONDING);
-Thread.sleep(sleepAfterDisconnectMillis);
-return;
+if (isHealthCheckFailedException(t)) {
+ LOG.warn(Service health check failed for + targetToMonitor
+ + : + t.getMessage());
+ enterState(State.SERVICE_UNHEALTHY);
+} else {
+ LOG.warn(Transport-level exception trying to monitor health of +
+ targetToMonitor + : + t.getCause() + +
t.getLocalizedMessage());
+ RPC.stopProxy(proxy);
+ proxy = null;
+ enterState(State.SERVICE_NOT_RESPONDING);
+ Thread.sleep(sleepAfterDisconnectMillis);
+ return;
+}
}
if (status != null) {
@@ -225,7 +228,15 @@ public class HealthMonitor {
Thread.sleep(checkIntervalMillis);
}
}
-
+
+ private boolean isHealthCheckFailedException(Throwable t) {
+return ((t instanceof HealthCheckFailedException) ||
+(t instanceof RemoteException
+((RemoteException)t).unwrapRemoteException(
+HealthCheckFailedException.class) instanceof
+HealthCheckFailedException));
+ }
+
private synchronized void setLastServiceStatus(HAServiceStatus status) {
this.lastServiceState = status;
for (ServiceStateCallback cb : serviceStateCallbacks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/005e1df5/hadoop-common-project/hadoop-common
hadoop git commit: HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause DataNode to register successfully with only one NameNode.(Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 23bf6c720 - 3d15728ff
HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause DataNode
to register successfully with only one NameNode.(Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d15728f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d15728f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d15728f
Branch: refs/heads/trunk
Commit: 3d15728ff5301296801e541d9b23bd1687c4adad
Parents: 23bf6c7
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Feb 10 10:43:08 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Feb 10 10:43:08 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 5 +
2 files changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d15728f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4a6bc11..1ca2263 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -896,6 +896,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7718. Store KeyProvider in ClientContext to avoid leaking key provider
threads when using FileContext (Arun Suresh via Colin P. McCabe)
+HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause
+DataNode to register successfully with only one NameNode.(vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d15728f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index e396727..917b5dd 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.util.Time.now;
+import java.io.EOFException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
@@ -797,6 +798,10 @@ class BPServiceActor implements Runnable {
// Use returned registration from namenode with updated fields
bpRegistration = bpNamenode.registerDatanode(bpRegistration);
break;
+ } catch(EOFException e) { // namenode might have just restarted
+LOG.info(Problem connecting to server: + nnAddr + :
++ e.getLocalizedMessage());
+sleepAndLogInterrupts(1000, connecting to server);
} catch(SocketTimeoutException e) { // namenode is busy
LOG.info(Problem connecting to server: + nnAddr);
sleepAndLogInterrupts(1000, connecting to server);
hadoop git commit: HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause DataNode to register successfully with only one NameNode.(Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 24d4d933f - a1bf7aecf
HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause DataNode
to register successfully with only one NameNode.(Contributed by Vinayakumar B)
(cherry picked from commit 3d15728ff5301296801e541d9b23bd1687c4adad)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1bf7aec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1bf7aec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1bf7aec
Branch: refs/heads/branch-2
Commit: a1bf7aecf7d018c5305fa3bd7a9e3ef9af3155c1
Parents: 24d4d93
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Feb 10 10:43:08 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Feb 10 10:45:14 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 5 +
2 files changed, 8 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1bf7aec/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 06e40bd..6d2715c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -596,6 +596,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7718. Store KeyProvider in ClientContext to avoid leaking key provider
threads when using FileContext (Arun Suresh via Colin P. McCabe)
+HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause
+DataNode to register successfully with only one NameNode.(vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1bf7aec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index e6409ab..c344027 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.util.Time.now;
+import java.io.EOFException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
@@ -803,6 +804,10 @@ class BPServiceActor implements Runnable {
// Use returned registration from namenode with updated fields
bpRegistration = bpNamenode.registerDatanode(bpRegistration);
break;
+ } catch(EOFException e) { // namenode might have just restarted
+LOG.info(Problem connecting to server: + nnAddr + :
++ e.getLocalizedMessage());
+sleepAndLogInterrupts(1000, connecting to server);
} catch(SocketTimeoutException e) { // namenode is busy
LOG.info(Problem connecting to server: + nnAddr);
sleepAndLogInterrupts(1000, connecting to server);
hadoop git commit: HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and ShellBasedIdMapping (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7adffad2b - 383e1c5f7
HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and
ShellBasedIdMapping (Contributed by Vinayakumar B)
(cherry picked from commit 9803ae374f69942aec82ec6eeeb9722523a1ade0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/383e1c5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/383e1c5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/383e1c5f
Branch: refs/heads/branch-2
Commit: 383e1c5f737f84fad658e5c7aed48be81c207f38
Parents: 7adffad
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:15:23 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:16:44 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 5 -
.../main/java/org/apache/hadoop/ha/ActiveStandbyElector.java| 4 +++-
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 2 +-
.../java/org/apache/hadoop/security/ShellBasedIdMapping.java| 2 +-
4 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7a57ef0..f0a1aae 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -318,7 +318,10 @@ Release 2.7.0 - UNRELEASED
HADOOP-11446. S3AOutputStream should use shared thread pool to
avoid OutOfMemoryError. (Ted Yu via stevel)
-
+
+HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils
+and ShellBasedIdMapping (vinayakumarb)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 123f119..947baa9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -1064,7 +1064,9 @@ public class ActiveStandbyElector implements
StatCallback, StringCallback {
public void process(WatchedEvent event) {
hasReceivedEvent.countDown();
try {
-hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
+if (!hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS)) {
+ LOG.debug(Event received with stale zk);
+}
ActiveStandbyElector.this.processWatchEvent(
zk, event);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index ffee402..ef1092b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -288,7 +288,7 @@ public class NetUtils {
try {
fqHost = SecurityUtil.getByName(host).getHostName();
// slight race condition, but won't hurt
-canonicalizedHostCache.put(host, fqHost);
+canonicalizedHostCache.putIfAbsent(host, fqHost);
} catch (UnknownHostException e) {
fqHost = host;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index 428e835..28ab248 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -290,7 +290,7 @@ public
hadoop git commit: HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 9803ae374 - 4cd66f7fb
HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cd66f7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cd66f7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cd66f7f
Branch: refs/heads/trunk
Commit: 4cd66f7fb280e53e2d398a62e922a8d68d150679
Parents: 9803ae3
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:31:35 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:31:35 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java | 5 +++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cd66f7f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 48e3695..e9d6ad2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -642,6 +642,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. (Arpit Agarwal via cnauroth)
+HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cd66f7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 1f52ff7..c1e9d7f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -574,11 +574,12 @@ public class TransferFsImage {
LOG.info(Deleting temporary files: + files);
for (File file : files) {
- file.delete(); // ignore the return value
+ if (!file.delete()) {
+LOG.warn(Deleting + file + has failed);
+ }
}
}
-
private static MD5Hash parseMD5Header(HttpURLConnection connection) {
String header = connection.getHeaderField(MD5_HEADER);
return (header != null) ? new MD5Hash(header) : null;
hadoop git commit: HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 383e1c5f7 - 53ecb6358
HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
(cherry picked from commit 4cd66f7fb280e53e2d398a62e922a8d68d150679)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53ecb635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53ecb635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53ecb635
Branch: refs/heads/branch-2
Commit: 53ecb63587f64ea94f9d1bae9b339d09059e727a
Parents: 383e1c5
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:31:35 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:32:56 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java | 5 +++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53ecb635/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c0296fb..6561fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -385,6 +385,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. (Arpit Agarwal via cnauroth)
+HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53ecb635/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 1f52ff7..c1e9d7f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -574,11 +574,12 @@ public class TransferFsImage {
LOG.info(Deleting temporary files: + files);
for (File file : files) {
- file.delete(); // ignore the return value
+ if (!file.delete()) {
+LOG.warn(Deleting + file + has failed);
+ }
}
}
-
private static MD5Hash parseMD5Header(HttpURLConnection connection) {
String header = connection.getHeaderField(MD5_HEADER);
return (header != null) ? new MD5Hash(header) : null;
hadoop git commit: HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and ShellBasedIdMapping (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0c4b11267 - 9803ae374
HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and
ShellBasedIdMapping (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9803ae37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9803ae37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9803ae37
Branch: refs/heads/trunk
Commit: 9803ae374f69942aec82ec6eeeb9722523a1ade0
Parents: 0c4b112
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:15:23 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:15:23 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 5 -
.../main/java/org/apache/hadoop/ha/ActiveStandbyElector.java| 4 +++-
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 2 +-
.../java/org/apache/hadoop/security/ShellBasedIdMapping.java| 2 +-
4 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 40e8d29..4ceb77d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -670,7 +670,10 @@ Release 2.7.0 - UNRELEASED
HADOOP-11446. S3AOutputStream should use shared thread pool to
avoid OutOfMemoryError. (Ted Yu via stevel)
-
+
+HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils
+and ShellBasedIdMapping (vinayakumarb)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 123f119..947baa9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -1064,7 +1064,9 @@ public class ActiveStandbyElector implements
StatCallback, StringCallback {
public void process(WatchedEvent event) {
hasReceivedEvent.countDown();
try {
-hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
+if (!hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS)) {
+ LOG.debug(Event received with stale zk);
+}
ActiveStandbyElector.this.processWatchEvent(
zk, event);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index ffee402..ef1092b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -288,7 +288,7 @@ public class NetUtils {
try {
fqHost = SecurityUtil.getByName(host).getHostName();
// slight race condition, but won't hurt
-canonicalizedHostCache.put(host, fqHost);
+canonicalizedHostCache.putIfAbsent(host, fqHost);
} catch (UnknownHostException e) {
fqHost = host;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index 428e835..28ab248 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -290,7 +290,7 @@ public class ShellBasedIdMapping implements
IdMappingServiceProvider {
return
hadoop git commit: HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce memory footprint of NameNode (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0d89859b5 - 50ae1a666
HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
memory footprint of NameNode (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50ae1a66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50ae1a66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50ae1a66
Branch: refs/heads/trunk
Commit: 50ae1a6664a92619aa683d2a864d0da9fb4af026
Parents: 0d89859
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Dec 23 12:35:53 2014 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Dec 23 12:35:53 2014 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/namenode/AclFeature.java | 37 +++-
.../hadoop/hdfs/server/namenode/AclStorage.java | 30 ++-
.../hdfs/server/namenode/INodeAttributes.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java| 9 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 +
.../namenode/INodeWithAdditionalFields.java | 5 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 5 +
.../snapshot/FileWithSnapshotFeature.java | 6 +
.../hadoop/hdfs/util/ReferenceCountMap.java | 118 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 207 ++-
.../server/namenode/TestFileContextAcl.java | 6 +-
.../hdfs/server/namenode/TestNameNodeAcl.java | 6 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 189 -
.../apache/hadoop/hdfs/web/TestWebHDFSAcl.java | 6 +-
15 files changed, 614 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50ae1a66/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 07a78c2..66b3a8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -630,6 +630,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
restart/failover. (Vinayakumar B via cnauroth)
+HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
+memory footprint of NameNode (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50ae1a66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index e097b05..97d4759 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.Arrays;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -28,9 +31,10 @@ import com.google.common.collect.ImmutableList;
* Feature that represents the ACLs of the inode.
*/
@InterfaceAudience.Private
-public class AclFeature implements INode.Feature {
+public class AclFeature implements INode.Feature, ReferenceCounter {
public static final ImmutableListAclEntry EMPTY_ENTRY_LIST =
ImmutableList.of();
+ private int refCount = 0;
private final int [] entries;
@@ -56,4 +60,35 @@ public class AclFeature implements INode.Feature {
Invalid position for AclEntry);
return entries[pos];
}
+
+ @Override
+ public boolean equals(Object o) {
+if (o == null) {
+ return false;
+}
+if (getClass() != o.getClass()) {
+ return false;
+}
+return Arrays.equals(entries, ((AclFeature) o).entries);
+ }
+
+ @Override
+ public int hashCode() {
+return Arrays.hashCode(entries);
+ }
+
+ @Override
+ public int getRefCount() {
+return refCount;
+ }
+
+ @Override
+ public int incrementAndGetRefCount() {
+return ++refCount;
+ }
+
+ @Override
+ public int decrementAndGetRefCount() {
+return (refCount 0) ? --refCount : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50ae1a66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop
hadoop git commit: HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce memory footprint of NameNode (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8b398a66c - cd8e78d50
HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
memory footprint of NameNode (Contributed by Vinayakumar B)
(cherry picked from commit 50ae1a6664a92619aa683d2a864d0da9fb4af026)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd8e78d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd8e78d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd8e78d5
Branch: refs/heads/branch-2
Commit: cd8e78d50fedd368ae1fef1324cf122af8d6f562
Parents: 8b398a6
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Dec 23 12:35:53 2014 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Dec 23 12:37:16 2014 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/namenode/AclFeature.java | 37 +++-
.../hadoop/hdfs/server/namenode/AclStorage.java | 30 ++-
.../hdfs/server/namenode/INodeAttributes.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java| 9 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 +
.../namenode/INodeWithAdditionalFields.java | 5 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 5 +
.../snapshot/FileWithSnapshotFeature.java | 6 +
.../hadoop/hdfs/util/ReferenceCountMap.java | 118 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 207 ++-
.../server/namenode/TestFileContextAcl.java | 6 +-
.../hdfs/server/namenode/TestNameNodeAcl.java | 6 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 189 -
.../apache/hadoop/hdfs/web/TestWebHDFSAcl.java | 6 +-
15 files changed, 614 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd8e78d5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d2d67e4..360d38b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -373,6 +373,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
restart/failover. (Vinayakumar B via cnauroth)
+HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
+memory footprint of NameNode (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd8e78d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index e097b05..97d4759 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.Arrays;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -28,9 +31,10 @@ import com.google.common.collect.ImmutableList;
* Feature that represents the ACLs of the inode.
*/
@InterfaceAudience.Private
-public class AclFeature implements INode.Feature {
+public class AclFeature implements INode.Feature, ReferenceCounter {
public static final ImmutableListAclEntry EMPTY_ENTRY_LIST =
ImmutableList.of();
+ private int refCount = 0;
private final int [] entries;
@@ -56,4 +60,35 @@ public class AclFeature implements INode.Feature {
Invalid position for AclEntry);
return entries[pos];
}
+
+ @Override
+ public boolean equals(Object o) {
+if (o == null) {
+ return false;
+}
+if (getClass() != o.getClass()) {
+ return false;
+}
+return Arrays.equals(entries, ((AclFeature) o).entries);
+ }
+
+ @Override
+ public int hashCode() {
+return Arrays.hashCode(entries);
+ }
+
+ @Override
+ public int getRefCount() {
+return refCount;
+ }
+
+ @Override
+ public int incrementAndGetRefCount() {
+return ++refCount;
+ }
+
+ @Override
+ public int decrementAndGetRefCount() {
+return (refCount 0) ? --refCount : 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob
hadoop git commit: HDFS-7481. Add ACL indicator to the 'Permission Denied' exception. (Contributed by Vinayakumar B )
Repository: hadoop
Updated Branches:
refs/heads/trunk 437322afc - d93f3b981
HDFS-7481. Add ACL indicator to the 'Permission Denied' exception. (Contributed
by Vinayakumar B )
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d93f3b98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d93f3b98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d93f3b98
Branch: refs/heads/trunk
Commit: d93f3b9815f90d24c838574a56013e6dc60dc5ad
Parents: 437322a
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Dec 10 08:27:15 2014 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Dec 10 08:27:15 2014 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSPermissionChecker.java | 11 ++-
2 files changed, 13 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93f3b98/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9398429..d141439 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -558,6 +558,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7473. Document setting dfs.namenode.fs-limits.max-directory-items to 0
is invalid. (Akira AJISAKA via cnauroth)
+HDFS-7481. Add ACL indicator to the Permission Denied exception.
+(vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93f3b98/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 8de8c54..0508484 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -47,6 +47,12 @@ class FSPermissionChecker {
/** @return a string for throwing {@link AccessControlException} */
private String toAccessControlString(INode inode, int snapshotId,
FsAction access, FsPermission mode) {
+return toAccessControlString(inode, snapshotId, access, mode, false);
+ }
+
+ /** @return a string for throwing {@link AccessControlException} */
+ private String toAccessControlString(INode inode, int snapshotId, FsAction
access,
+ FsPermission mode, boolean deniedFromAcl) {
StringBuilder sb = new StringBuilder(Permission denied: )
.append(user=).append(user).append(, )
.append(access=).append(access).append(, )
@@ -55,6 +61,9 @@ class FSPermissionChecker {
.append(inode.getGroupName(snapshotId)).append(':')
.append(inode.isDirectory() ? 'd' : '-')
.append(mode);
+if (deniedFromAcl) {
+ sb.append(+);
+}
return sb.toString();
}
@@ -338,7 +347,7 @@ class FSPermissionChecker {
}
throw new AccessControlException(
- toAccessControlString(inode, snapshotId, access, mode));
+ toAccessControlString(inode, snapshotId, access, mode, true));
}
/** Guarded by {@link FSNamesystem#readLock()} */
hadoop git commit: HDFS-7481. Add ACL indicator to the 'Permission Denied' exception. (Contributed by Vinayakumar B )
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b31dfb17d - 19627e589
HDFS-7481. Add ACL indicator to the 'Permission Denied' exception. (Contributed
by Vinayakumar B )
(cherry picked from commit d93f3b9815f90d24c838574a56013e6dc60dc5ad)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19627e58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19627e58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19627e58
Branch: refs/heads/branch-2
Commit: 19627e5897388dadb85bee5f022ee195d189844b
Parents: b31dfb1
Author: Vinayakumar B vinayakum...@apache.org
Authored: Wed Dec 10 08:27:15 2014 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Wed Dec 10 08:32:39 2014 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSPermissionChecker.java | 11 ++-
2 files changed, 13 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19627e58/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a04e7cc..2496083 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -301,6 +301,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7473. Document setting dfs.namenode.fs-limits.max-directory-items to 0
is invalid. (Akira AJISAKA via cnauroth)
+HDFS-7481. Add ACL indicator to the Permission Denied exception.
+(vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19627e58/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
index 8de8c54..0508484 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSPermissionChecker.java
@@ -47,6 +47,12 @@ class FSPermissionChecker {
/** @return a string for throwing {@link AccessControlException} */
private String toAccessControlString(INode inode, int snapshotId,
FsAction access, FsPermission mode) {
+return toAccessControlString(inode, snapshotId, access, mode, false);
+ }
+
+ /** @return a string for throwing {@link AccessControlException} */
+ private String toAccessControlString(INode inode, int snapshotId, FsAction
access,
+ FsPermission mode, boolean deniedFromAcl) {
StringBuilder sb = new StringBuilder(Permission denied: )
.append(user=).append(user).append(, )
.append(access=).append(access).append(, )
@@ -55,6 +61,9 @@ class FSPermissionChecker {
.append(inode.getGroupName(snapshotId)).append(':')
.append(inode.isDirectory() ? 'd' : '-')
.append(mode);
+if (deniedFromAcl) {
+ sb.append(+);
+}
return sb.toString();
}
@@ -338,7 +347,7 @@ class FSPermissionChecker {
}
throw new AccessControlException(
- toAccessControlString(inode, snapshotId, access, mode));
+ toAccessControlString(inode, snapshotId, access, mode, true));
}
/** Guarded by {@link FSNamesystem#readLock()} */
[02/29] hadoop git commit: HDFS-7448 TestBookKeeperHACheckpoints fails in trunk -move CHANGES.TXT entry
HDFS-7448 TestBookKeeperHACheckpoints fails in trunk -move CHANGES.TXT entry Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22afae89 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22afae89 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22afae89 Branch: refs/heads/HDFS-EC Commit: 22afae890d7cf34a9be84590e7457774755b7a4a Parents: e65b7c5 Author: Steve Loughran ste...@apache.org Authored: Wed Dec 3 12:21:42 2014 + Committer: Steve Loughran ste...@apache.org Committed: Wed Dec 3 12:21:42 2014 + -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/22afae89/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 85d00b7..1679a71 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -257,9 +257,6 @@ Trunk (Unreleased) HDFS-7407. Minor typo in privileged pid/out/log names (aw) -HDFS-7448 TestBookKeeperHACheckpoints fails in trunk build -(Akira Ajisaka via stevel) - Release 2.7.0 - UNRELEASED INCOMPATIBLE CHANGES @@ -522,6 +519,9 @@ Release 2.7.0 - UNRELEASED HDFS-7444. convertToBlockUnderConstruction should preserve BlockCollection. (wheat9) +HDFS-7448 TestBookKeeperHACheckpoints fails in trunk build +(Akira Ajisaka via stevel) + Release 2.6.1 - UNRELEASED INCOMPATIBLE CHANGES
[05/29] hadoop git commit: HDFS-7458. Add description to the nfs ports in core-site.xml used by nfs test to avoid confusion. Contributed by Yongjun Zhang
HDFS-7458. Add description to the nfs ports in core-site.xml used by nfs test to avoid confusion. Contributed by Yongjun Zhang Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1e82259 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1e82259 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1e82259 Branch: refs/heads/HDFS-EC Commit: a1e822595c3dc5eadbd5430e57bc4691d09d5e68 Parents: 1812241 Author: Brandon Li brando...@apache.org Authored: Wed Dec 3 13:31:26 2014 -0800 Committer: Brandon Li brando...@apache.org Committed: Wed Dec 3 13:31:26 2014 -0800 -- .../hadoop-hdfs-nfs/src/test/resources/core-site.xml | 14 ++ hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++ 2 files changed, 17 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1e82259/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml -- diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml index f90ca03..f400bf2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml +++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/resources/core-site.xml @@ -20,10 +20,24 @@ property namenfs.server.port/name value2079/value + description +Specify the port number used by Hadoop NFS. +Notice that the default value here is different than the default Hadoop nfs +port 2049 specified in hdfs-default.xml. 2049 is also the default port for +Linux nfs. The setting here allows starting Hadoop nfs for testing even if +nfs server (linux or Hadoop) is aready running on he same host. + /description /property property namenfs.mountd.port/name value4272/value + description +Specify the port number used by Hadoop mount daemon. +Notice that the default value here is different than 4242 specified in +hdfs-default.xml. This setting allows starting Hadoop nfs mountd for +testing even if the Linux or Hadoop mountd is already running on the +same host. + /description /property /configuration http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1e82259/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 1679a71..a244dab 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -421,6 +421,9 @@ Release 2.7.0 - UNRELEASED HDFS-6735. A minor optimization to avoid pread() be blocked by read() inside the same DFSInputStream (Lars Hofhansl via stack) + +HDFS-7458. Add description to the nfs ports in core-site.xml used by nfs +test to avoid confusion (Yongjun Zhang via brandonli) OPTIMIZATIONS
[07/29] hadoop git commit: YARN-2891. Failed Container Executor does not provide a clear error message. Contributed by Dustin Cote. (harsh)
YARN-2891. Failed Container Executor does not provide a clear error message.
Contributed by Dustin Cote. (harsh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a31e0164
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a31e0164
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a31e0164
Branch: refs/heads/HDFS-EC
Commit: a31e0164912236630c485e5aeb908b43e3a67c61
Parents: 799353e
Author: Harsh J ha...@cloudera.com
Authored: Thu Dec 4 03:16:08 2014 +0530
Committer: Harsh J ha...@cloudera.com
Committed: Thu Dec 4 03:17:15 2014 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../src/main/native/container-executor/impl/container-executor.c | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a31e0164/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d44f46d..91151ad 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -59,6 +59,9 @@ Release 2.7.0 - UNRELEASED
IMPROVEMENTS
+YARN-2891. Failed Container Executor does not provide a clear error
+message. (Dustin Cote via harsh)
+
YARN-1979. TestDirectoryCollection fails when the umask is unusual.
(Vinod Kumar Vavilapalli and Tsuyoshi OZAWA via junping_du)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a31e0164/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 9af9161..4fc78b6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -526,7 +526,7 @@ int check_dir(char* npath, mode_t st_mode, mode_t desired,
int finalComponent) {
int filePermInt = st_mode (S_IRWXU | S_IRWXG | S_IRWXO);
int desiredInt = desired (S_IRWXU | S_IRWXG | S_IRWXO);
if (filePermInt != desiredInt) {
- fprintf(LOGFILE, Path %s does not have desired permission.\n, npath);
+ fprintf(LOGFILE, Path %s has permission %o but needs permission %o.\n,
npath, filePermInt, desiredInt);
return -1;
}
}
[27/29] hadoop git commit: HADOOP-11313. Adding a document about NativeLibraryChecker. Contributed by Tsuyoshi OZAWA.
HADOOP-11313. Adding a document about NativeLibraryChecker. Contributed by Tsuyoshi OZAWA. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b3bb9e7 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b3bb9e7 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b3bb9e7 Branch: refs/heads/HDFS-EC Commit: 1b3bb9e7a33716c4d94786598b91a24a4b29fe67 Parents: 9297f98 Author: cnauroth cnaur...@apache.org Authored: Sat Dec 6 20:12:31 2014 -0800 Committer: cnauroth cnaur...@apache.org Committed: Sat Dec 6 20:12:31 2014 -0800 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../src/site/apt/NativeLibraries.apt.vm | 18 ++ 2 files changed, 21 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3bb9e7/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index 965c6d3..a626388 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -413,6 +413,9 @@ Release 2.7.0 - UNRELEASED HADOOP-11356. Removed deprecated o.a.h.fs.permission.AccessControlException. (Li Lu via wheat9) +HADOOP-11313. Adding a document about NativeLibraryChecker. +(Tsuyoshi OZAWA via cnauroth) + OPTIMIZATIONS HADOOP-11323. WritableComparator#compare keeps reference to byte array. http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3bb9e7/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm -- diff --git a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm index 49818af..866b428 100644 --- a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm +++ b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm @@ -164,6 +164,24 @@ Native Libraries Guide * If something goes wrong, then: INFO util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable +* Check + + NativeLibraryChecker is a tool to check whether native libraries are loaded correctly. + You can launch NativeLibraryChecker as follows: + + + $ hadoop checknative -a + 14/12/06 01:30:45 WARN bzip2.Bzip2Factory: Failed to load/initialize native-bzip2 library system-native, will use pure-Java version + 14/12/06 01:30:45 INFO zlib.ZlibFactory: Successfully loaded initialized native-zlib library + Native library checking: + hadoop: true /home/ozawa/hadoop/lib/native/libhadoop.so.1.0.0 + zlib: true /lib/x86_64-linux-gnu/libz.so.1 + snappy: true /usr/lib/libsnappy.so.1 + lz4:true revision:99 + bzip2: false + + + * Native Shared Libraries You can load any native shared library using DistributedCache for
[29/29] hadoop git commit: Merge remote-tracking branch 'origin/trunk' into HDFS-EC
Merge remote-tracking branch 'origin/trunk' into HDFS-EC Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83318534 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83318534 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83318534 Branch: refs/heads/HDFS-EC Commit: 8331853450bc52fb3d8eed61fde52639a5fdb619 Parents: bdc0101 120e1de Author: Vinayakumar B vinayakuma...@intel.com Authored: Mon Dec 8 14:42:52 2014 +0530 Committer: Vinayakumar B vinayakuma...@intel.com Committed: Mon Dec 8 14:42:52 2014 +0530 -- .../client/KerberosAuthenticator.java | 6 +- hadoop-common-project/hadoop-common/CHANGES.txt | 21 + .../hadoop-common/src/CMakeLists.txt| 2 +- .../apache/hadoop/crypto/AesCtrCryptoCodec.java | 27 +- .../fs/permission/AccessControlException.java | 66 --- .../hadoop/security/AccessControlException.java | 4 +- .../src/site/apt/NativeLibraries.apt.vm | 18 + .../apache/hadoop/crypto/TestCryptoCodec.java | 64 +++ .../hadoop/crypto/key/kms/server/KMSACLs.java | 26 +- .../kms/server/KeyAuthorizationKeyProvider.java | 4 + .../hadoop/crypto/key/kms/server/TestKMS.java | 13 +- hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 5 + .../hadoop/hdfs/nfs/conf/NfsConfigKeys.java | 10 + .../hadoop/hdfs/nfs/nfs3/Nfs3HttpServer.java| 111 .../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java| 24 +- .../hdfs/nfs/nfs3/TestNfs3HttpServer.java | 89 .../src/test/resources/core-site.xml| 14 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 31 +- hadoop-hdfs-project/hadoop-hdfs/pom.xml | 3 + .../datanode/ReplicaNotFoundException.java | 2 +- .../server/namenode/AclEntryStatusFormat.java | 136 + .../hadoop/hdfs/server/namenode/AclFeature.java | 24 +- .../hadoop/hdfs/server/namenode/AclStorage.java | 61 +-- .../server/namenode/EncryptionZoneManager.java | 4 +- .../hadoop/hdfs/server/namenode/FSDirAclOp.java | 244 + .../hdfs/server/namenode/FSDirConcatOp.java | 9 +- .../hdfs/server/namenode/FSDirMkdirOp.java | 17 +- .../hdfs/server/namenode/FSDirRenameOp.java | 74 ++- .../hdfs/server/namenode/FSDirSnapshotOp.java | 68 ++- .../server/namenode/FSDirStatAndListingOp.java | 35 +- .../hdfs/server/namenode/FSDirectory.java | 329 +++- .../hdfs/server/namenode/FSEditLogLoader.java | 18 +- .../server/namenode/FSImageFormatPBINode.java | 22 +- .../hdfs/server/namenode/FSNDNCacheOp.java | 124 + .../hdfs/server/namenode/FSNamesystem.java | 531 +++ .../server/namenode/FSPermissionChecker.java| 61 +-- .../hdfs/server/namenode/INodesInPath.java | 21 +- .../hadoop/hdfs/server/namenode/NNConf.java | 104 .../snapshot/FSImageFormatPBSnapshot.java | 13 +- .../namenode/snapshot/SnapshotManager.java | 50 +- .../hdfs/server/namenode/FSAclBaseTest.java | 4 +- .../hdfs/server/namenode/TestAuditLogger.java | 79 +-- .../namenode/TestFSPermissionChecker.java | 10 +- .../server/namenode/TestSnapshotPathINodes.java | 20 +- .../namenode/snapshot/TestSnapshotManager.java | 14 +- hadoop-mapreduce-project/CHANGES.txt| 3 + .../apache/hadoop/mapred/MapReduceChildJVM.java | 34 +- .../v2/app/job/impl/TestMapReduceChildJVM.java | 71 ++- .../apache/hadoop/mapreduce/v2/util/MRApps.java | 80 ++- .../apache/hadoop/mapred/FileOutputFormat.java | 4 +- .../java/org/apache/hadoop/mapred/TaskLog.java | 4 + .../apache/hadoop/mapreduce/MRJobConfig.java| 14 + .../src/main/resources/mapred-default.xml | 28 + .../org/apache/hadoop/mapred/YARNRunner.java| 9 +- hadoop-yarn-project/CHANGES.txt | 28 + hadoop-yarn-project/hadoop-yarn/bin/yarn| 5 + .../hadoop-yarn/hadoop-yarn-api/pom.xml | 1 + .../hadoop/yarn/conf/YarnConfiguration.java | 16 +- .../yarn/server/api/SCMAdminProtocol.java | 53 ++ .../yarn/server/api/SCMAdminProtocolPB.java | 31 ++ .../RunSharedCacheCleanerTaskRequest.java | 37 ++ .../RunSharedCacheCleanerTaskResponse.java | 58 ++ .../main/proto/server/SCM_Admin_protocol.proto | 29 + .../src/main/proto/yarn_service_protos.proto| 11 + .../org/apache/hadoop/yarn/client/SCMAdmin.java | 183 +++ .../hadoop/yarn/client/cli/ApplicationCLI.java | 8 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 41 +- .../hadoop/yarn/ContainerLogAppender.java | 11 +- .../yarn/ContainerRollingLogAppender.java | 11 +- .../pb/client/SCMAdminProtocolPBClientImpl.java | 73 +++ .../service/SCMAdminProtocolPBServiceImpl.java | 57 ++ .../RunSharedCacheCleanerTaskRequestPBImpl.java | 53 ++ ...RunSharedCacheCleanerTaskResponsePBImpl.java | 66 +++ .../src/main/resources/yarn-default.xml
