hadoop git commit: YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via junping_du)
Repository: hadoop
Updated Branches:
refs/heads/trunk ecf1469fa - a696fbb00
YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via
junping_du)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a696fbb0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a696fbb0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a696fbb0
Branch: refs/heads/trunk
Commit: a696fbb001b946ae75f3b8e962839c2fd3decfa1
Parents: ecf1469
Author: Junping Du junping...@apache.org
Authored: Mon Dec 22 03:06:37 2014 -0800
Committer: Junping Du junping...@apache.org
Committed: Mon Dec 22 03:06:37 2014 -0800
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../dev-support/findbugs-exclude.xml| 11
.../api/records/impl/pb/PriorityPBImpl.java | 2 +-
.../client/api/impl/TimelineClientImpl.java | 6 -
.../logaggregation/AggregatedLogFormat.java | 8 +++---
.../nodelabels/CommonNodeLabelsManager.java | 7 -
.../hadoop/yarn/nodelabels/NodeLabelsStore.java | 7 ++---
.../org/apache/hadoop/yarn/state/Graph.java | 13 +
.../yarn/state/VisualizeStateMachine.java | 2 +-
.../util/LinuxResourceCalculatorPlugin.java | 19 -
.../yarn/util/ProcfsBasedProcessTree.java | 28
.../yarn/util/ResourceCalculatorPlugin.java | 12 -
.../yarn/util/WindowsBasedProcessTree.java | 4 +--
.../client/api/impl/TestTimelineClient.java | 4 +--
.../nodelabels/RMNodeLabelsManager.java | 10 +++
15 files changed, 79 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a696fbb0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0d579e0..7a0904e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -261,6 +261,8 @@ Release 2.7.0 - UNRELEASED
YARN-2977. Fixed intermittent TestNMClient failure.
(Junping Du via ozawa)
+YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via
junping_du)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a696fbb0/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 45d7294..f531afe 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -265,6 +265,11 @@
Class name=org.apache.hadoop.yarn.YarnUncaughtExceptionHandler/
Bug pattern=DM_EXIT/
/Match
+ Match
+Class name=org.apache.hadoop.yarn.event.AsyncDispatcher$2/
+Method name=run /
+Bug pattern=DM_EXIT/
+ /Match
!-- AsyncDispatcher will kill the process if there is an error dispatching
--
Match
@@ -391,4 +396,10 @@
Bug pattern=UI_INHERITANCE_UNSAFE_GETRESOURCE/
/Match
+ !-- Ignore the false alarms on DM_DEFAULT_ENCODING (encoding already set)
--
+ Match
+Class
name=org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat$LogReader /
+Bug pattern=DM_DEFAULT_ENCODING /
+ /Match
+
/FindBugsFilter
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a696fbb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
index c8abdbe..b4d92af 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
@@ -68,7 +68,7 @@ public class PriorityPBImpl extends Priority {
@Override
public String toString() {
-return Integer.valueOf(getPriority()).toString();
+return Integer.toString(getPriority());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a696fbb0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
hadoop git commit: YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via junping_du) (cherry picked from commit a696fbb001b946ae75f3b8e962839c2fd3decfa1)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 370a137c1 - 6f252f7f3
YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via
junping_du)
(cherry picked from commit a696fbb001b946ae75f3b8e962839c2fd3decfa1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f252f7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f252f7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f252f7f
Branch: refs/heads/branch-2
Commit: 6f252f7f363a98a9675cf5ec88100dbef90f7015
Parents: 370a137
Author: Junping Du junping...@apache.org
Authored: Mon Dec 22 03:06:37 2014 -0800
Committer: Junping Du junping...@apache.org
Committed: Mon Dec 22 03:08:25 2014 -0800
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../dev-support/findbugs-exclude.xml| 11
.../api/records/impl/pb/PriorityPBImpl.java | 2 +-
.../client/api/impl/TimelineClientImpl.java | 6 -
.../logaggregation/AggregatedLogFormat.java | 8 +++---
.../nodelabels/CommonNodeLabelsManager.java | 7 -
.../hadoop/yarn/nodelabels/NodeLabelsStore.java | 7 ++---
.../org/apache/hadoop/yarn/state/Graph.java | 13 +
.../yarn/state/VisualizeStateMachine.java | 2 +-
.../util/LinuxResourceCalculatorPlugin.java | 19 -
.../yarn/util/ProcfsBasedProcessTree.java | 28
.../yarn/util/ResourceCalculatorPlugin.java | 12 -
.../yarn/util/WindowsBasedProcessTree.java | 4 +--
.../client/api/impl/TestTimelineClient.java | 4 +--
.../nodelabels/RMNodeLabelsManager.java | 10 +++
15 files changed, 79 insertions(+), 56 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f252f7f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 90a2943..96ff9fa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -229,6 +229,8 @@ Release 2.7.0 - UNRELEASED
YARN-2977. Fixed intermittent TestNMClient failure.
(Junping Du via ozawa)
+YARN-2939. Fix new findbugs warnings in hadoop-yarn-common. (Li Lu via
junping_du)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f252f7f/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 45d7294..f531afe 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -265,6 +265,11 @@
Class name=org.apache.hadoop.yarn.YarnUncaughtExceptionHandler/
Bug pattern=DM_EXIT/
/Match
+ Match
+Class name=org.apache.hadoop.yarn.event.AsyncDispatcher$2/
+Method name=run /
+Bug pattern=DM_EXIT/
+ /Match
!-- AsyncDispatcher will kill the process if there is an error dispatching
--
Match
@@ -391,4 +396,10 @@
Bug pattern=UI_INHERITANCE_UNSAFE_GETRESOURCE/
/Match
+ !-- Ignore the false alarms on DM_DEFAULT_ENCODING (encoding already set)
--
+ Match
+Class
name=org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat$LogReader /
+Bug pattern=DM_DEFAULT_ENCODING /
+ /Match
+
/FindBugsFilter
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f252f7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
index c8abdbe..b4d92af 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/PriorityPBImpl.java
@@ -68,7 +68,7 @@ public class PriorityPBImpl extends Priority {
@Override
public String toString() {
-return Integer.valueOf(getPriority()).toString();
+return Integer.toString(getPriority());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f252f7f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
hadoop git commit: HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode restart/failover. Contributed by Vinayakumar B.
Repository: hadoop
Updated Branches:
refs/heads/trunk a696fbb00 - 2cf90a2c3
HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
restart/failover. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2cf90a2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2cf90a2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2cf90a2c
Branch: refs/heads/trunk
Commit: 2cf90a2c338497a466bbad9e83966033bf14bfb7
Parents: a696fbb
Author: cnauroth cnaur...@apache.org
Authored: Mon Dec 22 13:59:10 2014 -0800
Committer: cnauroth cnaur...@apache.org
Committed: Mon Dec 22 13:59:10 2014 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 12 +++---
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 39
4 files changed, 44 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cf90a2c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c38b92a..07a78c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -627,6 +627,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7557. Fix spacing for a few keys in DFSConfigKeys.java
(Colin P.McCabe)
+HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
+restart/failover. (Vinayakumar B via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cf90a2c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index 7aaa21c..dff1c2e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -143,7 +143,7 @@ class FSDirAclOp {
try {
iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
- ListAclEntry newAcl = unprotectedSetAcl(fsd, src, aclSpec);
+ ListAclEntry newAcl = unprotectedSetAcl(fsd, src, aclSpec, false);
fsd.getEditLog().logSetAcl(src, newAcl);
} finally {
fsd.writeUnlock();
@@ -185,7 +185,7 @@ class FSDirAclOp {
}
static ListAclEntry unprotectedSetAcl(
- FSDirectory fsd, String src, ListAclEntry aclSpec)
+ FSDirectory fsd, String src, ListAclEntry aclSpec, boolean fromEdits)
throws IOException {
assert fsd.hasWriteLock();
final INodesInPath iip = fsd.getINodesInPath4Write(
@@ -199,9 +199,11 @@ class FSDirAclOp {
INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
-ListAclEntry existingAcl = AclStorage.readINodeLogicalAcl(inode);
-ListAclEntry newAcl = AclTransformation.replaceAclEntries(existingAcl,
- aclSpec);
+ListAclEntry newAcl = aclSpec;
+if (!fromEdits) {
+ ListAclEntry existingAcl = AclStorage.readINodeLogicalAcl(inode);
+ newAcl = AclTransformation.replaceAclEntries(existingAcl, aclSpec);
+}
AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
return newAcl;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2cf90a2c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 9d08d4e..2ae5e03 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -823,7 +823,8 @@ public class FSEditLogLoader {
}
case OP_SET_ACL: {
SetAclOp setAclOp = (SetAclOp) op;
- FSDirAclOp.unprotectedSetAcl(fsDir, setAclOp.src, setAclOp.aclEntries);
+ FSDirAclOp.unprotectedSetAcl(fsDir, setAclOp.src, setAclOp.aclEntries,
+ true);
break;
}
case OP_SET_XATTR: {
hadoop git commit: HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode restart/failover. Contributed by Vinayakumar B.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6f252f7f3 - 548612466
HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
restart/failover. Contributed by Vinayakumar B.
(cherry picked from commit 2cf90a2c338497a466bbad9e83966033bf14bfb7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/54861246
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/54861246
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/54861246
Branch: refs/heads/branch-2
Commit: 54861246684e9468df9d2b68b9103099ab48cdc6
Parents: 6f252f7
Author: cnauroth cnaur...@apache.org
Authored: Mon Dec 22 13:59:10 2014 -0800
Committer: cnauroth cnaur...@apache.org
Committed: Mon Dec 22 14:02:49 2014 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../hadoop/hdfs/server/namenode/FSDirAclOp.java | 12 +++---
.../hdfs/server/namenode/FSEditLogLoader.java | 3 +-
.../hdfs/server/namenode/FSAclBaseTest.java | 39
4 files changed, 44 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54861246/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 997552d..d2d67e4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -370,6 +370,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7557. Fix spacing for a few keys in DFSConfigKeys.java
(Colin P.McCabe)
+HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
+restart/failover. (Vinayakumar B via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54861246/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
index 7aaa21c..dff1c2e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAclOp.java
@@ -143,7 +143,7 @@ class FSDirAclOp {
try {
iip = fsd.getINodesInPath4Write(src);
fsd.checkOwner(pc, iip);
- ListAclEntry newAcl = unprotectedSetAcl(fsd, src, aclSpec);
+ ListAclEntry newAcl = unprotectedSetAcl(fsd, src, aclSpec, false);
fsd.getEditLog().logSetAcl(src, newAcl);
} finally {
fsd.writeUnlock();
@@ -185,7 +185,7 @@ class FSDirAclOp {
}
static ListAclEntry unprotectedSetAcl(
- FSDirectory fsd, String src, ListAclEntry aclSpec)
+ FSDirectory fsd, String src, ListAclEntry aclSpec, boolean fromEdits)
throws IOException {
assert fsd.hasWriteLock();
final INodesInPath iip = fsd.getINodesInPath4Write(
@@ -199,9 +199,11 @@ class FSDirAclOp {
INode inode = FSDirectory.resolveLastINode(iip);
int snapshotId = iip.getLatestSnapshotId();
-ListAclEntry existingAcl = AclStorage.readINodeLogicalAcl(inode);
-ListAclEntry newAcl = AclTransformation.replaceAclEntries(existingAcl,
- aclSpec);
+ListAclEntry newAcl = aclSpec;
+if (!fromEdits) {
+ ListAclEntry existingAcl = AclStorage.readINodeLogicalAcl(inode);
+ newAcl = AclTransformation.replaceAclEntries(existingAcl, aclSpec);
+}
AclStorage.updateINodeAcl(inode, newAcl, snapshotId);
return newAcl;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/54861246/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 9d79ffa..c275ea5 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -827,7 +827,8 @@ public class FSEditLogLoader {
}
case OP_SET_ACL: {
SetAclOp setAclOp = (SetAclOp) op;
- FSDirAclOp.unprotectedSetAcl(fsDir, setAclOp.src, setAclOp.aclEntries);
+ FSDirAclOp.unprotectedSetAcl(fsDir, setAclOp.src, setAclOp.aclEntries,
+
hadoop git commit: YARN-2920. Changed CapacityScheduler to kill containers on nodes where node labels are changed. Contributed by Wangda Tan (cherry picked from commit fdf042dfffa4d2474e3cac86cfb8fe9e
Repository: hadoop
Updated Branches:
refs/heads/branch-2 548612466 - 411836b74
YARN-2920. Changed CapacityScheduler to kill containers on nodes where node
labels are changed. Contributed by Wangda Tan
(cherry picked from commit fdf042dfffa4d2474e3cac86cfb8fe9ee4648beb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/411836b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/411836b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/411836b7
Branch: refs/heads/branch-2
Commit: 411836b74c6c02c0b5aebbbce29c209d93db1de2
Parents: 5486124
Author: Jian He jia...@apache.org
Authored: Mon Dec 22 16:50:15 2014 -0800
Committer: Jian He jia...@apache.org
Committed: Mon Dec 22 16:52:15 2014 -0800
--
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 3 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 3 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/ResourceManager.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 23 +++
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +-
.../scheduler/SchedulerNode.java| 24 ++-
.../scheduler/capacity/AbstractCSQueue.java | 5 +
.../scheduler/capacity/CSQueue.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 76 +++-
.../scheduler/capacity/LeafQueue.java | 26 +--
.../scheduler/capacity/ParentQueue.java | 24 ++-
.../common/fica/FiCaSchedulerNode.java | 11 +-
.../event/NodeLabelsUpdateSchedulerEvent.java | 37
.../scheduler/event/SchedulerEventType.java | 1 +
.../yarn/server/resourcemanager/MockNodes.java | 3 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 193 +++
17 files changed, 410 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/411836b7/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index fdddcf4..ee6eb7b 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -162,7 +163,7 @@ public class NodeInfo {
@Override
public SetString getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/411836b7/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 3b185ae..b64be1b 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -150,6 +151,6 @@ public class RMNodeWrapper implements RMNode {
@Override
public SetString getNodeLabels() {
-return null;
+return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/411836b7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 96ff9fa..4bcc1c5 100644
---
hadoop git commit: YARN-2920. Changed CapacityScheduler to kill containers on nodes where node labels are changed. Contributed by Wangda Tan
Repository: hadoop
Updated Branches:
refs/heads/trunk 2cf90a2c3 - fdf042dff
YARN-2920. Changed CapacityScheduler to kill containers on nodes where node
labels are changed. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fdf042df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fdf042df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fdf042df
Branch: refs/heads/trunk
Commit: fdf042dfffa4d2474e3cac86cfb8fe9ee4648beb
Parents: 2cf90a2
Author: Jian He jia...@apache.org
Authored: Mon Dec 22 16:50:15 2014 -0800
Committer: Jian He jia...@apache.org
Committed: Mon Dec 22 16:51:15 2014 -0800
--
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 3 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 3 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/ResourceManager.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 23 +++
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +-
.../scheduler/SchedulerNode.java| 24 ++-
.../scheduler/capacity/AbstractCSQueue.java | 5 +
.../scheduler/capacity/CSQueue.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 76 +++-
.../scheduler/capacity/LeafQueue.java | 26 +--
.../scheduler/capacity/ParentQueue.java | 24 ++-
.../common/fica/FiCaSchedulerNode.java | 11 +-
.../event/NodeLabelsUpdateSchedulerEvent.java | 37
.../scheduler/event/SchedulerEventType.java | 1 +
.../yarn/server/resourcemanager/MockNodes.java | 3 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 193 +++
17 files changed, 410 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf042df/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index fdddcf4..ee6eb7b 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -162,7 +163,7 @@ public class NodeInfo {
@Override
public SetString getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf042df/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 3b185ae..b64be1b 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -150,6 +151,6 @@ public class RMNodeWrapper implements RMNode {
@Override
public SetString getNodeLabels() {
-return null;
+return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fdf042df/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7a0904e..964c122 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -263,6 +263,9 @@
hadoop git commit: YARN-2340. Fixed NPE when queue is stopped during RM restart. Contributed by Rohith Sharmaks (cherry picked from commit 0d89859b51157078cc504ac81dc8aa75ce6b1782)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 411836b74 - 8b398a66c
YARN-2340. Fixed NPE when queue is stopped during RM restart. Contributed by
Rohith Sharmaks
(cherry picked from commit 0d89859b51157078cc504ac81dc8aa75ce6b1782)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b398a66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b398a66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b398a66
Branch: refs/heads/branch-2
Commit: 8b398a66ca3728f47363fc8b2fcf7e556e6bbf5a
Parents: 411836b
Author: Jian He jia...@apache.org
Authored: Mon Dec 22 21:53:22 2014 -0800
Committer: Jian He jia...@apache.org
Committed: Mon Dec 22 21:55:06 2014 -0800
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../scheduler/capacity/CapacityScheduler.java | 13 -
.../resourcemanager/TestWorkPreservingRMRestart.java | 2 ++
3 files changed, 13 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b398a66/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4bcc1c5..3ed97af 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -234,6 +234,9 @@ Release 2.7.0 - UNRELEASED
YARN-2920. Changed CapacityScheduler to kill containers on nodes where
node labels are changed. (Wangda Tan via jianhe)
+YARN-2340. Fixed NPE when queue is stopped during RM restart.
+(Rohith Sharmaks via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b398a66/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e865a65..3648c54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -706,11 +706,14 @@ public class CapacityScheduler extends
try {
queue.submitApplication(applicationId, user, queueName);
} catch (AccessControlException ace) {
- LOG.info(Failed to submit application + applicationId + to queue
- + queueName + from user + user, ace);
- this.rmContext.getDispatcher().getEventHandler()
- .handle(new RMAppRejectedEvent(applicationId, ace.toString()));
- return;
+ // Ignore the exception for recovered app as the app was previously
accepted
+ if (!isAppRecovering) {
+LOG.info(Failed to submit application + applicationId + to queue
++ queueName + from user + user, ace);
+this.rmContext.getDispatcher().getEventHandler()
+.handle(new RMAppRejectedEvent(applicationId, ace.toString()));
+return;
+ }
}
// update the metrics
queue.getMetrics().submitApp(user);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b398a66/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 853e0a5..842eaec 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++
hadoop git commit: YARN-2340. Fixed NPE when queue is stopped during RM restart. Contributed by Rohith Sharmaks
Repository: hadoop
Updated Branches:
refs/heads/trunk fdf042dff - 0d89859b5
YARN-2340. Fixed NPE when queue is stopped during RM restart. Contributed by
Rohith Sharmaks
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d89859b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d89859b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d89859b
Branch: refs/heads/trunk
Commit: 0d89859b51157078cc504ac81dc8aa75ce6b1782
Parents: fdf042d
Author: Jian He jia...@apache.org
Authored: Mon Dec 22 21:53:22 2014 -0800
Committer: Jian He jia...@apache.org
Committed: Mon Dec 22 21:53:22 2014 -0800
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../scheduler/capacity/CapacityScheduler.java | 13 -
.../resourcemanager/TestWorkPreservingRMRestart.java | 2 ++
3 files changed, 13 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d89859b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 964c122..cb2fc24 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -266,6 +266,9 @@ Release 2.7.0 - UNRELEASED
YARN-2920. Changed CapacityScheduler to kill containers on nodes where
node labels are changed. (Wangda Tan via jianhe)
+YARN-2340. Fixed NPE when queue is stopped during RM restart.
+(Rohith Sharmaks via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d89859b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e865a65..3648c54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -706,11 +706,14 @@ public class CapacityScheduler extends
try {
queue.submitApplication(applicationId, user, queueName);
} catch (AccessControlException ace) {
- LOG.info(Failed to submit application + applicationId + to queue
- + queueName + from user + user, ace);
- this.rmContext.getDispatcher().getEventHandler()
- .handle(new RMAppRejectedEvent(applicationId, ace.toString()));
- return;
+ // Ignore the exception for recovered app as the app was previously
accepted
+ if (!isAppRecovering) {
+LOG.info(Failed to submit application + applicationId + to queue
++ queueName + from user + user, ace);
+this.rmContext.getDispatcher().getEventHandler()
+.handle(new RMAppRejectedEvent(applicationId, ace.toString()));
+return;
+ }
}
// update the metrics
queue.getMetrics().submitApp(user);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d89859b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 853e0a5..842eaec 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -500,6 +500,8 @@ public class TestWorkPreservingRMRestart {
hadoop git commit: HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce memory footprint of NameNode (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0d89859b5 - 50ae1a666
HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
memory footprint of NameNode (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50ae1a66
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50ae1a66
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50ae1a66
Branch: refs/heads/trunk
Commit: 50ae1a6664a92619aa683d2a864d0da9fb4af026
Parents: 0d89859
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Dec 23 12:35:53 2014 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Dec 23 12:35:53 2014 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/namenode/AclFeature.java | 37 +++-
.../hadoop/hdfs/server/namenode/AclStorage.java | 30 ++-
.../hdfs/server/namenode/INodeAttributes.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java| 9 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 +
.../namenode/INodeWithAdditionalFields.java | 5 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 5 +
.../snapshot/FileWithSnapshotFeature.java | 6 +
.../hadoop/hdfs/util/ReferenceCountMap.java | 118 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 207 ++-
.../server/namenode/TestFileContextAcl.java | 6 +-
.../hdfs/server/namenode/TestNameNodeAcl.java | 6 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 189 -
.../apache/hadoop/hdfs/web/TestWebHDFSAcl.java | 6 +-
15 files changed, 614 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50ae1a66/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 07a78c2..66b3a8d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -630,6 +630,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
restart/failover. (Vinayakumar B via cnauroth)
+HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
+memory footprint of NameNode (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50ae1a66/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index e097b05..97d4759 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.Arrays;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -28,9 +31,10 @@ import com.google.common.collect.ImmutableList;
* Feature that represents the ACLs of the inode.
*/
@InterfaceAudience.Private
-public class AclFeature implements INode.Feature {
+public class AclFeature implements INode.Feature, ReferenceCounter {
public static final ImmutableListAclEntry EMPTY_ENTRY_LIST =
ImmutableList.of();
+ private int refCount = 0;
private final int [] entries;
@@ -56,4 +60,35 @@ public class AclFeature implements INode.Feature {
Invalid position for AclEntry);
return entries[pos];
}
+
+ @Override
+ public boolean equals(Object o) {
+if (o == null) {
+ return false;
+}
+if (getClass() != o.getClass()) {
+ return false;
+}
+return Arrays.equals(entries, ((AclFeature) o).entries);
+ }
+
+ @Override
+ public int hashCode() {
+return Arrays.hashCode(entries);
+ }
+
+ @Override
+ public int getRefCount() {
+return refCount;
+ }
+
+ @Override
+ public int incrementAndGetRefCount() {
+return ++refCount;
+ }
+
+ @Override
+ public int decrementAndGetRefCount() {
+return (refCount 0) ? --refCount : 0;
+ }
}
hadoop git commit: HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce memory footprint of NameNode (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8b398a66c - cd8e78d50
HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
memory footprint of NameNode (Contributed by Vinayakumar B)
(cherry picked from commit 50ae1a6664a92619aa683d2a864d0da9fb4af026)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd8e78d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd8e78d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd8e78d5
Branch: refs/heads/branch-2
Commit: cd8e78d50fedd368ae1fef1324cf122af8d6f562
Parents: 8b398a6
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Dec 23 12:35:53 2014 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Dec 23 12:37:16 2014 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/server/namenode/AclFeature.java | 37 +++-
.../hadoop/hdfs/server/namenode/AclStorage.java | 30 ++-
.../hdfs/server/namenode/INodeAttributes.java | 9 +-
.../hdfs/server/namenode/INodeDirectory.java| 9 +
.../hadoop/hdfs/server/namenode/INodeFile.java | 3 +
.../namenode/INodeWithAdditionalFields.java | 5 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 5 +
.../snapshot/FileWithSnapshotFeature.java | 6 +
.../hadoop/hdfs/util/ReferenceCountMap.java | 118 +++
.../hdfs/server/namenode/FSAclBaseTest.java | 207 ++-
.../server/namenode/TestFileContextAcl.java | 6 +-
.../hdfs/server/namenode/TestNameNodeAcl.java | 6 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 189 -
.../apache/hadoop/hdfs/web/TestWebHDFSAcl.java | 6 +-
15 files changed, 614 insertions(+), 25 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd8e78d5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d2d67e4..360d38b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -373,6 +373,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7560. ACLs removed by removeDefaultAcl() will be back after NameNode
restart/failover. (Vinayakumar B via cnauroth)
+HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
+memory footprint of NameNode (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd8e78d5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
index e097b05..97d4759 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclFeature.java
@@ -18,8 +18,11 @@
package org.apache.hadoop.hdfs.server.namenode;
+import java.util.Arrays;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.hdfs.util.ReferenceCountMap.ReferenceCounter;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
@@ -28,9 +31,10 @@ import com.google.common.collect.ImmutableList;
* Feature that represents the ACLs of the inode.
*/
@InterfaceAudience.Private
-public class AclFeature implements INode.Feature {
+public class AclFeature implements INode.Feature, ReferenceCounter {
public static final ImmutableListAclEntry EMPTY_ENTRY_LIST =
ImmutableList.of();
+ private int refCount = 0;
private final int [] entries;
@@ -56,4 +60,35 @@ public class AclFeature implements INode.Feature {
Invalid position for AclEntry);
return entries[pos];
}
+
+ @Override
+ public boolean equals(Object o) {
+if (o == null) {
+ return false;
+}
+if (getClass() != o.getClass()) {
+ return false;
+}
+return Arrays.equals(entries, ((AclFeature) o).entries);
+ }
+
+ @Override
+ public int hashCode() {
+return Arrays.hashCode(entries);
+ }
+
+ @Override
+ public int getRefCount() {
+return refCount;
+ }
+
+ @Override
+ public int incrementAndGetRefCount() {
+return ++refCount;
+ }
+
+ @Override
+ public int decrementAndGetRefCount() {
+return (refCount 0) ? --refCount : 0;
+ }
}
hadoop git commit: HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter. Contributed by Jing Zhao.
Repository: hadoop
Updated Branches:
refs/heads/trunk 50ae1a666 - 5caebbae8
HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter.
Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5caebbae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5caebbae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5caebbae
Branch: refs/heads/trunk
Commit: 5caebbae8c2fc9ba2e32384657aee21641a1a6d0
Parents: 50ae1a6
Author: Jing Zhao ji...@apache.org
Authored: Mon Dec 22 23:19:20 2014 -0800
Committer: Jing Zhao ji...@apache.org
Committed: Mon Dec 22 23:19:20 2014 -0800
--
.../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirMkdirOp.java | 325 ++-
.../hdfs/server/namenode/FSDirRenameOp.java | 178 ++
.../hdfs/server/namenode/FSDirSymlinkOp.java| 32 +-
.../hdfs/server/namenode/FSDirectory.java | 143
.../hdfs/server/namenode/FSEditLogLoader.java | 37 ++-
.../hdfs/server/namenode/FSNamesystem.java | 35 +-
.../hdfs/server/namenode/INodesInPath.java | 64 ++--
.../snapshot/TestRenameWithSnapshots.java | 8 +-
10 files changed, 444 insertions(+), 384 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5caebbae/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index 1f5b863..4d738f7 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -577,7 +577,8 @@ public abstract class SymlinkBaseTest {
} catch (IOException e) {
// Expected
if (wrapper instanceof FileContextTestWrapper) {
-assertEquals(No AbstractFileSystem for scheme: null, e.getMessage());
+GenericTestUtils.assertExceptionContains(
+No AbstractFileSystem configured for scheme: null, e);
} else if (wrapper instanceof FileSystemTestWrapper) {
assertEquals(No FileSystem for scheme: null, e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5caebbae/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 66b3a8d..8525402 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -476,6 +476,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7530. Allow renaming of encryption zone roots. (Charles Lamb via wang)
+HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter.
+(jing9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5caebbae/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index 4ea77e6..f51427f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -17,9 +17,10 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.InvalidPathException;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsAction;
@@ -29,18 +30,19 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import java.io.IOException;
+import java.util.AbstractMap;
hadoop git commit: HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter. Contributed by Jing Zhao.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 cd8e78d50 - 9aa9455da
HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter.
Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9aa9455d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9aa9455d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9aa9455d
Branch: refs/heads/branch-2
Commit: 9aa9455dab5a60a678325edc9bab0abddfb3f326
Parents: cd8e78d
Author: Jing Zhao ji...@apache.org
Authored: Mon Dec 22 23:19:20 2014 -0800
Committer: Jing Zhao ji...@apache.org
Committed: Mon Dec 22 23:20:18 2014 -0800
--
.../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirMkdirOp.java | 325 ++-
.../hdfs/server/namenode/FSDirRenameOp.java | 178 ++
.../hdfs/server/namenode/FSDirSymlinkOp.java| 32 +-
.../hdfs/server/namenode/FSDirectory.java | 143
.../hdfs/server/namenode/FSEditLogLoader.java | 37 ++-
.../hdfs/server/namenode/FSNamesystem.java | 35 +-
.../hdfs/server/namenode/INodesInPath.java | 64 ++--
.../snapshot/TestRenameWithSnapshots.java | 8 +-
10 files changed, 444 insertions(+), 384 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9aa9455d/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
index ba2b72e..4c10de9 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/SymlinkBaseTest.java
@@ -581,7 +581,8 @@ public abstract class SymlinkBaseTest {
} catch (IOException e) {
// Expected
if (wrapper instanceof FileContextTestWrapper) {
-assertEquals(No AbstractFileSystem for scheme: null, e.getMessage());
+GenericTestUtils.assertExceptionContains(
+No AbstractFileSystem configured for scheme: null, e);
} else if (wrapper instanceof FileSystemTestWrapper) {
assertEquals(No FileSystem for scheme: null, e.getMessage());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9aa9455d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 360d38b..3380ba5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -219,6 +219,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7530. Allow renaming of encryption zone roots. (Charles Lamb via wang)
+HDFS-7484. Make FSDirectory#addINode take existing INodes as its parameter.
+(jing9)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9aa9455d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index 4ea77e6..f51427f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -17,9 +17,10 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import com.google.common.base.Preconditions;
+import org.apache.commons.io.Charsets;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.InvalidPathException;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsAction;
@@ -29,18 +30,19 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
-import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import java.io.IOException;
+import java.util.AbstractMap;
