[24/50] [abbrv] hadoop git commit: HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file which has snapshots.

[asuresh]

HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file 
which has snapshots.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52b894db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52b894db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52b894db

Branch: refs/heads/YARN-5972
Commit: 52b894db33bc68b46eec5cdf2735dfcf4030853a
Parents: b0b535d
Author: Xiao Chen 
Authored: Thu Sep 7 15:54:52 2017 -0700
Committer: Xiao Chen 
Committed: Thu Sep 7 16:30:12 2017 -0700

--
 .../hdfs/server/namenode/FSNamesystem.java  |  1 +
 .../hdfs/server/namenode/LeaseManager.java  |  6 ++
 .../hdfs/server/namenode/TestDeleteRace.java| 84 
 3 files changed, 91 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c30999b..dedb11e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4993,6 +4993,7 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   }
 
   boolean isFileDeleted(INodeFile file) {
+assert hasReadLock();
 // Not in the inodeMap or in the snapshot but marked deleted.
 if (dir.getInode(file.getId()) == null) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 35ec063..45699cb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -557,6 +557,12 @@ public class LeaseManager {
   if (!p.startsWith("/")) {
 throw new IOException("Invalid path in the lease " + p);
   }
+  final INodeFile lastINode = iip.getLastINode().asFile();
+  if (fsnamesystem.isFileDeleted(lastINode)) {
+// INode referred by the lease could have been deleted.
+removeLease(lastINode.getId());
+continue;
+  }
   boolean completed = false;
   try {
 completed = fsnamesystem.internalReleaseLease(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 133a18e..a13574f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.FileNotFoundException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,16 +51,21 @@ import 
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.test.GenericTestUtils;
 import 

[04/45] hadoop git commit: HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file which has snapshots.

[inigoiri]

HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file 
which has snapshots.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52b894db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52b894db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52b894db

Branch: refs/heads/HDFS-10467
Commit: 52b894db33bc68b46eec5cdf2735dfcf4030853a
Parents: b0b535d
Author: Xiao Chen 
Authored: Thu Sep 7 15:54:52 2017 -0700
Committer: Xiao Chen 
Committed: Thu Sep 7 16:30:12 2017 -0700

--
 .../hdfs/server/namenode/FSNamesystem.java  |  1 +
 .../hdfs/server/namenode/LeaseManager.java  |  6 ++
 .../hdfs/server/namenode/TestDeleteRace.java| 84 
 3 files changed, 91 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c30999b..dedb11e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4993,6 +4993,7 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   }
 
   boolean isFileDeleted(INodeFile file) {
+assert hasReadLock();
 // Not in the inodeMap or in the snapshot but marked deleted.
 if (dir.getInode(file.getId()) == null) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 35ec063..45699cb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -557,6 +557,12 @@ public class LeaseManager {
   if (!p.startsWith("/")) {
 throw new IOException("Invalid path in the lease " + p);
   }
+  final INodeFile lastINode = iip.getLastINode().asFile();
+  if (fsnamesystem.isFileDeleted(lastINode)) {
+// INode referred by the lease could have been deleted.
+removeLease(lastINode.getId());
+continue;
+  }
   boolean completed = false;
   try {
 completed = fsnamesystem.internalReleaseLease(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 133a18e..a13574f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.FileNotFoundException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,16 +51,21 @@ import 
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.net.Node;
 import org.apache.hadoop.test.GenericTestUtils;
 import 

hadoop git commit: HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file which has snapshots.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/branch-3.0 728793239 -> 9a3334ec0


HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file 
which has snapshots.

(cherry picked from commit 3964b131f3f0131a663377685950d1b7bd3fe63d)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a3334ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a3334ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a3334ec

Branch: refs/heads/branch-3.0
Commit: 9a3334ec0fcebdbeabe796009a14a53905bfea9b
Parents: 7287932
Author: Xiao Chen 
Authored: Thu Sep 7 15:54:52 2017 -0700
Committer: Xiao Chen 
Committed: Thu Sep 7 16:30:26 2017 -0700

--
 .../hdfs/server/namenode/FSNamesystem.java  |  1 +
 .../hdfs/server/namenode/LeaseManager.java  |  6 ++
 .../hdfs/server/namenode/TestDeleteRace.java| 84 
 3 files changed, 91 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3334ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c30999b..dedb11e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4993,6 +4993,7 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   }
 
   boolean isFileDeleted(INodeFile file) {
+assert hasReadLock();
 // Not in the inodeMap or in the snapshot but marked deleted.
 if (dir.getInode(file.getId()) == null) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3334ec/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 35ec063..45699cb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -557,6 +557,12 @@ public class LeaseManager {
   if (!p.startsWith("/")) {
 throw new IOException("Invalid path in the lease " + p);
   }
+  final INodeFile lastINode = iip.getLastINode().asFile();
+  if (fsnamesystem.isFileDeleted(lastINode)) {
+// INode referred by the lease could have been deleted.
+removeLease(lastINode.getId());
+continue;
+  }
   boolean completed = false;
   try {
 completed = fsnamesystem.internalReleaseLease(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3334ec/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 133a18e..a13574f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.FileNotFoundException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,16 +51,21 @@ import 
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import 

hadoop git commit: HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file which has snapshots.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 c6db50503 -> 69d7b1ae5


HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file 
which has snapshots.

(cherry picked from commit 3964b131f3f0131a663377685950d1b7bd3fe63d)
(cherry picked from commit 8b7d597d3a8d728aa05302ccd769af84a1c9ec0f)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69d7b1ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69d7b1ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69d7b1ae

Branch: refs/heads/branch-2.8
Commit: 69d7b1ae506f4e65334d7af965b96081286bc70e
Parents: c6db505
Author: Xiao Chen 
Authored: Thu Sep 7 15:54:52 2017 -0700
Committer: Xiao Chen 
Committed: Thu Sep 7 16:30:53 2017 -0700

--
 .../hdfs/server/namenode/FSNamesystem.java  |  3 +-
 .../hdfs/server/namenode/LeaseManager.java  |  6 ++
 .../hdfs/server/namenode/TestDeleteRace.java| 84 
 3 files changed, 92 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d7b1ae/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 23f7f54..1db1711 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -5490,7 +5490,8 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
 return blockId;
   }
 
-  private boolean isFileDeleted(INodeFile file) {
+  boolean isFileDeleted(INodeFile file) {
+assert hasReadLock();
 // Not in the inodeMap or in the snapshot but marked deleted.
 if (dir.getInode(file.getId()) == null) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d7b1ae/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 99edc44..adebb00 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -457,6 +457,12 @@ public class LeaseManager {
   if (!p.startsWith("/")) {
 throw new IOException("Invalid path in the lease " + p);
   }
+  final INodeFile lastINode = iip.getLastINode().asFile();
+  if (fsnamesystem.isFileDeleted(lastINode)) {
+// INode referred by the lease could have been deleted.
+removeLease(lastINode.getId());
+continue;
+  }
   boolean completed = false;
   try {
 completed = fsnamesystem.internalReleaseLease(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d7b1ae/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 133a18e..a13574f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.FileNotFoundException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,16 +51,21 @@ import 
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import 

hadoop git commit: HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file which has snapshots.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 c004d57d4 -> c6a1e3fac


HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file 
which has snapshots.

(cherry picked from commit 3964b131f3f0131a663377685950d1b7bd3fe63d)
(cherry picked from commit 8b7d597d3a8d728aa05302ccd769af84a1c9ec0f)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6a1e3fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6a1e3fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6a1e3fa

Branch: refs/heads/branch-2
Commit: c6a1e3facbe568a3706a2a2dbe385157db3c7cfa
Parents: c004d57
Author: Xiao Chen 
Authored: Thu Sep 7 15:54:52 2017 -0700
Committer: Xiao Chen 
Committed: Thu Sep 7 16:30:38 2017 -0700

--
 .../hdfs/server/namenode/FSNamesystem.java  |  1 +
 .../hdfs/server/namenode/LeaseManager.java  |  6 ++
 .../hdfs/server/namenode/TestDeleteRace.java| 84 
 3 files changed, 91 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6a1e3fa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 59f38f4..5b55d82 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4794,6 +4794,7 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   }
 
   boolean isFileDeleted(INodeFile file) {
+assert hasReadLock();
 // Not in the inodeMap or in the snapshot but marked deleted.
 if (dir.getInode(file.getId()) == null) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6a1e3fa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 30c48dd..6b41fc9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -562,6 +562,12 @@ public class LeaseManager {
   if (!p.startsWith("/")) {
 throw new IOException("Invalid path in the lease " + p);
   }
+  final INodeFile lastINode = iip.getLastINode().asFile();
+  if (fsnamesystem.isFileDeleted(lastINode)) {
+// INode referred by the lease could have been deleted.
+removeLease(lastINode.getId());
+continue;
+  }
   boolean completed = false;
   try {
 completed = fsnamesystem.internalReleaseLease(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6a1e3fa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 133a18e..a13574f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.FileNotFoundException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,16 +51,21 @@ import 
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import 

hadoop git commit: HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file which has snapshots.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/trunk b0b535d9d -> 52b894db3


HDFS-12369. Edit log corruption due to hard lease recovery of not-closed file 
which has snapshots.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52b894db
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52b894db
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52b894db

Branch: refs/heads/trunk
Commit: 52b894db33bc68b46eec5cdf2735dfcf4030853a
Parents: b0b535d
Author: Xiao Chen 
Authored: Thu Sep 7 15:54:52 2017 -0700
Committer: Xiao Chen 
Committed: Thu Sep 7 16:30:12 2017 -0700

--
 .../hdfs/server/namenode/FSNamesystem.java  |  1 +
 .../hdfs/server/namenode/LeaseManager.java  |  6 ++
 .../hdfs/server/namenode/TestDeleteRace.java| 84 
 3 files changed, 91 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index c30999b..dedb11e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4993,6 +4993,7 @@ public class FSNamesystem implements Namesystem, 
FSNamesystemMBean,
   }
 
   boolean isFileDeleted(INodeFile file) {
+assert hasReadLock();
 // Not in the inodeMap or in the snapshot but marked deleted.
 if (dir.getInode(file.getId()) == null) {
   return true;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index 35ec063..45699cb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -557,6 +557,12 @@ public class LeaseManager {
   if (!p.startsWith("/")) {
 throw new IOException("Invalid path in the lease " + p);
   }
+  final INodeFile lastINode = iip.getLastINode().asFile();
+  if (fsnamesystem.isFileDeleted(lastINode)) {
+// INode referred by the lease could have been deleted.
+removeLease(lastINode.getId());
+continue;
+  }
   boolean completed = false;
   try {
 completed = fsnamesystem.internalReleaseLease(

http://git-wip-us.apache.org/repos/asf/hadoop/blob/52b894db/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
index 133a18e..a13574f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeleteRace.java
@@ -20,11 +20,13 @@ package org.apache.hadoop.hdfs.server.namenode;
 import java.io.FileNotFoundException;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import java.util.Comparator;
 import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.TreeSet;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -49,16 +51,21 @@ import 
org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
 import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
 import org.apache.hadoop.hdfs.server.datanode.DataNode;
 import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.namenode.LeaseManager.Lease;
 import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
 import org.apache.hadoop.io.IOUtils;
 import