hadoop git commit: HDFS-12191. Provide option to not capture the accessTime change of a file to snapshot if no other modification has been done to this file. Contributed by Yongjun Zhang.

2017-08-29 Thread yjzhangal 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 036cbd745 -> 870ef0b2f


HDFS-12191. Provide option to not capture the accessTime change of a file to 
snapshot if no other modification has been done to this file. Contributed by 
Yongjun Zhang.

(cherry picked from commit cf93d60d3f032000e5b78a08d320793d78799f3d)

Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSDirAttrOp.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/870ef0b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/870ef0b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/870ef0b2

Branch: refs/heads/branch-2
Commit: 870ef0b2fd26c9f748e9b309cf1a5235e31fc8ab
Parents: 036cbd7
Author: Yongjun Zhang 
Authored: Tue Aug 29 16:57:03 2017 -0700
Committer: Yongjun Zhang 
Committed: Tue Aug 29 21:57:25 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   3 +
 .../hdfs/server/namenode/FSDirAttrOp.java   |   4 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   4 +-
 .../hadoop/hdfs/server/namenode/INode.java  |   7 +-
 .../namenode/snapshot/SnapshotManager.java  |  28 
 .../src/main/resources/hdfs-default.xml |  44 +++--
 .../hdfs/server/namenode/TestFSDirAttrOp.java   |   7 +
 .../snapshot/TestSnapshotDiffReport.java| 163 ++-
 8 files changed, 233 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/870ef0b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index d5192c2..185c882 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -331,6 +331,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
   HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT;
 
+  public static final String 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE = 
"dfs.namenode.snapshot.skip.capture.accesstime-only-change";
+  public static final boolean 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE_DEFAULT = false;
+
   // Whether to enable datanode's stale state detection and usage for reads
   public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = 
"dfs.namenode.avoid.read.stale.datanode";
   public static final boolean 
DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/870ef0b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 9e714af..478d5ac 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -493,7 +493,9 @@ public class FSDirAttrOp {
 // then no need to store access time
 if (atime != -1 && (status || force
 || atime > inode.getAccessTime() + fsd.getAccessTimePrecision())) {
-  inode.setAccessTime(atime, latest);
+  inode.setAccessTime(atime, latest,
+  fsd.getFSNamesystem().getSnapshotManager().
+  getSkipCaptureAccessTimeOnlyChange());
   status = true;
 }
 return status;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/870ef0b2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 616c2e3..d603559 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++

hadoop git commit: HDFS-12191. Provide option to not capture the accessTime change of a file to snapshot if no other modification has been done to this file. Contributed by Yongjun Zhang.

2017-08-29 Thread yjzhangal 
Repository: hadoop
Updated Branches:
  refs/heads/trunk 26fafc359 -> cf93d60d3


HDFS-12191. Provide option to not capture the accessTime change of a file to 
snapshot if no other modification has been done to this file. Contributed by 
Yongjun Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf93d60d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf93d60d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf93d60d

Branch: refs/heads/trunk
Commit: cf93d60d3f032000e5b78a08d320793d78799f3d
Parents: 26fafc3
Author: Yongjun Zhang 
Authored: Tue Aug 29 16:57:03 2017 -0700
Committer: Yongjun Zhang 
Committed: Tue Aug 29 16:57:03 2017 -0700

--
 .../org/apache/hadoop/hdfs/DFSConfigKeys.java   |   3 +
 .../hdfs/server/namenode/FSDirAttrOp.java   |   4 +-
 .../hdfs/server/namenode/FSEditLogLoader.java   |   4 +-
 .../hadoop/hdfs/server/namenode/INode.java  |   7 +-
 .../namenode/snapshot/SnapshotManager.java  |  28 
 .../src/main/resources/hdfs-default.xml |  44 +++--
 .../hdfs/server/namenode/TestFSDirAttrOp.java   |   7 +
 .../snapshot/TestSnapshotDiffReport.java| 163 ++-
 8 files changed, 233 insertions(+), 27 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf93d60d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b836ff8..17cabad 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -357,6 +357,9 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
   public static final boolean DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT =
   HdfsClientConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES_DEFAULT;
 
+  public static final String 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE = 
"dfs.namenode.snapshot.skip.capture.accesstime-only-change";
+  public static final boolean 
DFS_NAMENODE_SNAPSHOT_SKIP_CAPTURE_ACCESSTIME_ONLY_CHANGE_DEFAULT = false;
+
   // Whether to enable datanode's stale state detection and usage for reads
   public static final String DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_KEY = 
"dfs.namenode.avoid.read.stale.datanode";
   public static final boolean 
DFS_NAMENODE_AVOID_STALE_DATANODE_FOR_READ_DEFAULT = false;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf93d60d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index d4b24f5..0dfaa8e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -494,7 +494,9 @@ public class FSDirAttrOp {
 // then no need to store access time
 if (atime != -1 && (status || force
 || atime > inode.getAccessTime() + fsd.getAccessTimePrecision())) {
-  inode.setAccessTime(atime, latest);
+  inode.setAccessTime(atime, latest,
+  fsd.getFSNamesystem().getSnapshotManager().
+  getSkipCaptureAccessTimeOnlyChange());
   status = true;
 }
 return status;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf93d60d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 060bd59..bc62a7e 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -413,7 +413,7 @@ public class FSEditLogLoader {
   // update the block list.
   
   // Update the salient file attributes.
-  newFile.setAccessTime(addCloseOp.atime, Snapshot.CURRENT_STATE_ID);
+