[06/13] hadoop git commit: HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)

2015-09-10 Thread vinodkv 
HDFS-8219. setStoragePolicy with folder behavior is different after cluster 
restart. (surendra singh lilhore via Xiaoyu Yao)

(cherry picked from commit 0100b155019496d077f958904de7d385697d65d9)
(cherry picked from commit e68e8b3b5cff85bfd8bb5b00b9033f63577856d6)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3562b4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3562b4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3562b4e

Branch: refs/heads/branch-2.7
Commit: d3562b4e1dd1c163a94dd33993109cf3aba23ef1
Parents: 4ea42b8
Author: Xiaoyu Yao 
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) 

Committed: Thu Sep 10 11:56:22 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 44 
 3 files changed, 48 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3562b4e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8758cba..268ebb0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -47,6 +47,9 @@ Release 2.7.2 - UNRELEASED
 HDFS-9033. dfsadmin -metasave prints "NaN" for cache used%.
 (Brahma Reddy Battula via aajisaka)
 
+HDFS-8219. setStoragePolicy with folder behavior is different after 
cluster restart.
+(surendra singh lilhore via Xiaoyu Yao)
+
 Release 2.7.1 - 2015-07-06
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3562b4e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 57c5832..d8f18e8 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -740,7 +740,7 @@ public class FSEditLog implements LogsPurgeable {
   .setClientMachine(
   newNode.getFileUnderConstructionFeature().getClientMachine())
   .setOverwrite(overwrite)
-  .setStoragePolicyId(newNode.getStoragePolicyID());
+  .setStoragePolicyId(newNode.getLocalStoragePolicyID());
 
 AclFeature f = newNode.getAclFeature();
 if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3562b4e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index ff7b743..fe69c32 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
@@ -1247,4 +1248,47 @@ public class TestBlockStoragePolicy {
   Assert.assertEquals(StorageType.ARCHIVE, i.next().getKey());
 }
   }
+
+  public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+//HDFS8219
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(REPLICATION)
+.storageTypes(
+new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+.build();
+cluster.waitActive();
+final DistributedFileSystem fs = cluster.getFileSystem();
+try {
+  final String file = "/testScheduleWithinSameNode/file";
+  Path dir = new Path("/testScheduleWithinSameNode");
+  fs.mkdirs(dir);
+  // 2. Set Dir policy
+  fs.setStoragePolicy(dir, "COLD");
+  // 3. Create file
+  final FSDataOutputStream out = fs.create(new Path(file));
+  out.writeChars("testScheduleWithinSameNode");
+  out.close();
+  // 4. Set Dir policy
+

hadoop git commit: HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)

2015-09-02 Thread vinodkv 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2.6.1 41601eae6 -> 08006b84f


HDFS-8219. setStoragePolicy with folder behavior is different after cluster 
restart. (surendra singh lilhore via Xiaoyu Yao)

(cherry picked from commit 0100b155019496d077f958904de7d385697d65d9)
(cherry picked from commit e68e8b3b5cff85bfd8bb5b00b9033f63577856d6)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java

(cherry picked from commit b054cb68fa0fc6d1e9e77ac84575731e7d1ec0c7)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08006b84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08006b84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08006b84

Branch: refs/heads/branch-2.6.1
Commit: 08006b84f8cb6119bd17f6cfc8da2598e24346ca
Parents: 41601ea
Author: Xiaoyu Yao 
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Vinod Kumar Vavilapalli 
Committed: Wed Sep 2 15:31:46 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 43 
 3 files changed, 47 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/08006b84/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b4adfd1..4ccbd5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -129,6 +129,9 @@ Release 2.6.1 - UNRELEASED
 HDFS-8070. Pre-HDFS-7915 DFSClient cannot use short circuit on
 post-HDFS-7915 DataNode (cmccabe)
 
+HDFS-8219. setStoragePolicy with folder behavior is different after 
cluster restart.
+(surendra singh lilhore via Xiaoyu Yao)
+
 Release 2.6.0 - 2014-11-18
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/08006b84/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 20aaf07..0154ed9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -721,7 +721,7 @@ public class FSEditLog implements LogsPurgeable {
   .setClientMachine(
   newNode.getFileUnderConstructionFeature().getClientMachine())
   .setOverwrite(overwrite)
-  .setStoragePolicyId(newNode.getStoragePolicyID());
+  .setStoragePolicyId(newNode.getLocalStoragePolicyID());
 
 AclFeature f = newNode.getAclFeature();
 if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/08006b84/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index d053a79..8ac25db 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.*;
@@ -1173,4 +1174,46 @@ public class TestBlockStoragePolicy {
   cluster.shutdown();
 }
   }
+
+  @Test
+  public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+//HDFS8219
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(REPLICATION)
+.storageTypes(
+new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+.build();
+cluster.waitActive();
+final DistributedFileSystem fs = cluster.getFileSystem();
+try {
+  final String file = "/testScheduleWithinSameNode/file";
+  Path dir = new Path("/testScheduleWithinSameNode");
+  fs.mkdirs(dir);
+  // 2. Set Dir policy
+

[33/43] hadoop git commit: HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)

2015-08-14 Thread sjlee 
HDFS-8219. setStoragePolicy with folder behavior is different after cluster 
restart. (surendra singh lilhore via Xiaoyu Yao)

(cherry picked from commit 0100b155019496d077f958904de7d385697d65d9)
(cherry picked from commit e68e8b3b5cff85bfd8bb5b00b9033f63577856d6)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b054cb68
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b054cb68
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b054cb68

Branch: refs/heads/sjlee/hdfs-merge
Commit: b054cb68fa0fc6d1e9e77ac84575731e7d1ec0c7
Parents: b4e227e
Author: Xiaoyu Yao x...@apache.org
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Sangjin Lee sj...@apache.org
Committed: Thu Aug 13 16:05:28 2015 -0700

--
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 43 
 2 files changed, 44 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b054cb68/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 20aaf07..0154ed9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -721,7 +721,7 @@ public class FSEditLog implements LogsPurgeable {
   .setClientMachine(
   newNode.getFileUnderConstructionFeature().getClientMachine())
   .setOverwrite(overwrite)
-  .setStoragePolicyId(newNode.getStoragePolicyID());
+  .setStoragePolicyId(newNode.getLocalStoragePolicyID());
 
 AclFeature f = newNode.getAclFeature();
 if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b054cb68/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index d053a79..8ac25db 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hdfs.protocol.*;
@@ -1173,4 +1174,46 @@ public class TestBlockStoragePolicy {
   cluster.shutdown();
 }
   }
+
+  @Test
+  public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+//HDFS8219
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(REPLICATION)
+.storageTypes(
+new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+.build();
+cluster.waitActive();
+final DistributedFileSystem fs = cluster.getFileSystem();
+try {
+  final String file = /testScheduleWithinSameNode/file;
+  Path dir = new Path(/testScheduleWithinSameNode);
+  fs.mkdirs(dir);
+  // 2. Set Dir policy
+  fs.setStoragePolicy(dir, COLD);
+  // 3. Create file
+  final FSDataOutputStream out = fs.create(new Path(file));
+  out.writeChars(testScheduleWithinSameNode);
+  out.close();
+  // 4. Set Dir policy
+  fs.setStoragePolicy(dir, HOT);
+  HdfsFileStatus status = fs.getClient().getFileInfo(file);
+  // 5. get file policy, it should be parent policy.
+  Assert
+  .assertTrue(
+  File storage policy should be HOT,
+  status.getStoragePolicy() == HOT);
+  // 6. restart NameNode for reloading edits logs.
+  cluster.restartNameNode(true);
+  // 7. get file policy, it should be parent policy.
+  status = fs.getClient().getFileInfo(file);
+  Assert
+  .assertTrue(
+  File storage policy should be HOT,
+  status.getStoragePolicy() == HOT);
+
+} finally {
+  cluster.shutdown();
+}
+  }
 }

[15/17] hadoop git commit: HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)

2015-05-05 Thread jitendra 
HDFS-8219. setStoragePolicy with folder behavior is different after cluster 
restart. (surendra singh lilhore via Xiaoyu Yao)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0100b155
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0100b155
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0100b155

Branch: refs/heads/HDFS-7240
Commit: 0100b155019496d077f958904de7d385697d65d9
Parents: ffce9a3
Author: Xiaoyu Yao x...@apache.org
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Xiaoyu Yao x...@apache.org
Committed: Tue May 5 13:41:14 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 45 
 3 files changed, 49 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0100b155/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 01de9b1..cc6758f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8290. WebHDFS calls before namesystem initialization can cause
 NullPointerException. (cnauroth)
 
+HDFS-8219. setStoragePolicy with folder behavior is different after 
cluster restart.
+(surendra singh lilhore via Xiaoyu Yao)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0100b155/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 28e150c..83e52bc 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -739,7 +739,7 @@ public class FSEditLog implements LogsPurgeable {
   .setClientMachine(
   newNode.getFileUnderConstructionFeature().getClientMachine())
   .setOverwrite(overwrite)
-  .setStoragePolicyId(newNode.getStoragePolicyID());
+  .setStoragePolicyId(newNode.getLocalStoragePolicyID());
 
 AclFeature f = newNode.getAclFeature();
 if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0100b155/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 9621dc8..5e3b55f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
@@ -1172,4 +1173,48 @@ public class TestBlockStoragePolicy {
   cluster.shutdown();
 }
   }
+
+  @Test
+  public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+//HDFS8219
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(REPLICATION)
+.storageTypes(
+new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+.build();
+cluster.waitActive();
+final DistributedFileSystem fs = cluster.getFileSystem();
+try {
+  final String file = /testScheduleWithinSameNode/file;
+  Path dir = new Path(/testScheduleWithinSameNode);
+  fs.mkdirs(dir);
+  // 2. Set Dir policy
+  fs.setStoragePolicy(dir, COLD);
+  // 3. Create file
+  final FSDataOutputStream out = fs.create(new Path(file));
+  out.writeChars(testScheduleWithinSameNode);
+  out.close();
+  // 4. Set Dir policy
+  fs.setStoragePolicy(dir, HOT);
+  HdfsFileStatus status = fs.getClient().getFileInfo(file);
+  // 5. get file policy, it should be parent policy.
+  Assert
+  .assertTrue(
+  File storage policy

hadoop git commit: HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)

2015-05-05 Thread xyao 
Repository: hadoop
Updated Branches:
  refs/heads/trunk ffce9a341 - 0100b1550


HDFS-8219. setStoragePolicy with folder behavior is different after cluster 
restart. (surendra singh lilhore via Xiaoyu Yao)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0100b155
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0100b155
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0100b155

Branch: refs/heads/trunk
Commit: 0100b155019496d077f958904de7d385697d65d9
Parents: ffce9a3
Author: Xiaoyu Yao x...@apache.org
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Xiaoyu Yao x...@apache.org
Committed: Tue May 5 13:41:14 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 45 
 3 files changed, 49 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0100b155/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 01de9b1..cc6758f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -626,6 +626,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8290. WebHDFS calls before namesystem initialization can cause
 NullPointerException. (cnauroth)
 
+HDFS-8219. setStoragePolicy with folder behavior is different after 
cluster restart.
+(surendra singh lilhore via Xiaoyu Yao)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0100b155/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 28e150c..83e52bc 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -739,7 +739,7 @@ public class FSEditLog implements LogsPurgeable {
   .setClientMachine(
   newNode.getFileUnderConstructionFeature().getClientMachine())
   .setOverwrite(overwrite)
-  .setStoragePolicyId(newNode.getStoragePolicyID());
+  .setStoragePolicyId(newNode.getLocalStoragePolicyID());
 
 AclFeature f = newNode.getAclFeature();
 if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0100b155/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 9621dc8..5e3b55f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
@@ -1172,4 +1173,48 @@ public class TestBlockStoragePolicy {
   cluster.shutdown();
 }
   }
+
+  @Test
+  public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+//HDFS8219
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(REPLICATION)
+.storageTypes(
+new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+.build();
+cluster.waitActive();
+final DistributedFileSystem fs = cluster.getFileSystem();
+try {
+  final String file = /testScheduleWithinSameNode/file;
+  Path dir = new Path(/testScheduleWithinSameNode);
+  fs.mkdirs(dir);
+  // 2. Set Dir policy
+  fs.setStoragePolicy(dir, COLD);
+  // 3. Create file
+  final FSDataOutputStream out = fs.create(new Path(file));
+  out.writeChars(testScheduleWithinSameNode);
+  out.close();
+  // 4. Set Dir policy
+  fs.setStoragePolicy(dir, HOT);
+  HdfsFileStatus status = fs.getClient().getFileInfo(file);
+  // 5. get file policy, it should be parent

hadoop git commit: HDFS-8219. setStoragePolicy with folder behavior is different after cluster restart. (surendra singh lilhore via Xiaoyu Yao)

2015-05-05 Thread xyao 
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 eb0c6d2ee - e68e8b3b5


HDFS-8219. setStoragePolicy with folder behavior is different after cluster 
restart. (surendra singh lilhore via Xiaoyu Yao)

(cherry picked from commit 0100b155019496d077f958904de7d385697d65d9)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e68e8b3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e68e8b3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e68e8b3b

Branch: refs/heads/branch-2
Commit: e68e8b3b5cff85bfd8bb5b00b9033f63577856d6
Parents: eb0c6d2
Author: Xiaoyu Yao x...@apache.org
Authored: Tue May 5 13:41:14 2015 -0700
Committer: Xiaoyu Yao x...@apache.org
Committed: Tue May 5 13:42:22 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../hadoop/hdfs/server/namenode/FSEditLog.java  |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 45 
 3 files changed, 49 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e68e8b3b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 75d0871..0aaba2f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -299,6 +299,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8290. WebHDFS calls before namesystem initialization can cause
 NullPointerException. (cnauroth)
 
+HDFS-8219. setStoragePolicy with folder behavior is different after 
cluster restart.
+(surendra singh lilhore via Xiaoyu Yao)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e68e8b3b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 9ec4902..0e097c7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -740,7 +740,7 @@ public class FSEditLog implements LogsPurgeable {
   .setClientMachine(
   newNode.getFileUnderConstructionFeature().getClientMachine())
   .setOverwrite(overwrite)
-  .setStoragePolicyId(newNode.getStoragePolicyID());
+  .setStoragePolicyId(newNode.getLocalStoragePolicyID());
 
 AclFeature f = newNode.getAclFeature();
 if (f != null) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e68e8b3b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 821d2db..4c3e7f0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -26,6 +26,7 @@ import java.util.*;
 
 import com.google.common.collect.Lists;
 import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.StorageType;
@@ -1177,4 +1178,48 @@ public class TestBlockStoragePolicy {
   cluster.shutdown();
 }
   }
+
+  @Test
+  public void testGetFileStoragePolicyAfterRestartNN() throws Exception {
+//HDFS8219
+final MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf)
+.numDataNodes(REPLICATION)
+.storageTypes(
+new StorageType[] {StorageType.DISK, StorageType.ARCHIVE})
+.build();
+cluster.waitActive();
+final DistributedFileSystem fs = cluster.getFileSystem();
+try {
+  final String file = /testScheduleWithinSameNode/file;
+  Path dir = new Path(/testScheduleWithinSameNode);
+  fs.mkdirs(dir);
+  // 2. Set Dir policy
+  fs.setStoragePolicy(dir, COLD);
+  // 3. Create file
+  final FSDataOutputStream out = fs.create(new Path(file));
+  out.writeChars(testScheduleWithinSameNode);
+  out.close();
+  // 4. Set Dir policy
+  fs.setStoragePolicy(dir, HOT);
+  HdfsFileStatus status =