[33/50] [abbrv] hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[vvasudev]

HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132deb4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132deb4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132deb4c

Branch: refs/heads/YARN-3926
Commit: 132deb4cacc413a85a6af2f390ec79a76c91961c
Parents: 12aa184
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Thu Jul 21 16:41:02 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 +++-
 .../datanode/TestDataNodeHotSwapVolumes.java| 23 +++-
 2 files changed, 35 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1cd2dee..ac55397 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -634,7 +634,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -646,6 +646,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -659,6 +665,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 659806b..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -82,7 +83,6 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.timeout;
@@ -256,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1, 1);
+DataNode dn = cluster.getDataNodes().get(0);
+Configuration conf = 

[39/50] [abbrv] hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[vvasudev]

HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132deb4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132deb4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132deb4c

Branch: refs/heads/YARN-4757
Commit: 132deb4cacc413a85a6af2f390ec79a76c91961c
Parents: 12aa184
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Thu Jul 21 16:41:02 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 +++-
 .../datanode/TestDataNodeHotSwapVolumes.java| 23 +++-
 2 files changed, 35 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1cd2dee..ac55397 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -634,7 +634,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -646,6 +646,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -659,6 +665,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 659806b..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -82,7 +83,6 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.timeout;
@@ -256,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1, 1);
+DataNode dn = cluster.getDataNodes().get(0);
+Configuration conf = 

hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.8 c96cb3fd4 -> 8559442ed


HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.

(cherry picked from commit 132deb4cacc413a85a6af2f390ec79a76c91961c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8559442e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8559442e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8559442e

Branch: refs/heads/branch-2.8
Commit: 8559442ed15c2e8a9aa6934ec35a1f4d1157e1a1
Parents: c96cb3f
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Fri Jul 22 15:04:21 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 -
 .../datanode/TestDataNodeHotSwapVolumes.java| 22 
 2 files changed, 35 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8559442e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 5d6514e..cd8212f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -625,7 +625,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -637,6 +637,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -650,6 +656,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8559442e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 0328e10..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -255,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1, 1);
+DataNode dn = cluster.getDataNodes().get(0);
+Configuration conf = dn.getConf();
+List oldLocations = DataNode.getStorageLocations(conf);
+
+// Change storage type of an existing StorageLocation
+String newLoc = String.format("[%s]%s", StorageType.SSD,
+

hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2f1e35291 -> 1a115ea31


HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.

(cherry picked from commit 132deb4cacc413a85a6af2f390ec79a76c91961c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a115ea3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a115ea3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a115ea3

Branch: refs/heads/branch-2
Commit: 1a115ea317d3381a96fd991d7ba48a896bed0bcb
Parents: 2f1e352
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Fri Jul 22 15:04:17 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 -
 .../datanode/TestDataNodeHotSwapVolumes.java| 22 
 2 files changed, 35 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a115ea3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index f9298f1..38b73e9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -630,7 +630,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -642,6 +642,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -655,6 +661,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a115ea3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 0328e10..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -255,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1, 1);
+DataNode dn = cluster.getDataNodes().get(0);
+Configuration conf = dn.getConf();
+List oldLocations = DataNode.getStorageLocations(conf);
+
+// Change storage type of an existing StorageLocation
+String newLoc = String.format("[%s]%s", StorageType.SSD,
+

hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/b28 [created] 8559442ed


HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.

(cherry picked from commit 132deb4cacc413a85a6af2f390ec79a76c91961c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8559442e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8559442e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8559442e

Branch: refs/heads/b28
Commit: 8559442ed15c2e8a9aa6934ec35a1f4d1157e1a1
Parents: c96cb3f
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Fri Jul 22 15:04:21 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 -
 .../datanode/TestDataNodeHotSwapVolumes.java| 22 
 2 files changed, 35 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8559442e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 5d6514e..cd8212f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -625,7 +625,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -637,6 +637,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -650,6 +656,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8559442e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 0328e10..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -255,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1, 1);
+DataNode dn = cluster.getDataNodes().get(0);
+Configuration conf = dn.getConf();
+List oldLocations = DataNode.getStorageLocations(conf);
+
+// Change storage type of an existing StorageLocation
+String newLoc = String.format("[%s]%s", StorageType.SSD,
+

hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/b2 [created] 1a115ea31


HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.

(cherry picked from commit 132deb4cacc413a85a6af2f390ec79a76c91961c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a115ea3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a115ea3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a115ea3

Branch: refs/heads/b2
Commit: 1a115ea317d3381a96fd991d7ba48a896bed0bcb
Parents: 2f1e352
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Fri Jul 22 15:04:17 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 -
 .../datanode/TestDataNodeHotSwapVolumes.java| 22 
 2 files changed, 35 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a115ea3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index f9298f1..38b73e9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -630,7 +630,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -642,6 +642,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -655,6 +661,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a115ea3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 0328e10..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -255,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1, 1);
+DataNode dn = cluster.getDataNodes().get(0);
+Configuration conf = dn.getConf();
+List oldLocations = DataNode.getStorageLocations(conf);
+
+// Change storage type of an existing StorageLocation
+String newLoc = String.format("[%s]%s", StorageType.SSD,
+

hadoop git commit: HDFS-10225. DataNode hot swap drives should disallow storage type changes. Contributed by Lei (Eddy) Xu.

[xiao]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 12aa18447 -> 132deb4ca


HDFS-10225. DataNode hot swap drives should disallow storage type changes. 
Contributed by Lei (Eddy) Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/132deb4c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/132deb4c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/132deb4c

Branch: refs/heads/trunk
Commit: 132deb4cacc413a85a6af2f390ec79a76c91961c
Parents: 12aa184
Author: Xiao Chen 
Authored: Thu Jul 21 16:41:02 2016 -0700
Committer: Xiao Chen 
Committed: Thu Jul 21 16:41:02 2016 -0700

--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 +++-
 .../datanode/TestDataNodeHotSwapVolumes.java| 23 +++-
 2 files changed, 35 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1cd2dee..ac55397 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -634,7 +634,7 @@ public class DataNode extends ReconfigurableBase
* @param newVolumes a comma separated string that specifies the data 
volumes.
* @return changed volumes.
* @throws IOException if none of the directories are specified in the
-   * configuration.
+   * configuration, or the storage type of a directory is changed.
*/
   @VisibleForTesting
   ChangedVolumes parseChangedVolumes(String newVolumes) throws IOException {
@@ -646,6 +646,12 @@ public class DataNode extends ReconfigurableBase
   throw new IOException("No directory is specified.");
 }
 
+// Use the existing StorageLocation to detect storage type changes.
+Map existingLocations = new HashMap<>();
+for (StorageLocation loc : getStorageLocations(this.conf)) {
+  existingLocations.put(loc.getFile().getCanonicalPath(), loc);
+}
+
 ChangedVolumes results = new ChangedVolumes();
 results.newLocations.addAll(locations);
 
@@ -659,6 +665,12 @@ public class DataNode extends ReconfigurableBase
 if (location.getFile().getCanonicalPath().equals(
 dir.getRoot().getCanonicalPath())) {
   sl.remove();
+  StorageLocation old = existingLocations.get(
+  location.getFile().getCanonicalPath());
+  if (old != null &&
+  old.getStorageType() != location.getStorageType()) {
+throw new IOException("Changing storage type is not allowed.");
+  }
   results.unchangedLocations.add(location);
   found = true;
   break;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/132deb4c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
index 659806b..c03b02b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
 import org.apache.hadoop.fs.FileStatus;
 import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.StorageType;
 import org.apache.hadoop.hdfs.BlockMissingException;
 import org.apache.hadoop.hdfs.DFSConfigKeys;
 import org.apache.hadoop.hdfs.DFSTestUtil;
@@ -82,7 +83,6 @@ import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
 import static org.junit.Assume.assumeTrue;
 import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyBoolean;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.timeout;
@@ -256,6 +256,27 @@ public class TestDataNodeHotSwapVolumes {
 }
   }
 
+  @Test
+  public void testParseStorageTypeChanges() throws IOException {
+startDFSCluster(1,