[18/50] [abbrv] hadoop git commit: HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu.
HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eaa30c6 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eaa30c6 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eaa30c6 Branch: refs/heads/YARN-2928 Commit: 7eaa30c637ddfff94a84ea2771eacd9c69018521 Parents: 5934949 Author: Andrew Wang w...@apache.org Authored: Tue Mar 24 09:07:02 2015 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Thu Mar 26 23:29:45 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 4 ++ .../datanode/TestDataNodeHotSwapVolumes.java| 42 3 files changed, 48 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eaa30c6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ee9a5db..70be18a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1251,6 +1251,8 @@ Release 2.7.0 - UNRELEASED HDFS-7960. The full block report should prune zombie storages even if they're not empty. (cmccabe and Eddy Xu via wang) +HDFS-7961. Trigger full block report after hot swapping disk. (Eddy Xu via wang) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eaa30c6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index e9befb4..d94375e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -634,6 +634,10 @@ public class DataNode extends ReconfigurableBase conf.set(DFS_DATANODE_DATA_DIR_KEY, Joiner.on(,).join(effectiveVolumes)); dataDirs = getStorageLocations(conf); + + // Send a full block report to let NN acknowledge the volume changes. + triggerBlockReport(new BlockReportOptions.Factory() + .setIncremental(false).build()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eaa30c6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 2f51d45..f5772e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; @@ -59,6 +63,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.mockito.Mockito; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.hamcrest.CoreMatchers.anyOf; @@
[35/51] [abbrv] hadoop git commit: HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu.
HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6413d349 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6413d349 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6413d349 Branch: refs/heads/YARN-2139 Commit: 6413d34986f3399023426c89c9a0d401c9557716 Parents: 4170c99 Author: Andrew Wang w...@apache.org Authored: Tue Mar 24 09:07:02 2015 -0700 Committer: Andrew Wang w...@apache.org Committed: Tue Mar 24 09:07:02 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 4 ++ .../datanode/TestDataNodeHotSwapVolumes.java| 42 3 files changed, 48 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ee9a5db..70be18a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1251,6 +1251,8 @@ Release 2.7.0 - UNRELEASED HDFS-7960. The full block report should prune zombie storages even if they're not empty. (cmccabe and Eddy Xu via wang) +HDFS-7961. Trigger full block report after hot swapping disk. (Eddy Xu via wang) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index e9befb4..d94375e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -634,6 +634,10 @@ public class DataNode extends ReconfigurableBase conf.set(DFS_DATANODE_DATA_DIR_KEY, Joiner.on(,).join(effectiveVolumes)); dataDirs = getStorageLocations(conf); + + // Send a full block report to let NN acknowledge the volume changes. + triggerBlockReport(new BlockReportOptions.Factory() + .setIncremental(false).build()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 2f51d45..f5772e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; @@ -59,6 +63,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.mockito.Mockito; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.hamcrest.CoreMatchers.anyOf; @@ -70,6
hadoop git commit: HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu.
Repository: hadoop Updated Branches: refs/heads/branch-2.7 01460c0bc - a7f447d46 HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu. (cherry picked from commit 6413d34986f3399023426c89c9a0d401c9557716) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7f447d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7f447d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7f447d4 Branch: refs/heads/branch-2.7 Commit: a7f447d4602e273626f57960a0141e549bce7059 Parents: 01460c0 Author: Andrew Wang w...@apache.org Authored: Tue Mar 24 09:07:02 2015 -0700 Committer: Andrew Wang w...@apache.org Committed: Tue Mar 24 09:07:27 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 4 ++ .../datanode/TestDataNodeHotSwapVolumes.java| 42 3 files changed, 48 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7f447d4/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index e2729a7..7092f70 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -922,6 +922,8 @@ Release 2.7.0 - UNRELEASED HDFS-7960. The full block report should prune zombie storages even if they're not empty. (cmccabe and Eddy Xu via wang) +HDFS-7961. Trigger full block report after hot swapping disk. (Eddy Xu via wang) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7f447d4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index b32a0fc..c6641f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -637,6 +637,10 @@ public class DataNode extends ReconfigurableBase conf.set(DFS_DATANODE_DATA_DIR_KEY, Joiner.on(,).join(effectiveVolumes)); dataDirs = getStorageLocations(conf); + + // Send a full block report to let NN acknowledge the volume changes. + triggerBlockReport(new BlockReportOptions.Factory() + .setIncremental(false).build()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7f447d4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 2f51d45..f5772e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; @@ -59,6 +63,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import
hadoop git commit: HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu.
Repository: hadoop Updated Branches: refs/heads/trunk 4170c9914 - 6413d3498 HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6413d349 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6413d349 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6413d349 Branch: refs/heads/trunk Commit: 6413d34986f3399023426c89c9a0d401c9557716 Parents: 4170c99 Author: Andrew Wang w...@apache.org Authored: Tue Mar 24 09:07:02 2015 -0700 Committer: Andrew Wang w...@apache.org Committed: Tue Mar 24 09:07:02 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 4 ++ .../datanode/TestDataNodeHotSwapVolumes.java| 42 3 files changed, 48 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ee9a5db..70be18a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1251,6 +1251,8 @@ Release 2.7.0 - UNRELEASED HDFS-7960. The full block report should prune zombie storages even if they're not empty. (cmccabe and Eddy Xu via wang) +HDFS-7961. Trigger full block report after hot swapping disk. (Eddy Xu via wang) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index e9befb4..d94375e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -634,6 +634,10 @@ public class DataNode extends ReconfigurableBase conf.set(DFS_DATANODE_DATA_DIR_KEY, Joiner.on(,).join(effectiveVolumes)); dataDirs = getStorageLocations(conf); + + // Send a full block report to let NN acknowledge the volume changes. + triggerBlockReport(new BlockReportOptions.Factory() + .setIncremental(false).build()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 2f51d45..f5772e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; @@ -59,6 +63,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.mockito.Mockito; import static
hadoop git commit: HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu.
Repository: hadoop Updated Branches: refs/heads/branch-2 94eb8e5b3 - e39c58fd9 HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu. (cherry picked from commit 6413d34986f3399023426c89c9a0d401c9557716) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e39c58fd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e39c58fd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e39c58fd Branch: refs/heads/branch-2 Commit: e39c58fd9dfa03851301d105abcb2e62f757b3f5 Parents: 94eb8e5 Author: Andrew Wang w...@apache.org Authored: Tue Mar 24 09:07:02 2015 -0700 Committer: Andrew Wang w...@apache.org Committed: Tue Mar 24 09:07:24 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 4 ++ .../datanode/TestDataNodeHotSwapVolumes.java| 42 3 files changed, 48 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e39c58fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8977904..dda75f9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -951,6 +951,8 @@ Release 2.7.0 - UNRELEASED HDFS-7960. The full block report should prune zombie storages even if they're not empty. (cmccabe and Eddy Xu via wang) +HDFS-7961. Trigger full block report after hot swapping disk. (Eddy Xu via wang) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/e39c58fd/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index b32a0fc..c6641f5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -637,6 +637,10 @@ public class DataNode extends ReconfigurableBase conf.set(DFS_DATANODE_DATA_DIR_KEY, Joiner.on(,).join(effectiveVolumes)); dataDirs = getStorageLocations(conf); + + // Send a full block report to let NN acknowledge the volume changes. + triggerBlockReport(new BlockReportOptions.Factory() + .setIncremental(false).build()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/e39c58fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 2f51d45..f5772e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; @@ -59,6 +63,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import
[12/50] [abbrv] hadoop git commit: HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu.
HDFS-7961. Trigger full block report after hot swapping disk. Contributed by Eddy Xu. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6413d349 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6413d349 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6413d349 Branch: refs/heads/HDFS-7285 Commit: 6413d34986f3399023426c89c9a0d401c9557716 Parents: 4170c99 Author: Andrew Wang w...@apache.org Authored: Tue Mar 24 09:07:02 2015 -0700 Committer: Andrew Wang w...@apache.org Committed: Tue Mar 24 09:07:02 2015 -0700 -- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../hadoop/hdfs/server/datanode/DataNode.java | 4 ++ .../datanode/TestDataNodeHotSwapVolumes.java| 42 3 files changed, 48 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index ee9a5db..70be18a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -1251,6 +1251,8 @@ Release 2.7.0 - UNRELEASED HDFS-7960. The full block report should prune zombie storages even if they're not empty. (cmccabe and Eddy Xu via wang) +HDFS-7961. Trigger full block report after hot swapping disk. (Eddy Xu via wang) + BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS HDFS-7720. Quota by Storage Type API, tools and ClientNameNode http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java index e9befb4..d94375e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java @@ -634,6 +634,10 @@ public class DataNode extends ReconfigurableBase conf.set(DFS_DATANODE_DATA_DIR_KEY, Joiner.on(,).join(effectiveVolumes)); dataDirs = getStorageLocations(conf); + + // Send a full block report to let NN acknowledge the volume changes. + triggerBlockReport(new BlockReportOptions.Factory() + .setIncremental(false).build()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/6413d349/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java -- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java index 2f51d45..f5772e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeHotSwapVolumes.java @@ -34,12 +34,16 @@ import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSNNTopology; import org.apache.hadoop.hdfs.protocol.BlockListAsLongs; +import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB; import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeSpi; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetTestUtil; import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl; +import org.apache.hadoop.hdfs.server.protocol.BlockReportContext; +import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; +import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.test.GenericTestUtils; import org.junit.After; import org.junit.Test; @@ -59,6 +63,7 @@ import java.util.concurrent.TimeoutException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.mockito.Mockito; import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY; import static org.hamcrest.CoreMatchers.anyOf; @@ -70,6
