Repository: hadoop Updated Branches: refs/heads/HDFS-7240 e5e7c454d -> f01017b5b
HDFS-12256. Ozone : handle inactive containers on DataNode. Contributed by Chen Liang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f01017b5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f01017b5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f01017b5 Branch: refs/heads/HDFS-7240 Commit: f01017b5be3c75376b20c843143426bcdb2c51ad Parents: e5e7c45 Author: Xiaoyu Yao <x...@apache.org> Authored: Tue Sep 19 13:07:17 2017 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Tue Sep 19 13:07:17 2017 -0700 ---------------------------------------------------------------------- .../common/impl/ContainerManagerImpl.java | 31 +++++++++++++++----- .../container/common/impl/ContainerStatus.java | 19 ++---------- .../common/impl/TestContainerPersistence.java | 2 +- 3 files changed, 26 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/f01017b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java index deed64a..b44d820 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java @@ -227,7 +227,7 @@ public class ContainerManagerImpl implements ContainerManager { // when loading the info we get a null, this often means last time // SCM was ending up at some middle phase causing that the metadata // was not populated. Such containers are marked as inactive. - containerMap.put(keyName, new ContainerStatus(null, false)); + containerMap.put(keyName, new ContainerStatus(null)); return; } containerData = ContainerData.getFromProtBuf(containerDataProto); @@ -241,12 +241,11 @@ public class ContainerManagerImpl implements ContainerManager { // Hopefully SCM will ask us to delete this container and rebuild it. LOG.error("Invalid SHA found for container data. Name :{}" + "cowardly refusing to read invalid data", containerName); - containerMap.put(keyName, new ContainerStatus(null, false)); + containerMap.put(keyName, new ContainerStatus(null)); return; } - ContainerStatus containerStatus = new ContainerStatus( - containerData, true); + ContainerStatus containerStatus = new ContainerStatus(containerData); // Initialize pending deletion blocks count in in-memory // container status. MetadataStore metadata = KeyUtils.getDB(containerData, conf); @@ -263,7 +262,7 @@ public class ContainerManagerImpl implements ContainerManager { // TODO : Add this file to a recovery Queue. // Remember that this container is busted and we cannot use it. - containerMap.put(keyName, new ContainerStatus(null, false)); + containerMap.put(keyName, new ContainerStatus(null)); throw new StorageContainerException("Unable to read container info", UNABLE_TO_READ_METADATA_DB); } finally { @@ -440,6 +439,11 @@ public class ContainerManagerImpl implements ContainerManager { throw new StorageContainerException("No such container. Name : " + containerName, CONTAINER_NOT_FOUND); } + if (status.getContainer() == null) { + LOG.debug("Invalid container data. Name: {}", containerName); + throw new StorageContainerException("Invalid container data. Name : " + + containerName, CONTAINER_NOT_FOUND); + } ContainerUtils.removeContainer(status.getContainer(), conf, forceDelete); containerMap.remove(containerName); } catch (StorageContainerException e) { @@ -514,7 +518,12 @@ public class ContainerManagerImpl implements ContainerManager { throw new StorageContainerException("Unable to find the container. Name: " + containerName, CONTAINER_NOT_FOUND); } - return containerMap.get(containerName).getContainer(); + ContainerData cData = containerMap.get(containerName).getContainer(); + if (cData == null) { + throw new StorageContainerException("Invalid container data. Name: " + + containerName, CONTAINER_INTERNAL_ERROR); + } + return cData; } /** @@ -547,7 +556,7 @@ public class ContainerManagerImpl implements ContainerManager { // I/O failure, this allows us to take quick action in case of container // issues. - ContainerStatus status = new ContainerStatus(containerData, true); + ContainerStatus status = new ContainerStatus(containerData); containerMap.put(containerName, status); } @@ -580,6 +589,12 @@ public class ContainerManagerImpl implements ContainerManager { try { Path location = locationManager.getContainerPath(); ContainerData orgData = containerMap.get(containerName).getContainer(); + if (orgData == null) { + // updating a invalid container + throw new StorageContainerException("Update a container with invalid" + + "container meta data", CONTAINER_INTERNAL_ERROR); + } + if (!forceUpdate && !orgData.isOpen()) { throw new StorageContainerException( "Update a closed container is not allowed. Name: " + containerName, @@ -611,7 +626,7 @@ public class ContainerManagerImpl implements ContainerManager { } // Update the in-memory map - ContainerStatus newStatus = new ContainerStatus(data, true); + ContainerStatus newStatus = new ContainerStatus(data); containerMap.replace(containerName, newStatus); } catch (IOException e) { // Restore the container file from backup http://git-wip-us.apache.org/repos/asf/hadoop/blob/f01017b5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java index 91c026c..183157d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java @@ -30,7 +30,6 @@ import org.apache.hadoop.ozone.container.common.helpers.ContainerData; */ public class ContainerStatus { private final ContainerData containerData; - private final boolean active; /** * Number of pending deletion blocks in container. @@ -41,12 +40,10 @@ public class ContainerStatus { * Creates a Container Status class. * * @param containerData - ContainerData. - * @param active - Active or not active. */ - ContainerStatus(ContainerData containerData, boolean active) { + ContainerStatus(ContainerData containerData) { this.numPendingDeletionBlocks = 0; this.containerData = containerData; - this.active = active; } /** @@ -56,19 +53,7 @@ public class ContainerStatus { * @return ContainerData. */ public ContainerData getContainer() { - if (active) { - return containerData; - } - return null; - } - - /** - * Indicates if a container is Active. - * - * @return true if it is active. - */ - public boolean isActive() { - return active; + return containerData; } /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/f01017b5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index 5b2fce1..9083e47 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -173,7 +173,7 @@ public class TestContainerPersistence { ContainerStatus status = containerManager .getContainerMap().get(containerName); - Assert.assertTrue(status.isActive()); + Assert.assertNotNull(status.getContainer()); Assert.assertNotNull(status.getContainer().getContainerPath()); Assert.assertNotNull(status.getContainer().getDBPath()); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
