Repository: hadoop Updated Branches: refs/heads/trunk 64e739e34 -> 3cc7ce816
HDDS-270. Move generic container util functions to ContianerUtils. Contributed by Hanisha Koneru. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3cc7ce81 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3cc7ce81 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3cc7ce81 Branch: refs/heads/trunk Commit: 3cc7ce816e4ffb7287aa05cc5e00b2e058b4a2a4 Parents: 64e739e Author: Anu Engineer <[email protected]> Authored: Fri Jul 27 07:12:21 2018 -0700 Committer: Anu Engineer <[email protected]> Committed: Fri Jul 27 07:12:21 2018 -0700 ---------------------------------------------------------------------- .../common/helpers/ContainerUtils.java | 34 ++++++++++++-------- .../container/common/interfaces/Container.java | 6 ++++ .../container/keyvalue/KeyValueContainer.java | 22 +++++++++---- .../helpers/KeyValueContainerLocationUtil.java | 17 ---------- .../container/ozoneimpl/ContainerReader.java | 27 ++++++---------- .../keyvalue/TestKeyValueContainer.java | 24 ++++---------- .../common/impl/TestContainerPersistence.java | 13 +++----- 7 files changed, 62 insertions(+), 81 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java index 77a891a..469c969 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java @@ -199,20 +199,6 @@ public final class ContainerUtils { } /** - * Returns container file location. - * - * @param containerData - Data - * @param location - Root path - * @return Path - */ - public static File getContainerFile(ContainerData containerData, - Path location) { - return location.resolve(Long.toString(containerData - .getContainerID()).concat(CONTAINER_EXTENSION)) - .toFile(); - } - - /** * Persistent a {@link DatanodeDetails} to a local file. * * @throws IOException when read/write error occurs @@ -300,4 +286,24 @@ public final class ContainerUtils { } } + /** + * Get the .container file from the containerBaseDir + * @param containerBaseDir container base directory. The name of this + * directory is same as the containerID + * @return the .container file + */ + public static File getContainerFile(File containerBaseDir) { + // Container file layout is + // .../<<containerID>>/metadata/<<containerID>>.container + String containerFilePath = OzoneConsts.CONTAINER_META_PATH + File.separator + + getContainerID(containerBaseDir) + OzoneConsts.CONTAINER_EXTENSION; + return new File(containerBaseDir, containerFilePath); + } + + /** + * ContainerID can be decoded from the container base directory name + */ + public static long getContainerID(File containerBaseDir) { + return Long.parseLong(containerBaseDir.getName()); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java index fe35e1d..fc91920 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.util.RwLock; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; +import java.io.File; import java.util.Map; @@ -92,6 +93,11 @@ public interface Container extends RwLock { ContainerProtos.ContainerType getContainerType(); /** + * Returns containerFile. + */ + File getContainerFile(); + + /** * updates the DeleteTransactionId. * @param deleteTransactionId */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java index 14f731a..d0e77d2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainer.java @@ -114,18 +114,16 @@ public class KeyValueContainer implements Container { containerMetaDataPath = KeyValueContainerLocationUtil .getContainerMetaDataPath(hddsVolumeDir, scmId, containerID); + containerData.setMetadataPath(containerMetaDataPath.getPath()); + File chunksPath = KeyValueContainerLocationUtil.getChunksLocationPath( hddsVolumeDir, scmId, containerID); - File containerFile = KeyValueContainerLocationUtil.getContainerFile( - containerMetaDataPath, containerID); - File dbFile = KeyValueContainerLocationUtil.getContainerDBFile( - containerMetaDataPath, containerID); - // Check if it is new Container. ContainerUtils.verifyIsNewContainer(containerMetaDataPath); //Create Metadata path chunks path and metadata db + File dbFile = getContainerDBFile(); KeyValueContainerUtil.createContainerMetaData(containerMetaDataPath, chunksPath, dbFile, config); @@ -133,13 +131,13 @@ public class KeyValueContainer implements Container { OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT); //Set containerData for the KeyValueContainer. - containerData.setMetadataPath(containerMetaDataPath.getPath()); containerData.setChunksPath(chunksPath.getPath()); containerData.setContainerDBType(impl); containerData.setDbFile(dbFile); containerData.setVolume(containerVolume); // Create .container file + File containerFile = getContainerFile(); writeToContainerFile(containerFile, true); } catch (StorageContainerException ex) { @@ -415,12 +413,22 @@ public class KeyValueContainer implements Container { * Returns containerFile. * @return .container File name */ - private File getContainerFile() { + @Override + public File getContainerFile() { return new File(containerData.getMetadataPath(), containerData .getContainerID() + OzoneConsts.CONTAINER_EXTENSION); } /** + * Returns container DB file + * @return + */ + public File getContainerDBFile() { + return new File(containerData.getMetadataPath(), containerData + .getContainerID() + OzoneConsts.DN_CONTAINER_DB); + } + + /** * Creates a temporary file. * @param file * @return http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java index 02a8e73..0a81ed8 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerLocationUtil.java @@ -99,27 +99,10 @@ public final class KeyValueContainerLocationUtil { } /** - * Returns containerFile. - * @param containerMetaDataPath - * @param containerID - * @return .container File name - */ - public static File getContainerFile(File containerMetaDataPath, - long containerID) { - Preconditions.checkNotNull(containerMetaDataPath); - return new File(containerMetaDataPath, containerID + - OzoneConsts.CONTAINER_EXTENSION); - } - - /** * Return containerDB File. - * @param containerMetaDataPath - * @param containerID - * @return containerDB File name */ public static File getContainerDBFile(File containerMetaDataPath, long containerID) { - Preconditions.checkNotNull(containerMetaDataPath); return new File(containerMetaDataPath, containerID + OzoneConsts .DN_CONTAINER_DB); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java index dc33f2e..7c986f0 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/ContainerReader.java @@ -23,8 +23,8 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; -import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; @@ -32,7 +32,6 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerLocationUtil; import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyValueContainerUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -119,8 +118,7 @@ public class ContainerReader implements Runnable { } for (File scmLoc : scmDir) { - File currentDir = null; - currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT); + File currentDir = new File(scmLoc, Storage.STORAGE_DIR_CURRENT); File[] containerTopDirs = currentDir.listFiles(); if (containerTopDirs != null) { for (File containerTopDir : containerTopDirs) { @@ -128,21 +126,14 @@ public class ContainerReader implements Runnable { File[] containerDirs = containerTopDir.listFiles(); if (containerDirs != null) { for (File containerDir : containerDirs) { - File metadataPath = new File(containerDir + File.separator + - OzoneConsts.CONTAINER_META_PATH); - long containerID = Long.parseLong(containerDir.getName()); - if (metadataPath.exists()) { - File containerFile = KeyValueContainerLocationUtil - .getContainerFile(metadataPath, containerID); - if (containerFile.exists()) { - verifyContainerFile(containerID, containerFile); - } else { - LOG.error("Missing .container file for ContainerID: {}", - containerID); - } + File containerFile = ContainerUtils.getContainerFile( + containerDir); + long containerID = ContainerUtils.getContainerID(containerDir); + if (containerFile.exists()) { + verifyContainerFile(containerID, containerFile); } else { - LOG.error("Missing container metadata directory for " + - "ContainerID: {}", containerID); + LOG.error("Missing .container file for ContainerID: {}", + containerDir.getName()); } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index 2bf41e5..35772ff 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -111,11 +111,9 @@ public class TestKeyValueContainer { File containerMetaDataLoc = new File(containerMetaDataPath); //Check whether container file and container db file exists or not. - assertTrue(KeyValueContainerLocationUtil.getContainerFile( - containerMetaDataLoc, containerID).exists(), ".Container File does" + - " not exist"); - assertTrue(KeyValueContainerLocationUtil.getContainerDBFile( - containerMetaDataLoc, containerID).exists(), "Container DB does " + + assertTrue(keyValueContainer.getContainerFile().exists(), + ".Container File does not exist"); + assertTrue(keyValueContainer.getContainerDBFile().exists(), "Container DB does " + "not exist"); } @@ -166,11 +164,9 @@ public class TestKeyValueContainer { .getParentFile().exists()); assertFalse("Container File still exists", - KeyValueContainerLocationUtil.getContainerFile(containerMetaDataLoc, - containerID).exists()); + keyValueContainer.getContainerFile().exists()); assertFalse("Container DB file still exists", - KeyValueContainerLocationUtil.getContainerDBFile(containerMetaDataLoc, - containerID).exists()); + keyValueContainer.getContainerDBFile().exists()); } @@ -188,9 +184,7 @@ public class TestKeyValueContainer { //Check state in the .container file String containerMetaDataPath = keyValueContainerData .getMetadataPath(); - File containerMetaDataLoc = new File(containerMetaDataPath); - File containerFile = KeyValueContainerLocationUtil.getContainerFile( - containerMetaDataLoc, containerID); + File containerFile = keyValueContainer.getContainerFile(); keyValueContainerData = (KeyValueContainerData) ContainerDataYaml .readContainerFile(containerFile); @@ -227,11 +221,7 @@ public class TestKeyValueContainer { assertEquals(2, keyValueContainerData.getMetadata().size()); //Check metadata in the .container file - String containerMetaDataPath = keyValueContainerData - .getMetadataPath(); - File containerMetaDataLoc = new File(containerMetaDataPath); - File containerFile = KeyValueContainerLocationUtil.getContainerFile( - containerMetaDataLoc, containerID); + File containerFile = keyValueContainer.getContainerFile(); keyValueContainerData = (KeyValueContainerData) ContainerDataYaml .readContainerFile(containerFile); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3cc7ce81/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index c2e1645..5322c8e 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -32,6 +32,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.ozone.container.ContainerTestHelper; +import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume @@ -40,8 +41,6 @@ import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; -import org.apache.hadoop.ozone.container.keyvalue.helpers - .KeyValueContainerLocationUtil; import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl; import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl; import org.apache.hadoop.ozone.container.keyvalue.interfaces.ChunkManager; @@ -714,9 +713,7 @@ public class TestContainerPersistence { KeyValueContainer container = (KeyValueContainer) addContainer(containerSet, testContainerID); - File orgContainerFile = KeyValueContainerLocationUtil.getContainerFile( - new File(container.getContainerData().getMetadataPath()), - testContainerID); + File orgContainerFile = container.getContainerFile(); Assert.assertTrue(orgContainerFile.exists()); Map<String, String> newMetadata = Maps.newHashMap(); @@ -738,9 +735,9 @@ public class TestContainerPersistence { actualNewData.getMetadata().get("owner")); // Verify container data on disk - File newContainerFile = KeyValueContainerLocationUtil.getContainerFile( - new File(actualNewData.getMetadataPath()), - testContainerID); + File containerBaseDir = new File(actualNewData.getMetadataPath()) + .getParentFile(); + File newContainerFile = ContainerUtils.getContainerFile(containerBaseDir); Assert.assertTrue("Container file should exist.", newContainerFile.exists()); Assert.assertEquals("Container file should be in same location.", --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
