http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java deleted file mode 100644 index d120a5c..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.states.endpoint;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java deleted file mode 100644 index 7755345..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestHddsVolume.java +++ /dev/null @@ -1,145 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.volume; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.GetSpaceUsed; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.ozone.container.common.helpers.DatanodeVersionFile; -import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil; -import static org.junit.Assert.*; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import java.io.File; -import java.util.Properties; -import java.util.UUID; - -/** - * Unit tests for {@link HddsVolume}. - */ -public class TestHddsVolume { - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - private static final String DATANODE_UUID = UUID.randomUUID().toString(); - private static final String CLUSTER_ID = UUID.randomUUID().toString(); - private static final Configuration CONF = new Configuration(); - private static final String DU_CACHE_FILE = "scmUsed"; - - private File rootDir; - private HddsVolume volume; - private File versionFile; - - @Before - public void setup() throws Exception { - rootDir = new File(folder.getRoot(), HddsVolume.HDDS_VOLUME_DIR); - volume = new HddsVolume.Builder(folder.getRoot().getPath()) - .datanodeUuid(DATANODE_UUID) - .conf(CONF) - .build(); - versionFile = HddsVolumeUtil.getVersionFile(rootDir); - } - - @Test - public void testHddsVolumeInitialization() throws Exception { - - // The initial state of HddsVolume should be "NOT_FORMATTED" when - // clusterID is not specified and the version file should not be written - // to disk. - assertTrue(volume.getClusterID() == null); - assertEquals(volume.getStorageType(), StorageType.DEFAULT); - assertEquals(volume.getStorageState(), - HddsVolume.VolumeState.NOT_FORMATTED); - assertFalse("Version file should not be created when clusterID is not " + - "known.", versionFile.exists()); - - - // Format the volume with clusterID. - volume.format(CLUSTER_ID); - - // The state of HddsVolume after formatting with clusterID should be - // NORMAL and the version file should exist. - assertTrue("Volume format should create Version file", - versionFile.exists()); - assertEquals(volume.getClusterID(), CLUSTER_ID); - assertEquals(volume.getStorageState(), HddsVolume.VolumeState.NORMAL); - } - - @Test - public void testReadPropertiesFromVersionFile() throws Exception { - volume.format(CLUSTER_ID); - - Properties properties = DatanodeVersionFile.readFrom(versionFile); - - String storageID = HddsVolumeUtil.getStorageID(properties, versionFile); - String clusterID = HddsVolumeUtil.getClusterID( - properties, versionFile, CLUSTER_ID); - String datanodeUuid = HddsVolumeUtil.getDatanodeUUID( - properties, versionFile, DATANODE_UUID); - long cTime = HddsVolumeUtil.getCreationTime( - properties, versionFile); - int layoutVersion = HddsVolumeUtil.getLayOutVersion( - properties, versionFile); - - assertEquals(volume.getStorageID(), storageID); - assertEquals(volume.getClusterID(), clusterID); - assertEquals(volume.getDatanodeUuid(), datanodeUuid); - assertEquals(volume.getCTime(), cTime); - assertEquals(volume.getLayoutVersion(), layoutVersion); - } - - @Test - public void testShutdown() throws Exception{ - // Return dummy value > 0 for scmUsage so that scm cache file is written - // during shutdown. - GetSpaceUsed scmUsageMock = Mockito.mock(GetSpaceUsed.class); - volume.setScmUsageForTesting(scmUsageMock); - Mockito.when(scmUsageMock.getUsed()).thenReturn(Long.valueOf(100)); - - assertTrue("Available volume should be positive", - volume.getAvailable() > 0); - - // Shutdown the volume. - volume.shutdown(); - - // Volume state should be "NON_EXISTENT" when volume is shutdown. - assertEquals(volume.getStorageState(), - HddsVolume.VolumeState.NON_EXISTENT); - - // Volume should save scmUsed cache file once volume is shutdown - File scmUsedFile = new File(folder.getRoot(), DU_CACHE_FILE); - System.out.println("scmUsedFile: " + scmUsedFile); - assertTrue("scmUsed cache file should be saved on shutdown", - scmUsedFile.exists()); - - try { - // Volume.getAvailable() should fail with NullPointerException as usage - // is shutdown. - volume.getAvailable(); - fail("HddsVolume#shutdown test failed"); - } catch (Exception ex){ - assertTrue(ex instanceof NullPointerException); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java deleted file mode 100644 index 80594d35..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestRoundRobinVolumeChoosingPolicy.java +++ /dev/null @@ -1,126 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.volume; - -import org.apache.hadoop.fs.GetSpaceUsed; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.util.DiskChecker.DiskOutOfSpaceException; -import org.apache.hadoop.util.ReflectionUtils; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.List; -import java.util.UUID; - -/** - * Tests {@link RoundRobinVolumeChoosingPolicy}. - */ -public class TestRoundRobinVolumeChoosingPolicy { - - private RoundRobinVolumeChoosingPolicy policy; - private List<HddsVolume> volumes; - - private final String baseDir = MiniDFSCluster.getBaseDirectory(); - private final String volume1 = baseDir + "disk1"; - private final String volume2 = baseDir + "disk2"; - private static final String DUMMY_IP_ADDR = "0.0.0.0"; - - @Before - public void setup() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - String dataDirKey = volume1 + "," + volume2; - conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDirKey); - policy = ReflectionUtils.newInstance( - RoundRobinVolumeChoosingPolicy.class, null); - VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); - volumes = volumeSet.getVolumesList(); - } - - @Test - public void testRRVolumeChoosingPolicy() throws Exception { - HddsVolume hddsVolume1 = volumes.get(0); - HddsVolume hddsVolume2 = volumes.get(1); - - // Set available space in volume1 to 100L - setAvailableSpace(hddsVolume1, 100L); - - // Set available space in volume1 to 200L - setAvailableSpace(hddsVolume2, 200L); - - Assert.assertEquals(100L, hddsVolume1.getAvailable()); - Assert.assertEquals(200L, hddsVolume2.getAvailable()); - - // Test two rounds of round-robin choosing - Assert.assertEquals(hddsVolume1, policy.chooseVolume(volumes, 0)); - Assert.assertEquals(hddsVolume2, policy.chooseVolume(volumes, 0)); - Assert.assertEquals(hddsVolume1, policy.chooseVolume(volumes, 0)); - Assert.assertEquals(hddsVolume2, policy.chooseVolume(volumes, 0)); - - // The first volume has only 100L space, so the policy should - // choose the second one in case we ask for more. - Assert.assertEquals(hddsVolume2, - policy.chooseVolume(volumes, 150)); - - // Fail if no volume has enough space available - try { - policy.chooseVolume(volumes, Long.MAX_VALUE); - Assert.fail(); - } catch (IOException e) { - // Passed. - } - } - - @Test - public void testRRPolicyExceptionMessage() throws Exception { - HddsVolume hddsVolume1 = volumes.get(0); - HddsVolume hddsVolume2 = volumes.get(1); - - // Set available space in volume1 to 100L - setAvailableSpace(hddsVolume1, 100L); - - // Set available space in volume1 to 200L - setAvailableSpace(hddsVolume2, 200L); - - int blockSize = 300; - try { - policy.chooseVolume(volumes, blockSize); - Assert.fail("expected to throw DiskOutOfSpaceException"); - } catch(DiskOutOfSpaceException e) { - Assert.assertEquals("Not returnig the expected message", - "Out of space: The volume with the most available space (=" + 200 - + " B) is less than the container size (=" + blockSize + " B).", - e.getMessage()); - } - } - - private void setAvailableSpace(HddsVolume hddsVolume, long availableSpace) - throws IOException { - GetSpaceUsed scmUsageMock = Mockito.mock(GetSpaceUsed.class); - hddsVolume.setScmUsageForTesting(scmUsageMock); - // Set used space to capacity -requiredAvailableSpace so that - // getAvailable() returns us the specified availableSpace. - Mockito.when(scmUsageMock.getUsed()).thenReturn( - (hddsVolume.getCapacity() - availableSpace)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java deleted file mode 100644 index fca68b1..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/volume/TestVolumeSet.java +++ /dev/null @@ -1,254 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.volume; - -import java.io.IOException; -import org.apache.commons.io.FileUtils; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.ozone.container.common.utils.HddsVolumeUtil; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.test.GenericTestUtils.LogCapturer; - -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; -import static org.apache.hadoop.ozone.container.common.volume.HddsVolume - .HDDS_VOLUME_DIR; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.Timeout; - -import java.io.File; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -/** - * Tests {@link VolumeSet} operations. - */ -public class TestVolumeSet { - - private OzoneConfiguration conf; - private VolumeSet volumeSet; - private final String baseDir = MiniDFSCluster.getBaseDirectory(); - private final String volume1 = baseDir + "disk1"; - private final String volume2 = baseDir + "disk2"; - private final List<String> volumes = new ArrayList<>(); - - private static final String DUMMY_IP_ADDR = "0.0.0.0"; - - private void initializeVolumeSet() throws Exception { - volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); - } - - @Rule - public Timeout testTimeout = new Timeout(300_000); - - @Before - public void setup() throws Exception { - conf = new OzoneConfiguration(); - String dataDirKey = volume1 + "," + volume2; - volumes.add(volume1); - volumes.add(volume2); - conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDirKey); - initializeVolumeSet(); - } - - @After - public void shutdown() throws IOException { - // Delete the hdds volume root dir - List<HddsVolume> hddsVolumes = new ArrayList<>(); - hddsVolumes.addAll(volumeSet.getVolumesList()); - hddsVolumes.addAll(volumeSet.getFailedVolumesList()); - - for (HddsVolume volume : hddsVolumes) { - FileUtils.deleteDirectory(volume.getHddsRootDir()); - } - volumeSet.shutdown(); - - FileUtil.fullyDelete(new File(baseDir)); - } - - private boolean checkVolumeExistsInVolumeSet(String volume) { - for (HddsVolume hddsVolume : volumeSet.getVolumesList()) { - if (hddsVolume.getHddsRootDir().getPath().equals( - HddsVolumeUtil.getHddsRoot(volume))) { - return true; - } - } - return false; - } - - @Test - public void testVolumeSetInitialization() throws Exception { - - List<HddsVolume> volumesList = volumeSet.getVolumesList(); - - // VolumeSet initialization should add volume1 and volume2 to VolumeSet - assertEquals("VolumeSet intialization is incorrect", - volumesList.size(), volumes.size()); - assertTrue("VolumeSet not initailized correctly", - checkVolumeExistsInVolumeSet(volume1)); - assertTrue("VolumeSet not initailized correctly", - checkVolumeExistsInVolumeSet(volume2)); - } - - @Test - public void testAddVolume() { - - assertEquals(2, volumeSet.getVolumesList().size()); - - // Add a volume to VolumeSet - String volume3 = baseDir + "disk3"; - boolean success = volumeSet.addVolume(volume3); - - assertTrue(success); - assertEquals(3, volumeSet.getVolumesList().size()); - assertTrue("AddVolume did not add requested volume to VolumeSet", - checkVolumeExistsInVolumeSet(volume3)); - } - - @Test - public void testFailVolume() throws Exception { - - //Fail a volume - volumeSet.failVolume(volume1); - - // Failed volume should not show up in the volumeList - assertEquals(1, volumeSet.getVolumesList().size()); - - // Failed volume should be added to FailedVolumeList - assertEquals("Failed volume not present in FailedVolumeMap", - 1, volumeSet.getFailedVolumesList().size()); - assertEquals("Failed Volume list did not match", - HddsVolumeUtil.getHddsRoot(volume1), - volumeSet.getFailedVolumesList().get(0).getHddsRootDir().getPath()); - assertTrue(volumeSet.getFailedVolumesList().get(0).isFailed()); - - // Failed volume should not exist in VolumeMap - Path volume1Path = new Path(volume1); - assertFalse(volumeSet.getVolumeMap().containsKey(volume1Path)); - } - - @Test - public void testRemoveVolume() throws Exception { - - assertEquals(2, volumeSet.getVolumesList().size()); - - // Remove a volume from VolumeSet - volumeSet.removeVolume(volume1); - assertEquals(1, volumeSet.getVolumesList().size()); - - // Attempting to remove a volume which does not exist in VolumeSet should - // log a warning. - LogCapturer logs = LogCapturer.captureLogs( - LogFactory.getLog(VolumeSet.class)); - volumeSet.removeVolume(volume1); - assertEquals(1, volumeSet.getVolumesList().size()); - String expectedLogMessage = "Volume : " + - HddsVolumeUtil.getHddsRoot(volume1) + " does not exist in VolumeSet"; - assertTrue("Log output does not contain expected log message: " - + expectedLogMessage, logs.getOutput().contains(expectedLogMessage)); - } - - @Test - public void testVolumeInInconsistentState() throws Exception { - assertEquals(2, volumeSet.getVolumesList().size()); - - // Add a volume to VolumeSet - String volume3 = baseDir + "disk3"; - - // Create the root volume dir and create a sub-directory within it. - File newVolume = new File(volume3, HDDS_VOLUME_DIR); - System.out.println("new volume root: " + newVolume); - newVolume.mkdirs(); - assertTrue("Failed to create new volume root", newVolume.exists()); - File dataDir = new File(newVolume, "chunks"); - dataDir.mkdirs(); - assertTrue(dataDir.exists()); - - // The new volume is in an inconsistent state as the root dir is - // non-empty but the version file does not exist. Add Volume should - // return false. - boolean success = volumeSet.addVolume(volume3); - - assertFalse(success); - assertEquals(2, volumeSet.getVolumesList().size()); - assertTrue("AddVolume should fail for an inconsistent volume", - !checkVolumeExistsInVolumeSet(volume3)); - - // Delete volume3 - File volume = new File(volume3); - FileUtils.deleteDirectory(volume); - } - - @Test - public void testShutdown() throws Exception { - List<HddsVolume> volumesList = volumeSet.getVolumesList(); - - volumeSet.shutdown(); - - // Verify that the volumes are shutdown and the volumeUsage is set to null. - for (HddsVolume volume : volumesList) { - Assert.assertNull(volume.getVolumeInfo().getUsageForTesting()); - try { - // getAvailable() should throw null pointer exception as usage is null. - volume.getAvailable(); - fail("Volume shutdown failed."); - } catch (NullPointerException ex) { - // Do Nothing. Exception is expected. - } - } - } - - @Test - public void testFailVolumes() throws Exception{ - VolumeSet volSet = null; - File readOnlyVolumePath = new File(baseDir); - //Set to readonly, so that this volume will be failed - readOnlyVolumePath.setReadOnly(); - File volumePath = GenericTestUtils.getRandomizedTestDir(); - OzoneConfiguration ozoneConfig = new OzoneConfiguration(); - ozoneConfig.set(HDDS_DATANODE_DIR_KEY, readOnlyVolumePath.getAbsolutePath() - + "," + volumePath.getAbsolutePath()); - volSet = new VolumeSet(UUID.randomUUID().toString(), ozoneConfig); - assertTrue(volSet.getFailedVolumesList().size() == 1); - assertEquals(readOnlyVolumePath, volSet.getFailedVolumesList().get(0) - .getHddsRootDir()); - - //Set back to writable - try { - readOnlyVolumePath.setWritable(true); - } finally { - FileUtil.fullyDelete(volumePath); - } - - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java deleted file mode 100644 index 6fe6d81..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue; - -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.ozone.container.common.helpers.BlockData; -import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.volume - .RoundRobinVolumeChoosingPolicy; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.common.volume.HddsVolume; -import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; - -import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; - -/** - * This class is used to test key related operations on the container. - */ -public class TestBlockManagerImpl { - - private OzoneConfiguration config; - private String scmId = UUID.randomUUID().toString(); - private VolumeSet volumeSet; - private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; - private KeyValueContainerData keyValueContainerData; - private KeyValueContainer keyValueContainer; - private BlockData blockData; - private BlockManagerImpl blockManager; - private BlockID blockID; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - - @Before - public void setUp() throws Exception { - config = new OzoneConfiguration(); - - HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID() - .toString()).build(); - - volumeSet = mock(VolumeSet.class); - - volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); - Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) - .thenReturn(hddsVolume); - - keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); - - keyValueContainer = new KeyValueContainer( - keyValueContainerData, config); - - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - - // Creating BlockData - blockID = new BlockID(1L, 1L); - blockData = new BlockData(blockID); - blockData.addMetadata("VOLUME", "ozone"); - blockData.addMetadata("OWNER", "hdfs"); - List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, 1024); - chunkList.add(info.getProtoBufMessage()); - blockData.setChunks(chunkList); - - // Create KeyValueContainerManager - blockManager = new BlockManagerImpl(config); - - } - - @Test - public void testPutAndGetBlock() throws Exception { - assertEquals(0, keyValueContainer.getContainerData().getKeyCount()); - //Put Block - blockManager.putBlock(keyValueContainer, blockData); - - assertEquals(1, keyValueContainer.getContainerData().getKeyCount()); - //Get Block - BlockData fromGetBlockData = blockManager.getBlock(keyValueContainer, - blockData.getBlockID()); - - assertEquals(blockData.getContainerID(), fromGetBlockData.getContainerID()); - assertEquals(blockData.getLocalID(), fromGetBlockData.getLocalID()); - assertEquals(blockData.getChunks().size(), - fromGetBlockData.getChunks().size()); - assertEquals(blockData.getMetadata().size(), fromGetBlockData.getMetadata() - .size()); - - } - - - @Test - public void testDeleteBlock() throws Exception { - try { - assertEquals(0, - keyValueContainer.getContainerData().getKeyCount()); - //Put Block - blockManager.putBlock(keyValueContainer, blockData); - assertEquals(1, - keyValueContainer.getContainerData().getKeyCount()); - //Delete Block - blockManager.deleteBlock(keyValueContainer, blockID); - assertEquals(0, - keyValueContainer.getContainerData().getKeyCount()); - try { - blockManager.getBlock(keyValueContainer, blockID); - fail("testDeleteBlock"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains( - "Unable to find the block", ex); - } - } catch (IOException ex) { - fail("testDeleteBlock failed"); - } - } - - @Test - public void testListBlock() throws Exception { - try { - blockManager.putBlock(keyValueContainer, blockData); - List<BlockData> listBlockData = blockManager.listBlock( - keyValueContainer, 1, 10); - assertNotNull(listBlockData); - assertTrue(listBlockData.size() == 1); - - for (long i = 2; i <= 10; i++) { - blockID = new BlockID(1L, i); - blockData = new BlockData(blockID); - blockData.addMetadata("VOLUME", "ozone"); - blockData.addMetadata("OWNER", "hdfs"); - List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, 1024); - chunkList.add(info.getProtoBufMessage()); - blockData.setChunks(chunkList); - blockManager.putBlock(keyValueContainer, blockData); - } - - listBlockData = blockManager.listBlock( - keyValueContainer, 1, 10); - assertNotNull(listBlockData); - assertTrue(listBlockData.size() == 10); - - } catch (IOException ex) { - fail("testListBlock failed"); - } - } - - @Test - public void testGetNoSuchBlock() throws Exception { - try { - assertEquals(0, - keyValueContainer.getContainerData().getKeyCount()); - //Put Block - blockManager.putBlock(keyValueContainer, blockData); - assertEquals(1, - keyValueContainer.getContainerData().getKeyCount()); - //Delete Block - blockManager.deleteBlock(keyValueContainer, blockID); - assertEquals(0, - keyValueContainer.getContainerData().getKeyCount()); - try { - //Since the block has been deleted, we should not be able to find it - blockManager.getBlock(keyValueContainer, blockID); - fail("testGetNoSuchBlock failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains( - "Unable to find the block", ex); - assertEquals(ContainerProtos.Result.NO_SUCH_BLOCK, ex.getResult()); - } - } catch (IOException ex) { - fail("testGetNoSuchBlock failed"); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java deleted file mode 100644 index 3c0876b..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue; - -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.keyvalue.helpers.ChunkUtils; -import org.apache.hadoop.ozone.container.common.volume.HddsVolume; -import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; -import org.apache.hadoop.ozone.container.common.volume.VolumeIOStats; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.keyvalue.impl.ChunkManagerImpl; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import java.io.File; -import java.util.Arrays; -import java.util.UUID; - -import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; - -/** - * This class is used to test ChunkManager operations. - */ -public class TestChunkManagerImpl { - - private OzoneConfiguration config; - private String scmId = UUID.randomUUID().toString(); - private VolumeSet volumeSet; - private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; - private HddsVolume hddsVolume; - private KeyValueContainerData keyValueContainerData; - private KeyValueContainer keyValueContainer; - private BlockID blockID; - private ChunkManagerImpl chunkManager; - private ChunkInfo chunkInfo; - private byte[] data; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Before - public void setUp() throws Exception { - config = new OzoneConfiguration(); - hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID() - .toString()).build(); - - volumeSet = mock(VolumeSet.class); - - volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); - Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) - .thenReturn(hddsVolume); - - keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); - - keyValueContainer = new KeyValueContainer(keyValueContainerData, config); - - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - - data = "testing write chunks".getBytes(); - // Creating BlockData - blockID = new BlockID(1L, 1L); - chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, data.length); - - // Create a ChunkManager object. - chunkManager = new ChunkManagerImpl(); - - } - - @Test - public void testWriteChunkStageWriteAndCommit() throws Exception { - //As in Setup, we try to create container, these paths should exist. - assertTrue(keyValueContainerData.getChunksPath() != null); - File chunksPath = new File(keyValueContainerData.getChunksPath()); - assertTrue(chunksPath.exists()); - // Initially chunks folder should be empty. - assertTrue(chunksPath.listFiles().length == 0); - - // As no chunks are written to the volume writeBytes should be 0 - checkWriteIOStats(0, 0); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.WRITE_DATA); - // Now a chunk file is being written with Stage WRITE_DATA, so it should - // create a temporary chunk file. - assertTrue(chunksPath.listFiles().length == 1); - - File chunkFile = ChunkUtils.getChunkFile(keyValueContainerData, chunkInfo); - File tempChunkFile = new File(chunkFile.getParent(), - chunkFile.getName() + - OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER + - OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX); - - // As chunk write stage is WRITE_DATA, temp chunk file will be created. - assertTrue(tempChunkFile.exists()); - - checkWriteIOStats(data.length, 1); - - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMMIT_DATA); - - checkWriteIOStats(data.length, 1); - - // Old temp file should have been renamed to chunk file. - assertTrue(chunksPath.listFiles().length == 1); - - // As commit happened, chunk file should exist. - assertTrue(chunkFile.exists()); - assertFalse(tempChunkFile.exists()); - - } - - @Test - public void testWriteChunkIncorrectLength() throws Exception { - try { - long randomLength = 200L; - chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, randomLength); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.WRITE_DATA); - fail("testWriteChunkIncorrectLength failed"); - } catch (StorageContainerException ex) { - // As we got an exception, writeBytes should be 0. - checkWriteIOStats(0, 0); - GenericTestUtils.assertExceptionContains("data array does not match " + - "the length ", ex); - assertEquals(ContainerProtos.Result.INVALID_WRITE_SIZE, ex.getResult()); - } - } - - @Test - public void testWriteChunkStageCombinedData() throws Exception { - //As in Setup, we try to create container, these paths should exist. - assertTrue(keyValueContainerData.getChunksPath() != null); - File chunksPath = new File(keyValueContainerData.getChunksPath()); - assertTrue(chunksPath.exists()); - // Initially chunks folder should be empty. - assertTrue(chunksPath.listFiles().length == 0); - checkWriteIOStats(0, 0); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMBINED); - // Now a chunk file is being written with Stage COMBINED_DATA, so it should - // create a chunk file. - assertTrue(chunksPath.listFiles().length == 1); - File chunkFile = ChunkUtils.getChunkFile(keyValueContainerData, chunkInfo); - assertTrue(chunkFile.exists()); - checkWriteIOStats(data.length, 1); - } - - @Test - public void testReadChunk() throws Exception { - checkWriteIOStats(0, 0); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMBINED); - checkWriteIOStats(data.length, 1); - checkReadIOStats(0, 0); - byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID, - chunkInfo); - assertEquals(expectedData.length, data.length); - assertTrue(Arrays.equals(expectedData, data)); - checkReadIOStats(data.length, 1); - } - - @Test - public void testDeleteChunk() throws Exception { - File chunksPath = new File(keyValueContainerData.getChunksPath()); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMBINED); - assertTrue(chunksPath.listFiles().length == 1); - chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo); - assertTrue(chunksPath.listFiles().length == 0); - } - - @Test - public void testDeleteChunkUnsupportedRequest() throws Exception { - try { - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMBINED); - long randomLength = 200L; - chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, randomLength); - chunkManager.deleteChunk(keyValueContainer, blockID, chunkInfo); - fail("testDeleteChunkUnsupportedRequest"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Not Supported Operation.", ex); - assertEquals(ContainerProtos.Result.UNSUPPORTED_REQUEST, ex.getResult()); - } - } - - @Test - public void testWriteChunkChecksumMismatch() throws Exception { - try { - chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, data.length); - //Setting checksum to some value. - chunkInfo.setChecksum("some garbage"); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMBINED); - fail("testWriteChunkChecksumMismatch failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Checksum mismatch.", ex); - assertEquals(ContainerProtos.Result.CHECKSUM_MISMATCH, ex.getResult()); - } - } - - @Test - public void testReadChunkFileNotExists() throws Exception { - try { - // trying to read a chunk, where chunk file does not exist - byte[] expectedData = chunkManager.readChunk(keyValueContainer, blockID, - chunkInfo); - fail("testReadChunkFileNotExists failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Unable to find the chunk " + - "file.", ex); - assertEquals(ContainerProtos.Result.UNABLE_TO_FIND_CHUNK, ex.getResult()); - } - } - - @Test - public void testWriteAndReadChunkMultipleTimes() throws Exception { - for (int i=0; i<100; i++) { - chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), i), 0, data.length); - chunkManager.writeChunk(keyValueContainer, blockID, chunkInfo, data, - ContainerProtos.Stage.COMBINED); - } - checkWriteIOStats(data.length*100, 100); - assertTrue(hddsVolume.getVolumeIOStats().getWriteTime() > 0); - - for (int i=0; i<100; i++) { - chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), i), 0, data.length); - chunkManager.readChunk(keyValueContainer, blockID, chunkInfo); - } - checkReadIOStats(data.length*100, 100); - assertTrue(hddsVolume.getVolumeIOStats().getReadTime() > 0); - } - - - /** - * Check WriteIO stats. - * @param length - * @param opCount - */ - private void checkWriteIOStats(long length, long opCount) { - VolumeIOStats volumeIOStats = hddsVolume.getVolumeIOStats(); - assertEquals(length, volumeIOStats.getWriteBytes()); - assertEquals(opCount, volumeIOStats.getWriteOpCount()); - } - - /** - * Check ReadIO stats. - * @param length - * @param opCount - */ - private void checkReadIOStats(long length, long opCount) { - VolumeIOStats volumeIOStats = hddsVolume.getVolumeIOStats(); - assertEquals(length, volumeIOStats.getReadBytes()); - assertEquals(opCount, volumeIOStats.getReadOpCount()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java deleted file mode 100644 index fbc5ad0..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java +++ /dev/null @@ -1,277 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue; - -import com.google.common.primitives.Longs; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.common.helpers.BlockData; -import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.utils.MetadataKeyFilters; -import org.apache.hadoop.utils.MetadataStore; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.File; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.NoSuchElementException; -import java.util.UUID; - -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; -import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_METADATA_STORE_IMPL_LEVELDB; -import static org.apache.hadoop.ozone.OzoneConfigKeys - .OZONE_METADATA_STORE_IMPL_ROCKSDB; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * This class is used to test KeyValue container block iterator. - */ -@RunWith(Parameterized.class) -public class TestKeyValueBlockIterator { - - private KeyValueContainer container; - private KeyValueContainerData containerData; - private VolumeSet volumeSet; - private Configuration conf; - private File testRoot; - - private final String storeImpl; - - public TestKeyValueBlockIterator(String metadataImpl) { - this.storeImpl = metadataImpl; - } - - @Parameterized.Parameters - public static Collection<Object[]> data() { - return Arrays.asList(new Object[][] { - {OZONE_METADATA_STORE_IMPL_LEVELDB}, - {OZONE_METADATA_STORE_IMPL_ROCKSDB}}); - } - - @Before - public void setUp() throws Exception { - testRoot = GenericTestUtils.getRandomizedTestDir(); - conf = new OzoneConfiguration(); - conf.set(HDDS_DATANODE_DIR_KEY, testRoot.getAbsolutePath()); - conf.set(OZONE_METADATA_STORE_IMPL, storeImpl); - volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); - } - - - @After - public void tearDown() { - volumeSet.shutdown(); - FileUtil.fullyDelete(testRoot); - } - - @Test - public void testKeyValueBlockIteratorWithMixedBlocks() throws Exception { - - long containerID = 100L; - int deletedBlocks = 5; - int normalBlocks = 5; - createContainerWithBlocks(containerID, normalBlocks, deletedBlocks); - String containerPath = new File(containerData.getMetadataPath()) - .getParent(); - KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator( - containerID, new File(containerPath)); - - int counter = 0; - while(keyValueBlockIterator.hasNext()) { - BlockData blockData = keyValueBlockIterator.nextBlock(); - assertEquals(blockData.getLocalID(), counter++); - } - - assertFalse(keyValueBlockIterator.hasNext()); - - keyValueBlockIterator.seekToFirst(); - counter = 0; - while(keyValueBlockIterator.hasNext()) { - BlockData blockData = keyValueBlockIterator.nextBlock(); - assertEquals(blockData.getLocalID(), counter++); - } - assertFalse(keyValueBlockIterator.hasNext()); - - try { - keyValueBlockIterator.nextBlock(); - } catch (NoSuchElementException ex) { - GenericTestUtils.assertExceptionContains("Block Iterator reached end " + - "for ContainerID " + containerID, ex); - } - } - - @Test - public void testKeyValueBlockIteratorWithNextBlock() throws Exception { - long containerID = 101L; - createContainerWithBlocks(containerID, 2, 0); - String containerPath = new File(containerData.getMetadataPath()) - .getParent(); - KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator( - containerID, new File(containerPath)); - long blockID = 0L; - assertEquals(blockID++, keyValueBlockIterator.nextBlock().getLocalID()); - assertEquals(blockID, keyValueBlockIterator.nextBlock().getLocalID()); - - try { - keyValueBlockIterator.nextBlock(); - } catch (NoSuchElementException ex) { - GenericTestUtils.assertExceptionContains("Block Iterator reached end " + - "for ContainerID " + containerID, ex); - } - } - - @Test - public void testKeyValueBlockIteratorWithHasNext() throws Exception { - long containerID = 102L; - createContainerWithBlocks(containerID, 2, 0); - String containerPath = new File(containerData.getMetadataPath()) - .getParent(); - KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator( - containerID, new File(containerPath)); - long blockID = 0L; - - // Even calling multiple times hasNext() should not move entry forward. - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertEquals(blockID++, keyValueBlockIterator.nextBlock().getLocalID()); - - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertTrue(keyValueBlockIterator.hasNext()); - assertEquals(blockID, keyValueBlockIterator.nextBlock().getLocalID()); - - keyValueBlockIterator.seekToLast(); - assertTrue(keyValueBlockIterator.hasNext()); - assertEquals(blockID, keyValueBlockIterator.nextBlock().getLocalID()); - - keyValueBlockIterator.seekToFirst(); - blockID = 0L; - assertEquals(blockID++, keyValueBlockIterator.nextBlock().getLocalID()); - assertEquals(blockID, keyValueBlockIterator.nextBlock().getLocalID()); - - try { - keyValueBlockIterator.nextBlock(); - } catch (NoSuchElementException ex) { - GenericTestUtils.assertExceptionContains("Block Iterator reached end " + - "for ContainerID " + containerID, ex); - } - - - } - - @Test - public void testKeyValueBlockIteratorWithFilter() throws Exception { - long containerId = 103L; - int deletedBlocks = 5; - int normalBlocks = 5; - createContainerWithBlocks(containerId, normalBlocks, deletedBlocks); - String containerPath = new File(containerData.getMetadataPath()) - .getParent(); - KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator( - containerId, new File(containerPath), MetadataKeyFilters - .getDeletingKeyFilter()); - - int counter = 5; - while(keyValueBlockIterator.hasNext()) { - BlockData blockData = keyValueBlockIterator.nextBlock(); - assertEquals(blockData.getLocalID(), counter++); - } - } - - @Test - public void testKeyValueBlockIteratorWithOnlyDeletedBlocks() throws - Exception { - long containerId = 104L; - createContainerWithBlocks(containerId, 0, 5); - String containerPath = new File(containerData.getMetadataPath()) - .getParent(); - KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator( - containerId, new File(containerPath)); - //As all blocks are deleted blocks, blocks does not match with normal key - // filter. - assertFalse(keyValueBlockIterator.hasNext()); - } - - /** - * Creates a container with specified number of normal blocks and deleted - * blocks. First it will insert normal blocks, and then it will insert - * deleted blocks. - * @param containerId - * @param normalBlocks - * @param deletedBlocks - * @throws Exception - */ - private void createContainerWithBlocks(long containerId, int - normalBlocks, int deletedBlocks) throws - Exception { - containerData = new KeyValueContainerData(containerId, - (long) StorageUnit.GB.toBytes(1)); - container = new KeyValueContainer(containerData, conf); - container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID - .randomUUID().toString()); - MetadataStore metadataStore = BlockUtils.getDB(containerData, conf); - - List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = new ChunkInfo("chunkfile", 0, 1024); - chunkList.add(info.getProtoBufMessage()); - - for (int i=0; i<normalBlocks; i++) { - BlockID blockID = new BlockID(containerId, i); - BlockData blockData = new BlockData(blockID); - blockData.setChunks(chunkList); - metadataStore.put(Longs.toByteArray(blockID.getLocalID()), blockData - .getProtoBufMessage().toByteArray()); - } - - for (int i=normalBlocks; i<deletedBlocks; i++) { - BlockID blockID = new BlockID(containerId, i); - BlockData blockData = new BlockData(blockID); - blockData.setChunks(chunkList); - metadataStore.put(DFSUtil.string2Bytes(OzoneConsts - .DELETING_KEY_PREFIX + blockID.getLocalID()), blockData - .getProtoBufMessage().toByteArray()); - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java deleted file mode 100644 index bf6b8b0..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ /dev/null @@ -1,374 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue; - -import com.google.common.primitives.Longs; -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerLifeCycleState; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.ozone.container.common.helpers.BlockData; -import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; -import org.apache.hadoop.ozone.container.common.volume.HddsVolume; -import org.apache.hadoop.ozone.container.common.volume - .RoundRobinVolumeChoosingPolicy; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; -import org.apache.hadoop.test.GenericTestUtils; -import org.apache.hadoop.util.DiskChecker; -import org.apache.hadoop.utils.MetadataStore; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import org.mockito.Mockito; - -import java.io.File; - -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; -import java.util.List; -import java.util.LinkedList; -import java.util.UUID; - -import static org.apache.ratis.util.Preconditions.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; - -/** - * Class to test KeyValue Container operations. - */ -public class TestKeyValueContainer { - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - - private OzoneConfiguration conf; - private String scmId = UUID.randomUUID().toString(); - private VolumeSet volumeSet; - private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; - private KeyValueContainerData keyValueContainerData; - private KeyValueContainer keyValueContainer; - - @Before - public void setUp() throws Exception { - conf = new OzoneConfiguration(); - HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(conf).datanodeUuid(UUID.randomUUID() - .toString()).build(); - - volumeSet = mock(VolumeSet.class); - volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); - Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) - .thenReturn(hddsVolume); - - keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); - - keyValueContainer = new KeyValueContainer( - keyValueContainerData, conf); - - } - - @Test - public void testBlockIterator() throws Exception{ - keyValueContainerData = new KeyValueContainerData(100L, - (long) StorageUnit.GB.toBytes(1)); - keyValueContainer = new KeyValueContainer( - keyValueContainerData, conf); - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - KeyValueBlockIterator blockIterator = keyValueContainer.blockIterator(); - //As no blocks created, hasNext should return false. - assertFalse(blockIterator.hasNext()); - int blockCount = 10; - addBlocks(blockCount); - blockIterator = keyValueContainer.blockIterator(); - assertTrue(blockIterator.hasNext()); - BlockData blockData; - int blockCounter = 0; - while(blockIterator.hasNext()) { - blockData = blockIterator.nextBlock(); - assertEquals(blockCounter++, blockData.getBlockID().getLocalID()); - } - assertEquals(blockCount, blockCounter); - } - - private void addBlocks(int count) throws Exception { - long containerId = keyValueContainerData.getContainerID(); - - MetadataStore metadataStore = BlockUtils.getDB(keyValueContainer - .getContainerData(), conf); - for (int i=0; i < count; i++) { - // Creating BlockData - BlockID blockID = new BlockID(containerId, i); - BlockData blockData = new BlockData(blockID); - blockData.addMetadata("VOLUME", "ozone"); - blockData.addMetadata("OWNER", "hdfs"); - List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, 1024); - chunkList.add(info.getProtoBufMessage()); - blockData.setChunks(chunkList); - metadataStore.put(Longs.toByteArray(blockID.getLocalID()), blockData - .getProtoBufMessage().toByteArray()); - } - - } - - @SuppressWarnings("RedundantCast") - @Test - public void testCreateContainer() throws Exception { - - // Create Container. - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - - keyValueContainerData = keyValueContainer - .getContainerData(); - - String containerMetaDataPath = keyValueContainerData - .getMetadataPath(); - String chunksPath = keyValueContainerData.getChunksPath(); - - // Check whether containerMetaDataPath and chunksPath exists or not. - assertTrue(containerMetaDataPath != null); - assertTrue(chunksPath != null); - File containerMetaDataLoc = new File(containerMetaDataPath); - - //Check whether container file and container db file exists or not. - assertTrue(keyValueContainer.getContainerFile().exists(), - ".Container File does not exist"); - assertTrue(keyValueContainer.getContainerDBFile().exists(), "Container " + - "DB does not exist"); - } - - @Test - public void testContainerImportExport() throws Exception { - - long containerId = keyValueContainer.getContainerData().getContainerID(); - // Create Container. - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - - - keyValueContainerData = keyValueContainer - .getContainerData(); - - keyValueContainerData.setState(ContainerLifeCycleState.CLOSED); - - int numberOfKeysToWrite = 12; - //write one few keys to check the key count after import - MetadataStore metadataStore = BlockUtils.getDB(keyValueContainerData, conf); - for (int i = 0; i < numberOfKeysToWrite; i++) { - metadataStore.put(("test" + i).getBytes(), "test".getBytes()); - } - metadataStore.close(); - - Map<String, String> metadata = new HashMap<>(); - metadata.put("key1", "value1"); - keyValueContainer.update(metadata, true); - - //destination path - File folderToExport = folder.newFile("exported.tar.gz"); - - TarContainerPacker packer = new TarContainerPacker(); - - //export the container - try (FileOutputStream fos = new FileOutputStream(folderToExport)) { - keyValueContainer - .exportContainerData(fos, packer); - } - - //delete the original one - keyValueContainer.delete(true); - - //create a new one - KeyValueContainerData containerData = - new KeyValueContainerData(containerId, 1, - keyValueContainerData.getMaxSize()); - KeyValueContainer container = new KeyValueContainer(containerData, conf); - - HddsVolume containerVolume = volumeChoosingPolicy.chooseVolume(volumeSet - .getVolumesList(), 1); - String hddsVolumeDir = containerVolume.getHddsRootDir().toString(); - - container.populatePathFields(scmId, containerVolume, hddsVolumeDir); - try (FileInputStream fis = new FileInputStream(folderToExport)) { - container.importContainerData(fis, packer); - } - - Assert.assertEquals("value1", containerData.getMetadata().get("key1")); - Assert.assertEquals(keyValueContainerData.getContainerDBType(), - containerData.getContainerDBType()); - Assert.assertEquals(keyValueContainerData.getState(), - containerData.getState()); - Assert.assertEquals(numberOfKeysToWrite, - containerData.getKeyCount()); - Assert.assertEquals(keyValueContainerData.getLayOutVersion(), - containerData.getLayOutVersion()); - Assert.assertEquals(keyValueContainerData.getMaxSize(), - containerData.getMaxSize()); - Assert.assertEquals(keyValueContainerData.getBytesUsed(), - containerData.getBytesUsed()); - - //Can't overwrite existing container - try { - try (FileInputStream fis = new FileInputStream(folderToExport)) { - container.importContainerData(fis, packer); - } - fail("Container is imported twice. Previous files are overwritten"); - } catch (Exception ex) { - //all good - } - - } - - @Test - public void testDuplicateContainer() throws Exception { - try { - // Create Container. - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - fail("testDuplicateContainer failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("ContainerFile already " + - "exists", ex); - assertEquals(ContainerProtos.Result.CONTAINER_ALREADY_EXISTS, ex - .getResult()); - } - } - - @Test - public void testDiskFullExceptionCreateContainer() throws Exception { - - Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) - .thenThrow(DiskChecker.DiskOutOfSpaceException.class); - try { - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - fail("testDiskFullExceptionCreateContainer failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("disk out of space", - ex); - assertEquals(ContainerProtos.Result.DISK_OUT_OF_SPACE, ex.getResult()); - } - } - - @Test - public void testDeleteContainer() throws Exception { - keyValueContainerData.setState(ContainerProtos.ContainerLifeCycleState - .CLOSED); - keyValueContainer = new KeyValueContainer( - keyValueContainerData, conf); - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - keyValueContainer.delete(true); - - String containerMetaDataPath = keyValueContainerData - .getMetadataPath(); - File containerMetaDataLoc = new File(containerMetaDataPath); - - assertFalse("Container directory still exists", containerMetaDataLoc - .getParentFile().exists()); - - assertFalse("Container File still exists", - keyValueContainer.getContainerFile().exists()); - assertFalse("Container DB file still exists", - keyValueContainer.getContainerDBFile().exists()); - } - - - @Test - public void testCloseContainer() throws Exception { - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - keyValueContainer.close(); - - keyValueContainerData = keyValueContainer - .getContainerData(); - - assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, - keyValueContainerData.getState()); - - //Check state in the .container file - String containerMetaDataPath = keyValueContainerData - .getMetadataPath(); - File containerFile = keyValueContainer.getContainerFile(); - - keyValueContainerData = (KeyValueContainerData) ContainerDataYaml - .readContainerFile(containerFile); - assertEquals(ContainerProtos.ContainerLifeCycleState.CLOSED, - keyValueContainerData.getState()); - } - - @Test - public void testUpdateContainer() throws IOException { - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - Map<String, String> metadata = new HashMap<>(); - metadata.put("VOLUME", "ozone"); - metadata.put("OWNER", "hdfs"); - keyValueContainer.update(metadata, true); - - keyValueContainerData = keyValueContainer - .getContainerData(); - - assertEquals(2, keyValueContainerData.getMetadata().size()); - - //Check metadata in the .container file - File containerFile = keyValueContainer.getContainerFile(); - - keyValueContainerData = (KeyValueContainerData) ContainerDataYaml - .readContainerFile(containerFile); - assertEquals(2, keyValueContainerData.getMetadata().size()); - - } - - @Test - public void testUpdateContainerUnsupportedRequest() throws Exception { - try { - keyValueContainerData.setState(ContainerProtos.ContainerLifeCycleState - .CLOSED); - keyValueContainer = new KeyValueContainer(keyValueContainerData, conf); - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - Map<String, String> metadata = new HashMap<>(); - metadata.put("VOLUME", "ozone"); - keyValueContainer.update(metadata, false); - fail("testUpdateContainerUnsupportedRequest failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Updating a closed container " + - "without force option is not allowed", ex); - assertEquals(ContainerProtos.Result.UNSUPPORTED_REQUEST, ex - .getResult()); - } - } - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java deleted file mode 100644 index e1904c1..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue; - - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.fs.FileUtil; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; -import org.apache.hadoop.ozone.container.common.impl.ContainerSet; -import org.apache.hadoop.ozone.container.common.impl.HddsDispatcher; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TestRule; -import org.junit.rules.Timeout; - -import org.mockito.Mockito; - -import static org.apache.hadoop.hdds.HddsConfigKeys - .HDDS_DATANODE_VOLUME_CHOOSING_POLICY; -import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY; -import static org.junit.Assert.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.doCallRealMethod; -import static org.mockito.Mockito.times; - - -import java.io.File; -import java.util.UUID; - -/** - * Unit tests for {@link KeyValueHandler}. - */ -public class TestKeyValueHandler { - - @Rule - public TestRule timeout = new Timeout(300000); - - private static HddsDispatcher dispatcher; - private static KeyValueHandler handler; - - private final static String DATANODE_UUID = UUID.randomUUID().toString(); - - private final String baseDir = MiniDFSCluster.getBaseDirectory(); - private final String volume = baseDir + "disk1"; - - private static final long DUMMY_CONTAINER_ID = 9999; - - @BeforeClass - public static void setup() throws StorageContainerException { - // Create mock HddsDispatcher and KeyValueHandler. - handler = Mockito.mock(KeyValueHandler.class); - dispatcher = Mockito.mock(HddsDispatcher.class); - Mockito.when(dispatcher.getHandler(any())).thenReturn(handler); - Mockito.when(dispatcher.dispatch(any())).thenCallRealMethod(); - Mockito.when(dispatcher.getContainer(anyLong())).thenReturn( - Mockito.mock(KeyValueContainer.class)); - Mockito.when(handler.handle(any(), any())).thenCallRealMethod(); - doCallRealMethod().when(dispatcher).setMetricsForTesting(any()); - dispatcher.setMetricsForTesting(Mockito.mock(ContainerMetrics.class)); - } - - @Test - /** - * Test that Handler handles different command types correctly. - */ - public void testHandlerCommandHandling() throws Exception { - - // Test Create Container Request handling - ContainerCommandRequestProto createContainerRequest = - ContainerProtos.ContainerCommandRequestProto.newBuilder() - .setCmdType(ContainerProtos.Type.CreateContainer) - .setContainerID(DUMMY_CONTAINER_ID) - .setDatanodeUuid(DATANODE_UUID) - .setCreateContainer(ContainerProtos.CreateContainerRequestProto - .getDefaultInstance()) - .build(); - dispatcher.dispatch(createContainerRequest); - Mockito.verify(handler, times(1)).handleCreateContainer( - any(ContainerCommandRequestProto.class), any()); - - // Test Read Container Request handling - ContainerCommandRequestProto readContainerRequest = - getDummyCommandRequestProto(ContainerProtos.Type.ReadContainer); - dispatcher.dispatch(readContainerRequest); - Mockito.verify(handler, times(1)).handleReadContainer( - any(ContainerCommandRequestProto.class), any()); - - // Test Update Container Request handling - ContainerCommandRequestProto updateContainerRequest = - getDummyCommandRequestProto(ContainerProtos.Type.UpdateContainer); - dispatcher.dispatch(updateContainerRequest); - Mockito.verify(handler, times(1)).handleUpdateContainer( - any(ContainerCommandRequestProto.class), any()); - - // Test Delete Container Request handling - ContainerCommandRequestProto deleteContainerRequest = - getDummyCommandRequestProto(ContainerProtos.Type.DeleteContainer); - dispatcher.dispatch(deleteContainerRequest); - Mockito.verify(handler, times(1)).handleDeleteContainer( - any(ContainerCommandRequestProto.class), any()); - - // Test List Container Request handling - ContainerCommandRequestProto listContainerRequest = - getDummyCommandRequestProto(ContainerProtos.Type.ListContainer); - dispatcher.dispatch(listContainerRequest); - Mockito.verify(handler, times(1)).handleUnsupportedOp( - any(ContainerCommandRequestProto.class)); - - // Test Close Container Request handling - ContainerCommandRequestProto closeContainerRequest = - getDummyCommandRequestProto(ContainerProtos.Type.CloseContainer); - dispatcher.dispatch(closeContainerRequest); - Mockito.verify(handler, times(1)).handleCloseContainer( - any(ContainerCommandRequestProto.class), any()); - - // Test Put Block Request handling - ContainerCommandRequestProto putBlockRequest = - getDummyCommandRequestProto(ContainerProtos.Type.PutBlock); - dispatcher.dispatch(putBlockRequest); - Mockito.verify(handler, times(1)).handlePutBlock( - any(ContainerCommandRequestProto.class), any()); - - // Test Get Block Request handling - ContainerCommandRequestProto getBlockRequest = - getDummyCommandRequestProto(ContainerProtos.Type.GetBlock); - dispatcher.dispatch(getBlockRequest); - Mockito.verify(handler, times(1)).handleGetBlock( - any(ContainerCommandRequestProto.class), any()); - - // Test Delete Block Request handling - ContainerCommandRequestProto deleteBlockRequest = - getDummyCommandRequestProto(ContainerProtos.Type.DeleteBlock); - dispatcher.dispatch(deleteBlockRequest); - Mockito.verify(handler, times(1)).handleDeleteBlock( - any(ContainerCommandRequestProto.class), any()); - - // Test List Block Request handling - ContainerCommandRequestProto listBlockRequest = - getDummyCommandRequestProto(ContainerProtos.Type.ListBlock); - dispatcher.dispatch(listBlockRequest); - Mockito.verify(handler, times(2)).handleUnsupportedOp( - any(ContainerCommandRequestProto.class)); - - // Test Read Chunk Request handling - ContainerCommandRequestProto readChunkRequest = - getDummyCommandRequestProto(ContainerProtos.Type.ReadChunk); - dispatcher.dispatch(readChunkRequest); - Mockito.verify(handler, times(1)).handleReadChunk( - any(ContainerCommandRequestProto.class), any()); - - // Test Delete Chunk Request handling - ContainerCommandRequestProto deleteChunkRequest = - getDummyCommandRequestProto(ContainerProtos.Type.DeleteChunk); - dispatcher.dispatch(deleteChunkRequest); - Mockito.verify(handler, times(1)).handleDeleteChunk( - any(ContainerCommandRequestProto.class), any()); - - // Test Write Chunk Request handling - ContainerCommandRequestProto writeChunkRequest = - getDummyCommandRequestProto(ContainerProtos.Type.WriteChunk); - dispatcher.dispatch(writeChunkRequest); - Mockito.verify(handler, times(1)).handleWriteChunk( - any(ContainerCommandRequestProto.class), any()); - - // Test List Chunk Request handling - ContainerCommandRequestProto listChunkRequest = - getDummyCommandRequestProto(ContainerProtos.Type.ListChunk); - dispatcher.dispatch(listChunkRequest); - Mockito.verify(handler, times(3)).handleUnsupportedOp( - any(ContainerCommandRequestProto.class)); - - // Test Put Small File Request handling - ContainerCommandRequestProto putSmallFileRequest = - getDummyCommandRequestProto(ContainerProtos.Type.PutSmallFile); - dispatcher.dispatch(putSmallFileRequest); - Mockito.verify(handler, times(1)).handlePutSmallFile( - any(ContainerCommandRequestProto.class), any()); - - // Test Get Small File Request handling - ContainerCommandRequestProto getSmallFileRequest = - getDummyCommandRequestProto(ContainerProtos.Type.GetSmallFile); - dispatcher.dispatch(getSmallFileRequest); - Mockito.verify(handler, times(1)).handleGetSmallFile( - any(ContainerCommandRequestProto.class), any()); - } - - @Test - public void testVolumeSetInKeyValueHandler() throws Exception{ - File path = GenericTestUtils.getRandomizedTestDir(); - try { - Configuration conf = new OzoneConfiguration(); - conf.set(HDDS_DATANODE_DIR_KEY, path.getAbsolutePath()); - ContainerSet cset = new ContainerSet(); - int[] interval = new int[1]; - interval[0] = 2; - ContainerMetrics metrics = new ContainerMetrics(interval); - VolumeSet volumeSet = new VolumeSet(UUID.randomUUID().toString(), conf); - KeyValueHandler keyValueHandler = new KeyValueHandler(conf, cset, - volumeSet, metrics); - assertEquals(keyValueHandler.getVolumeChoosingPolicyForTesting() - .getClass().getName(), "org.apache.hadoop.ozone.container.common" + - ".volume.RoundRobinVolumeChoosingPolicy"); - - //Set a class which is not of sub class of VolumeChoosingPolicy - conf.set(HDDS_DATANODE_VOLUME_CHOOSING_POLICY, - "org.apache.hadoop.ozone.container.common.impl.HddsDispatcher"); - try { - new KeyValueHandler(conf, cset, volumeSet, metrics); - } catch (RuntimeException ex) { - GenericTestUtils.assertExceptionContains("class org.apache.hadoop" + - ".ozone.container.common.impl.HddsDispatcher not org.apache" + - ".hadoop.ozone.container.common.interfaces.VolumeChoosingPolicy", - ex); - } - } finally { - FileUtil.fullyDelete(path); - } - } - - private ContainerCommandRequestProto getDummyCommandRequestProto( - ContainerProtos.Type cmdType) { - ContainerCommandRequestProto request = - ContainerProtos.ContainerCommandRequestProto.newBuilder() - .setCmdType(cmdType) - .setContainerID(DUMMY_CONTAINER_ID) - .setDatanodeUuid(DATANODE_UUID) - .build(); - - return request; - } - - - @Test - public void testCloseInvalidContainer() { - long containerID = 1234L; - Configuration conf = new Configuration(); - KeyValueContainerData kvData = new KeyValueContainerData(containerID, - (long) StorageUnit.GB.toBytes(1)); - KeyValueContainer container = new KeyValueContainer(kvData, conf); - kvData.setState(ContainerProtos.ContainerLifeCycleState.INVALID); - - // Create Close container request - ContainerCommandRequestProto closeContainerRequest = - ContainerProtos.ContainerCommandRequestProto.newBuilder() - .setCmdType(ContainerProtos.Type.CloseContainer) - .setContainerID(DUMMY_CONTAINER_ID) - .setDatanodeUuid(DATANODE_UUID) - .setCloseContainer(ContainerProtos.CloseContainerRequestProto - .getDefaultInstance()) - .build(); - dispatcher.dispatch(closeContainerRequest); - - Mockito.when(handler.handleCloseContainer(any(), any())) - .thenCallRealMethod(); - // Closing invalid container should return error response. - ContainerProtos.ContainerCommandResponseProto response = - handler.handleCloseContainer(closeContainerRequest, container); - - Assert.assertTrue("Close container should return Invalid container error", - response.getResult().equals( - ContainerProtos.Result.INVALID_CONTAINER_STATE)); - } -} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
