http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java index fae4c49..e51c3f7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/impl/TestContainerPersistence.java @@ -19,8 +19,9 @@ package org.apache.hadoop.ozone.container.common.impl; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.RandomUtils; import org.apache.hadoop.fs.FileUtil; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; import org.apache.hadoop.hdfs.server.datanode.StorageLocation; import org.apache.hadoop.ozone.OzoneConfigKeys; @@ -28,6 +29,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.hdds.scm.TestUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; import org.apache.hadoop.ozone.container.common.helpers.ContainerData; @@ -36,6 +38,7 @@ import org.apache.hadoop.ozone.container.common.helpers.KeyData; import org.apache.hadoop.ozone.container.common.helpers.KeyUtils; import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; +import org.apache.hadoop.util.Time; import org.apache.hadoop.utils.MetadataStore; import org.junit.After; import org.junit.AfterClass; @@ -177,19 +180,21 @@ public class TestContainerPersistence { } } + private long getTestContainerID() { + return ContainerTestHelper.getTestContainerID(); + } + @Test public void testCreateContainer() throws Exception { - - String containerName = OzoneUtils.getRequestID(); - ContainerData data = new ContainerData(containerName, containerID++, conf); + long testContainerID = getTestContainerID(); + ContainerData data = new ContainerData(testContainerID, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); - containerManager.createContainer(createSingleNodePipeline(containerName), - data); + containerManager.createContainer(data); Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName)); + .containsKey(testContainerID)); ContainerStatus status = containerManager - .getContainerMap().get(containerName); + .getContainerMap().get(testContainerID); Assert.assertNotNull(status.getContainer()); Assert.assertNotNull(status.getContainer().getContainerPath()); @@ -215,16 +220,14 @@ public class TestContainerPersistence { @Test public void testCreateDuplicateContainer() throws Exception { - String containerName = OzoneUtils.getRequestID(); + long testContainerID = getTestContainerID(); - ContainerData data = new ContainerData(containerName, containerID++, conf); + ContainerData data = new ContainerData(testContainerID, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); - containerManager.createContainer(createSingleNodePipeline(containerName), - data); + containerManager.createContainer(data); try { - containerManager.createContainer(createSingleNodePipeline( - containerName), data); + containerManager.createContainer(data); fail("Expected Exception not thrown."); } catch (IOException ex) { Assert.assertNotNull(ex); @@ -233,85 +236,76 @@ public class TestContainerPersistence { @Test public void testDeleteContainer() throws Exception { - String containerName1 = OzoneUtils.getRequestID(); - String containerName2 = OzoneUtils.getRequestID(); - + long testContainerID1 = getTestContainerID(); + Thread.sleep(100); + long testContainerID2 = getTestContainerID(); - ContainerData data = new ContainerData(containerName1, containerID++, conf); + ContainerData data = new ContainerData(testContainerID1, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); - containerManager.createContainer(createSingleNodePipeline(containerName1), - data); - containerManager.closeContainer(containerName1); + containerManager.createContainer(data); + containerManager.closeContainer(testContainerID1); - data = new ContainerData(containerName2, containerID++, conf); + data = new ContainerData(testContainerID2, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); - containerManager.createContainer(createSingleNodePipeline(containerName2), - data); - containerManager.closeContainer(containerName2); + containerManager.createContainer(data); + containerManager.closeContainer(testContainerID2); Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName1)); + .containsKey(testContainerID1)); Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName2)); + .containsKey(testContainerID2)); - containerManager.deleteContainer(createSingleNodePipeline(containerName1), - containerName1, false); + containerManager.deleteContainer(testContainerID1, false); Assert.assertFalse(containerManager.getContainerMap() - .containsKey(containerName1)); + .containsKey(testContainerID1)); // Let us make sure that we are able to re-use a container name after // delete. - data = new ContainerData(containerName1, containerID++, conf); + data = new ContainerData(testContainerID1, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); - containerManager.createContainer(createSingleNodePipeline(containerName1), - data); - containerManager.closeContainer(containerName1); + containerManager.createContainer(data); + containerManager.closeContainer(testContainerID1); // Assert we still have both containers. Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName1)); + .containsKey(testContainerID1)); Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName2)); + .containsKey(testContainerID2)); // Add some key to a container and then delete. // Delete should fail because the container is no longer empty. - KeyData someKey = new KeyData(containerName1, "someKey"); + BlockID blockID1 = ContainerTestHelper.getTestBlockID(testContainerID1); + KeyData someKey = new KeyData(blockID1); someKey.setChunks(new LinkedList<ContainerProtos.ChunkInfo>()); - keyManager.putKey( - createSingleNodePipeline(containerName1), - someKey); + keyManager.putKey(someKey); exception.expect(StorageContainerException.class); exception.expectMessage( "Container cannot be deleted because it is not empty."); - containerManager.deleteContainer( - createSingleNodePipeline(containerName1), - containerName1, false); + containerManager.deleteContainer(testContainerID1, false); Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName1)); + .containsKey(testContainerID1)); } @Test public void testGetContainerReports() throws Exception{ final int count = 10; - List<String> containerNames = new ArrayList<String>(); + List<Long> containerIDs = new ArrayList<>(); for (int i = 0; i < count; i++) { - String containerName = OzoneUtils.getRequestID(); - ContainerData data = new ContainerData(containerName, containerID++, - conf); - containerManager.createContainer(createSingleNodePipeline(containerName), - data); + long testContainerID = getTestContainerID(); + ContainerData data = new ContainerData(testContainerID, conf); + containerManager.createContainer(data); // Close a bunch of containers. // Put closed container names to a list. if (i%3 == 0) { - containerManager.closeContainer(containerName); - containerNames.add(containerName); + containerManager.closeContainer(testContainerID); + containerIDs.add(testContainerID); } } @@ -319,10 +313,10 @@ public class TestContainerPersistence { List<ContainerData> reports = containerManager.getContainerReports(); Assert.assertEquals(4, reports.size()); for(ContainerData report : reports) { - String actualName = report.getContainerName(); - Assert.assertTrue(containerNames.remove(actualName)); + long actualContainerID = report.getContainerID(); + Assert.assertTrue(containerIDs.remove(actualContainerID)); } - Assert.assertTrue(containerNames.isEmpty()); + Assert.assertTrue(containerIDs.isEmpty()); } /** @@ -336,31 +330,29 @@ public class TestContainerPersistence { final int count = 50; final int step = 5; - Map<String, ContainerData> testMap = new HashMap<>(); + Map<Long, ContainerData> testMap = new HashMap<>(); for (int x = 0; x < count; x++) { - String containerName = OzoneUtils.getRequestID(); - ContainerData data = new ContainerData(containerName, containerID++, - conf); + long testContainerID = getTestContainerID(); + ContainerData data = new ContainerData(testContainerID, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner)", "bilbo"); - containerManager.createContainer(createSingleNodePipeline(containerName), - data); - testMap.put(containerName, data); + containerManager.createContainer(data); + testMap.put(testContainerID, data); } int counter = 0; - String prevKey = ""; + long prevKey = 0; List<ContainerData> results = new LinkedList<>(); while (counter < count) { - containerManager.listContainer(null, step, prevKey, results); + containerManager.listContainer(prevKey, step, results); for (int y = 0; y < results.size(); y++) { - testMap.remove(results.get(y).getContainerName()); + testMap.remove(results.get(y).getContainerID()); } counter += step; - String nextKey = results.get(results.size() - 1).getContainerName(); + long nextKey = results.get(results.size() - 1).getContainerID(); //Assert that container is returning results in a sorted fashion. - Assert.assertTrue(prevKey.compareTo(nextKey) < 0); + Assert.assertTrue(prevKey < nextKey); prevKey = nextKey; results.clear(); } @@ -369,23 +361,22 @@ public class TestContainerPersistence { Assert.assertTrue(testMap.isEmpty()); } - private ChunkInfo writeChunkHelper(String containerName, String keyName, + private ChunkInfo writeChunkHelper(BlockID blockID, Pipeline pipeline) throws IOException, NoSuchAlgorithmException { final int datalen = 1024; - Pipeline newPipeline = - new Pipeline(containerName, pipeline.getPipelineChannel()); - ContainerData cData = new ContainerData(containerName, containerID++, conf); + long testContainerID = blockID.getContainerID(); + ContainerData cData = new ContainerData(testContainerID, conf); cData.addMetadata("VOLUME", "shire"); cData.addMetadata("owner", "bilbo"); if(!containerManager.getContainerMap() - .containsKey(containerName)) { - containerManager.createContainer(newPipeline, cData); + .containsKey(testContainerID)) { + containerManager.createContainer(cData); } - ChunkInfo info = getChunk(keyName, 0, 0, datalen); + ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen); byte[] data = getData(datalen); setDataChecksum(info, data); - chunkManager.writeChunk(newPipeline, keyName, info, data, COMBINED); + chunkManager.writeChunk(blockID, info, data, COMBINED); return info; } @@ -399,10 +390,10 @@ public class TestContainerPersistence { @Test public void testWriteChunk() throws IOException, NoSuchAlgorithmException { - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); - writeChunkHelper(containerName, keyName, pipeline); + BlockID blockID = ContainerTestHelper. + getTestBlockID(getTestContainerID()); + Pipeline pipeline = createSingleNodePipeline(); + writeChunkHelper(blockID, pipeline); } /** @@ -418,29 +409,30 @@ public class TestContainerPersistence { final int datalen = 1024; final int chunkCount = 1024; - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); + long testContainerID = getTestContainerID(); Map<String, ChunkInfo> fileHashMap = new HashMap<>(); - ContainerData cData = new ContainerData(containerName, containerID++, conf); + ContainerData cData = new ContainerData(testContainerID, conf); cData.addMetadata("VOLUME", "shire"); cData.addMetadata("owner)", "bilbo"); - containerManager.createContainer(pipeline, cData); + containerManager.createContainer(cData); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); + for (int x = 0; x < chunkCount; x++) { - ChunkInfo info = getChunk(keyName, x, 0, datalen); + ChunkInfo info = getChunk(blockID.getLocalID(), x, 0, datalen); byte[] data = getData(datalen); setDataChecksum(info, data); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); - String fileName = String.format("%s.data.%d", keyName, x); + chunkManager.writeChunk(blockID, info, data, COMBINED); + String fileName = String.format("%s.data.%d", blockID.getLocalID(), x); fileHashMap.put(fileName, info); } - ContainerData cNewData = containerManager.readContainer(containerName); + ContainerData cNewData = containerManager.readContainer(testContainerID); Assert.assertNotNull(cNewData); Path dataDir = ContainerUtils.getDataDirectory(cNewData); - String globFormat = String.format("%s.data.*", keyName); + String globFormat = String.format("%s.data.*", blockID.getLocalID()); MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH); // Read chunk via file system and verify. @@ -460,9 +452,9 @@ public class TestContainerPersistence { // Read chunk via ReadChunk call. sha.reset(); for (int x = 0; x < chunkCount; x++) { - String fileName = String.format("%s.data.%d", keyName, x); + String fileName = String.format("%s.data.%d", blockID.getLocalID(), x); ChunkInfo info = fileHashMap.get(fileName); - byte[] data = chunkManager.readChunk(pipeline, keyName, info); + byte[] data = chunkManager.readChunk(blockID, info); sha.update(data); Assert.assertEquals(Hex.encodeHexString(sha.digest()), info.getChecksum()); @@ -482,24 +474,24 @@ public class TestContainerPersistence { final int start = datalen/4; final int length = datalen/2; - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); - ContainerData cData = new ContainerData(containerName, containerID++, conf); + ContainerData cData = new ContainerData(testContainerID, conf); cData.addMetadata("VOLUME", "shire"); cData.addMetadata("owner)", "bilbo"); - containerManager.createContainer(pipeline, cData); - ChunkInfo info = getChunk(keyName, 0, 0, datalen); + containerManager.createContainer(cData); + ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen); byte[] data = getData(datalen); setDataChecksum(info, data); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); + chunkManager.writeChunk(blockID, info, data, COMBINED); - byte[] readData = chunkManager.readChunk(pipeline, keyName, info); + byte[] readData = chunkManager.readChunk(blockID, info); assertTrue(Arrays.equals(data, readData)); - ChunkInfo info2 = getChunk(keyName, 0, start, length); - byte[] readData2 = chunkManager.readChunk(pipeline, keyName, info2); + ChunkInfo info2 = getChunk(blockID.getLocalID(), 0, start, length); + byte[] readData2 = chunkManager.readChunk(blockID, info2); assertEquals(length, readData2.length); assertTrue(Arrays.equals( Arrays.copyOfRange(data, start, start + length), readData2)); @@ -516,20 +508,21 @@ public class TestContainerPersistence { public void testOverWrite() throws IOException, NoSuchAlgorithmException { final int datalen = 1024; - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); - ContainerData cData = new ContainerData(containerName, containerID++, conf); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); + + ContainerData cData = new ContainerData(testContainerID, conf); cData.addMetadata("VOLUME", "shire"); cData.addMetadata("owner)", "bilbo"); - containerManager.createContainer(pipeline, cData); - ChunkInfo info = getChunk(keyName, 0, 0, datalen); + containerManager.createContainer(cData); + ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen); byte[] data = getData(datalen); setDataChecksum(info, data); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); + chunkManager.writeChunk(blockID, info, data, COMBINED); try { - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); + chunkManager.writeChunk(blockID, info, data, COMBINED); } catch (IOException ex) { Assert.assertTrue(ex.getCause().getMessage().contains( "Rejecting write chunk request. OverWrite flag required")); @@ -537,11 +530,11 @@ public class TestContainerPersistence { // With the overwrite flag it should work now. info.addMetadata(OzoneConsts.CHUNK_OVERWRITE, "true"); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); - long bytesUsed = containerManager.getBytesUsed(containerName); + chunkManager.writeChunk(blockID, info, data, COMBINED); + long bytesUsed = containerManager.getBytesUsed(testContainerID); Assert.assertEquals(datalen, bytesUsed); - long bytesWrite = containerManager.getWriteBytes(containerName); + long bytesWrite = containerManager.getWriteBytes(testContainerID); Assert.assertEquals(datalen * 2, bytesWrite); } @@ -558,28 +551,28 @@ public class TestContainerPersistence { final int datalen = 1024; final int chunkCount = 1024; - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); - ContainerData cData = new ContainerData(containerName, containerID++, conf); + ContainerData cData = new ContainerData(testContainerID, conf); cData.addMetadata("VOLUME", "shire"); cData.addMetadata("owner)", "bilbo"); - containerManager.createContainer(pipeline, cData); + containerManager.createContainer(cData); MessageDigest oldSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH); for (int x = 0; x < chunkCount; x++) { // we are writing to the same chunk file but at different offsets. long offset = x * datalen; - ChunkInfo info = getChunk(keyName, 0, offset, datalen); + ChunkInfo info = getChunk(blockID.getLocalID(), 0, offset, datalen); byte[] data = getData(datalen); oldSha.update(data); setDataChecksum(info, data); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); + chunkManager.writeChunk(blockID, info, data, COMBINED); } // Request to read the whole data in a single go. - ChunkInfo largeChunk = getChunk(keyName, 0, 0, datalen * chunkCount); - byte[] newdata = chunkManager.readChunk(pipeline, keyName, largeChunk); + ChunkInfo largeChunk = getChunk(blockID.getLocalID(), 0, 0, datalen * chunkCount); + byte[] newdata = chunkManager.readChunk(blockID, largeChunk); MessageDigest newSha = MessageDigest.getInstance(OzoneConsts.FILE_HASH); newSha.update(newdata); Assert.assertEquals(Hex.encodeHexString(oldSha.digest()), @@ -596,22 +589,22 @@ public class TestContainerPersistence { public void testDeleteChunk() throws IOException, NoSuchAlgorithmException { final int datalen = 1024; - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); - ContainerData cData = new ContainerData(containerName, containerID++, conf); + ContainerData cData = new ContainerData(testContainerID, conf); cData.addMetadata("VOLUME", "shire"); cData.addMetadata("owner)", "bilbo"); - containerManager.createContainer(pipeline, cData); - ChunkInfo info = getChunk(keyName, 0, 0, datalen); + containerManager.createContainer(cData); + ChunkInfo info = getChunk(blockID.getLocalID(), 0, 0, datalen); byte[] data = getData(datalen); setDataChecksum(info, data); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); - chunkManager.deleteChunk(pipeline, keyName, info); + chunkManager.writeChunk(blockID, info, data, COMBINED); + chunkManager.deleteChunk(blockID, info); exception.expect(StorageContainerException.class); exception.expectMessage("Unable to find the chunk file."); - chunkManager.readChunk(pipeline, keyName, info); + chunkManager.readChunk(blockID, info); } /** @@ -622,15 +615,16 @@ public class TestContainerPersistence { */ @Test public void testPutKey() throws IOException, NoSuchAlgorithmException { - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); - ChunkInfo info = writeChunkHelper(containerName, keyName, pipeline); - KeyData keyData = new KeyData(containerName, keyName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); + Pipeline pipeline = createSingleNodePipeline(); + ChunkInfo info = writeChunkHelper(blockID, pipeline); + KeyData keyData = new KeyData(blockID); List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); chunkList.add(info.getProtoBufMessage()); keyData.setChunks(chunkList); - keyManager.putKey(pipeline, keyData); + keyManager.putKey(keyData); KeyData readKeyData = keyManager.getKey(keyData); ChunkInfo readChunk = ChunkInfo.getFromProtoBuf(readKeyData.getChunks().get(0)); @@ -649,39 +643,40 @@ public class TestContainerPersistence { final int chunkCount = 2; final int datalen = 1024; long totalSize = 0L; - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); + Pipeline pipeline = createSingleNodePipeline(); List<ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = writeChunkHelper(containerName, keyName, pipeline); + ChunkInfo info = writeChunkHelper(blockID, pipeline); totalSize += datalen; chunkList.add(info); for (int x = 1; x < chunkCount; x++) { // with holes in the front (before x * datalen) - info = getChunk(keyName, x, x * datalen, datalen); + info = getChunk(blockID.getLocalID(), x, x * datalen, datalen); byte[] data = getData(datalen); setDataChecksum(info, data); - chunkManager.writeChunk(pipeline, keyName, info, data, COMBINED); + chunkManager.writeChunk(blockID, info, data, COMBINED); totalSize += datalen * (x + 1); chunkList.add(info); } - long bytesUsed = containerManager.getBytesUsed(containerName); + long bytesUsed = containerManager.getBytesUsed(testContainerID); Assert.assertEquals(totalSize, bytesUsed); - long writeBytes = containerManager.getWriteBytes(containerName); + long writeBytes = containerManager.getWriteBytes(testContainerID); Assert.assertEquals(chunkCount * datalen, writeBytes); - long readCount = containerManager.getReadCount(containerName); + long readCount = containerManager.getReadCount(testContainerID); Assert.assertEquals(0, readCount); - long writeCount = containerManager.getWriteCount(containerName); + long writeCount = containerManager.getWriteCount(testContainerID); Assert.assertEquals(chunkCount, writeCount); - KeyData keyData = new KeyData(containerName, keyName); + KeyData keyData = new KeyData(blockID); List<ContainerProtos.ChunkInfo> chunkProtoList = new LinkedList<>(); for (ChunkInfo i : chunkList) { chunkProtoList.add(i.getProtoBufMessage()); } keyData.setChunks(chunkProtoList); - keyManager.putKey(pipeline, keyData); + keyManager.putKey(keyData); KeyData readKeyData = keyManager.getKey(keyData); ChunkInfo lastChunk = chunkList.get(chunkList.size() - 1); ChunkInfo readChunk = @@ -698,16 +693,16 @@ public class TestContainerPersistence { */ @Test public void testDeleteKey() throws IOException, NoSuchAlgorithmException { - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); - ChunkInfo info = writeChunkHelper(containerName, keyName, pipeline); - KeyData keyData = new KeyData(containerName, keyName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper.getTestBlockID(testContainerID); + Pipeline pipeline = createSingleNodePipeline(); + ChunkInfo info = writeChunkHelper(blockID, pipeline); + KeyData keyData = new KeyData(blockID); List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); chunkList.add(info.getProtoBufMessage()); keyData.setChunks(chunkList); - keyManager.putKey(pipeline, keyData); - keyManager.deleteKey(pipeline, keyName); + keyManager.putKey(keyData); + keyManager.deleteKey(blockID); exception.expect(StorageContainerException.class); exception.expectMessage("Unable to find the key."); keyManager.getKey(keyData); @@ -722,19 +717,20 @@ public class TestContainerPersistence { @Test public void testDeleteKeyTwice() throws IOException, NoSuchAlgorithmException { - String containerName = OzoneUtils.getRequestID(); - String keyName = OzoneUtils.getRequestID(); - Pipeline pipeline = createSingleNodePipeline(containerName); - ChunkInfo info = writeChunkHelper(containerName, keyName, pipeline); - KeyData keyData = new KeyData(containerName, keyName); + long testContainerID = getTestContainerID(); + BlockID blockID = ContainerTestHelper. + getTestBlockID(testContainerID); + Pipeline pipeline = createSingleNodePipeline(); + ChunkInfo info = writeChunkHelper(blockID, pipeline); + KeyData keyData = new KeyData(blockID); List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); chunkList.add(info.getProtoBufMessage()); keyData.setChunks(chunkList); - keyManager.putKey(pipeline, keyData); - keyManager.deleteKey(pipeline, keyName); + keyManager.putKey(keyData); + keyManager.deleteKey(blockID); exception.expect(StorageContainerException.class); exception.expectMessage("Unable to find the key."); - keyManager.deleteKey(pipeline, keyName); + keyManager.deleteKey(blockID); } /** @@ -745,35 +741,30 @@ public class TestContainerPersistence { */ @Test public void testUpdateContainer() throws IOException { - String containerName = OzoneUtils.getRequestID(); - ContainerData data = new ContainerData(containerName, containerID++, conf); + long testContainerID = ContainerTestHelper. + getTestContainerID(); + ContainerData data = new ContainerData(testContainerID, conf); data.addMetadata("VOLUME", "shire"); data.addMetadata("owner", "bilbo"); - containerManager.createContainer( - createSingleNodePipeline(containerName), - data); + containerManager.createContainer(data); File orgContainerFile = containerManager.getContainerFile(data); Assert.assertTrue(orgContainerFile.exists()); - ContainerData newData = new ContainerData(containerName, containerID++, - conf); + ContainerData newData = new ContainerData(testContainerID, conf); newData.addMetadata("VOLUME", "shire_new"); newData.addMetadata("owner", "bilbo_new"); - containerManager.updateContainer( - createSingleNodePipeline(containerName), - containerName, - newData, false); + containerManager.updateContainer(testContainerID, newData, false); Assert.assertEquals(1, containerManager.getContainerMap().size()); Assert.assertTrue(containerManager.getContainerMap() - .containsKey(containerName)); + .containsKey(testContainerID)); // Verify in-memory map ContainerData actualNewData = containerManager.getContainerMap() - .get(containerName).getContainer(); + .get(testContainerID).getContainer(); Assert.assertEquals("shire_new", actualNewData.getAllMetadata().get("VOLUME")); Assert.assertEquals("bilbo_new", @@ -802,23 +793,21 @@ public class TestContainerPersistence { // Delete container file then try to update without force update flag. FileUtil.fullyDelete(newContainerFile); try { - containerManager.updateContainer(createSingleNodePipeline(containerName), - containerName, newData, false); + containerManager.updateContainer(testContainerID, newData, false); } catch (StorageContainerException ex) { Assert.assertEquals("Container file not exists or " - + "corrupted. Name: " + containerName, ex.getMessage()); + + "corrupted. ID: " + testContainerID, ex.getMessage()); } // Update with force flag, it should be success. - newData = new ContainerData(containerName, containerID++, conf); + newData = new ContainerData(testContainerID, conf); newData.addMetadata("VOLUME", "shire_new_1"); newData.addMetadata("owner", "bilbo_new_1"); - containerManager.updateContainer(createSingleNodePipeline(containerName), - containerName, newData, true); + containerManager.updateContainer(testContainerID, newData, true); // Verify in-memory map actualNewData = containerManager.getContainerMap() - .get(containerName).getContainer(); + .get(testContainerID).getContainer(); Assert.assertEquals("shire_new_1", actualNewData.getAllMetadata().get("VOLUME")); Assert.assertEquals("bilbo_new_1", @@ -827,16 +816,14 @@ public class TestContainerPersistence { // Update a non-existing container exception.expect(StorageContainerException.class); exception.expectMessage("Container doesn't exist."); - containerManager.updateContainer( - createSingleNodePipeline("non_exist_container"), - "non_exist_container", newData, false); + containerManager.updateContainer(RandomUtils.nextLong(), + newData, false); } - private KeyData writeKeyHelper(Pipeline pipeline, - String containerName, String keyName) + private KeyData writeKeyHelper(Pipeline pipeline, BlockID blockID) throws IOException, NoSuchAlgorithmException { - ChunkInfo info = writeChunkHelper(containerName, keyName, pipeline); - KeyData keyData = new KeyData(containerName, keyName); + ChunkInfo info = writeChunkHelper(blockID, pipeline); + KeyData keyData = new KeyData(blockID); List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); chunkList.add(info.getProtoBufMessage()); keyData.setChunks(chunkList); @@ -845,61 +832,43 @@ public class TestContainerPersistence { @Test public void testListKey() throws Exception { - String containerName = "c0" + RandomStringUtils.randomAlphanumeric(10); - Pipeline pipeline = createSingleNodePipeline(containerName); - List<String> expectedKeys = new ArrayList<String>(); + + long testContainerID = getTestContainerID(); + Pipeline pipeline = createSingleNodePipeline(); + List<BlockID> expectedKeys = new ArrayList<>(); for (int i = 0; i < 10; i++) { - String keyName = "k" + i + "-" + UUID.randomUUID(); - expectedKeys.add(keyName); - KeyData kd = writeKeyHelper(pipeline, containerName, keyName); - keyManager.putKey(pipeline, kd); + BlockID blockID = new BlockID( + testContainerID, i); + expectedKeys.add(blockID); + KeyData kd = writeKeyHelper(pipeline, blockID); + keyManager.putKey(kd); } // List all keys - List<KeyData> result = keyManager.listKey(pipeline, null, null, 100); + List<KeyData> result = keyManager.listKey(testContainerID, 0, 100); Assert.assertEquals(10, result.size()); int index = 0; for (int i = index; i < result.size(); i++) { KeyData data = result.get(i); - Assert.assertEquals(containerName, data.getContainerName()); - Assert.assertEquals(expectedKeys.get(i), data.getKeyName()); + Assert.assertEquals(testContainerID, data.getContainerID()); + Assert.assertEquals(expectedKeys.get(i).getLocalID(), data.getLocalID()); index++; } - // List key with prefix - result = keyManager.listKey(pipeline, "k1", null, 100); - // There is only one key with prefix k1 - Assert.assertEquals(1, result.size()); - Assert.assertEquals(expectedKeys.get(1), result.get(0).getKeyName()); - - // List key with startKey filter - String k6 = expectedKeys.get(6); - result = keyManager.listKey(pipeline, null, k6, 100); + long k6 = expectedKeys.get(6).getLocalID(); + result = keyManager.listKey(testContainerID, k6, 100); Assert.assertEquals(4, result.size()); for (int i = 6; i < 10; i++) { - Assert.assertEquals(expectedKeys.get(i), - result.get(i - 6).getKeyName()); - } - - // List key with both prefix and startKey filter - String k7 = expectedKeys.get(7); - result = keyManager.listKey(pipeline, "k3", k7, 100); - // k3 is after k7, enhance we get an empty result - Assert.assertTrue(result.isEmpty()); - - // Set a pretty small cap for the key count - result = keyManager.listKey(pipeline, null, null, 3); - Assert.assertEquals(3, result.size()); - for (int i = 0; i < 3; i++) { - Assert.assertEquals(expectedKeys.get(i), result.get(i).getKeyName()); + Assert.assertEquals(expectedKeys.get(i).getLocalID(), + result.get(i - 6).getLocalID()); } // Count must be >0 exception.expect(IllegalArgumentException.class); exception.expectMessage("Count must be a positive number."); - keyManager.listKey(pipeline, null, null, -1); + keyManager.listKey(testContainerID, 0, -1); } }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java index 0034e8e..fbe43d7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestCloseContainerHandler.java @@ -79,32 +79,32 @@ public class TestCloseContainerHandler { cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions() .get(0).getBlocksLatestVersionOnly().get(0); - String containerName = ksmKeyLocationInfo.getContainerName(); + long containerID = ksmKeyLocationInfo.getContainerID(); - Assert.assertFalse(isContainerClosed(cluster, containerName)); + Assert.assertFalse(isContainerClosed(cluster, containerID)); DatanodeDetails datanodeDetails = cluster.getHddsDatanodes().get(0) .getDatanodeDetails(); //send the order to close the container cluster.getStorageContainerManager().getScmNodeManager() .addDatanodeCommand(datanodeDetails.getUuid(), - new CloseContainerCommand(containerName)); + new CloseContainerCommand(containerID)); - GenericTestUtils.waitFor(() -> isContainerClosed(cluster, containerName), + GenericTestUtils.waitFor(() -> isContainerClosed(cluster, containerID), 500, 5 * 1000); //double check if it's really closed (waitFor also throws an exception) - Assert.assertTrue(isContainerClosed(cluster, containerName)); + Assert.assertTrue(isContainerClosed(cluster, containerID)); } private Boolean isContainerClosed(MiniOzoneCluster cluster, - String containerName) { + long containerID) { ContainerData containerData; try { containerData = cluster.getHddsDatanodes().get(0) .getDatanodeStateMachine().getContainer().getContainerManager() - .readContainer(containerName); + .readContainer(containerID); return !containerData.isOpen(); } catch (StorageContainerException e) { throw new AssertionError(e); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java index 1565fbc..0bba5c1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/metrics/TestContainerMetrics.java @@ -22,6 +22,7 @@ import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges; import static org.apache.hadoop.test.MetricsAsserts.getMetrics; import static org.mockito.Mockito.mock; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.DatanodeDetails; @@ -53,13 +54,13 @@ public class TestContainerMetrics { public void testContainerMetrics() throws Exception { XceiverServer server = null; XceiverClient client = null; - String containerName = OzoneUtils.getRequestID(); + long containerID = ContainerTestHelper.getTestContainerID(); String keyName = OzoneUtils.getRequestID(); try { final int interval = 1; Pipeline pipeline = ContainerTestHelper - .createSingleNodePipeline(containerName); + .createSingleNodePipeline(); OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader().getContainerPort()); @@ -72,12 +73,12 @@ public class TestContainerMetrics { ContainerManager containerManager = mock(ContainerManager.class); ChunkManager chunkManager = mock(ChunkManager.class); Mockito.doNothing().when(chunkManager).writeChunk( - Mockito.any(Pipeline.class), Mockito.anyString(), + Mockito.any(BlockID.class), Mockito.any(ChunkInfo.class), Mockito.any(byte[].class), Mockito.any(ContainerProtos.Stage.class)); Mockito.doReturn(chunkManager).when(containerManager).getChunkManager(); - Mockito.doReturn(true).when(containerManager).isOpen(containerName); + Mockito.doReturn(true).when(containerManager).isOpen(containerID); Dispatcher dispatcher = new Dispatcher(containerManager, conf); dispatcher.init(); @@ -90,16 +91,17 @@ public class TestContainerMetrics { // Create container ContainerCommandRequestProto request = ContainerTestHelper - .getCreateContainerRequest(containerName, pipeline); + .getCreateContainerRequest(containerID, pipeline); ContainerCommandResponseProto response = client.sendCommand(request); Assert.assertTrue(request.getTraceID().equals(response.getTraceID())); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); // Write Chunk + BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest( - pipeline, containerName, keyName, 1024); + pipeline, blockID, 1024); response = client.sendCommand(writeChunkRequest); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java index 4a6ca1d..4e1d14b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java @@ -18,17 +18,18 @@ package org.apache.hadoop.ozone.container.ozoneimpl; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.apache.hadoop.hdds.scm.TestUtils; -import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.hadoop.hdds.scm.XceiverClient; import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.test.GenericTestUtils; +import org.apache.hadoop.util.Time; import org.junit.Assert; import org.junit.Rule; import org.junit.Test; @@ -52,7 +53,7 @@ public class TestOzoneContainer { @Test public void testCreateOzoneContainer() throws Exception { - String containerName = OzoneUtils.getRequestID(); + long containerID = ContainerTestHelper.getTestContainerID(); OzoneConfiguration conf = newOzoneConfiguration(); OzoneContainer container = null; MiniOzoneCluster cluster = null; @@ -61,8 +62,7 @@ public class TestOzoneContainer { cluster.waitForClusterToBeReady(); // We don't start Ozone Container via data node, we will do it // independently in our test path. - Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline( - containerName); + Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader().getContainerPort()); conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, false); @@ -71,7 +71,7 @@ public class TestOzoneContainer { XceiverClient client = new XceiverClient(pipeline, conf); client.connect(); - createContainerForTesting(client, containerName); + createContainerForTesting(client, containerID); } finally { if (container != null) { container.stop(); @@ -93,13 +93,14 @@ public class TestOzoneContainer { public void testOzoneContainerViaDataNode() throws Exception { MiniOzoneCluster cluster = null; try { - String containerName = OzoneUtils.getRequestID(); + long containerID = + ContainerTestHelper.getTestContainerID(); OzoneConfiguration conf = newOzoneConfiguration(); // Start ozone container Via Datanode create. Pipeline pipeline = - ContainerTestHelper.createSingleNodePipeline(containerName); + ContainerTestHelper.createSingleNodePipeline(); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader().getContainerPort()); @@ -111,7 +112,7 @@ public class TestOzoneContainer { // This client talks to ozone container via datanode. XceiverClient client = new XceiverClient(pipeline, conf); - runTestOzoneContainerViaDataNode(containerName, client); + runTestOzoneContainerViaDataNode(containerID, client); } finally { if (cluster != null) { cluster.shutdown(); @@ -120,7 +121,7 @@ public class TestOzoneContainer { } static void runTestOzoneContainerViaDataNode( - String containerName, XceiverClientSpi client) throws Exception { + long testContainerID, XceiverClientSpi client) throws Exception { ContainerProtos.ContainerCommandRequestProto request, writeChunkRequest, putKeyRequest, updateRequest1, updateRequest2; @@ -129,12 +130,12 @@ public class TestOzoneContainer { try { client.connect(); - // Create container - createContainerForTesting(client, containerName); - writeChunkRequest = writeChunkForContainer(client, containerName, 1024); + Pipeline pipeline = client.getPipeline(); + createContainerForTesting(client, testContainerID); + writeChunkRequest = writeChunkForContainer(client, testContainerID, 1024); // Read Chunk - request = ContainerTestHelper.getReadChunkRequest(writeChunkRequest + request = ContainerTestHelper.getReadChunkRequest(pipeline, writeChunkRequest .getWriteChunk()); response = client.sendCommand(request); @@ -143,7 +144,7 @@ public class TestOzoneContainer { Assert.assertTrue(request.getTraceID().equals(response.getTraceID())); // Put Key - putKeyRequest = ContainerTestHelper.getPutKeyRequest(writeChunkRequest + putKeyRequest = ContainerTestHelper.getPutKeyRequest(pipeline, writeChunkRequest .getWriteChunk()); @@ -154,21 +155,21 @@ public class TestOzoneContainer { .assertTrue(putKeyRequest.getTraceID().equals(response.getTraceID())); // Get Key - request = ContainerTestHelper.getKeyRequest(putKeyRequest.getPutKey()); + request = ContainerTestHelper.getKeyRequest(pipeline, putKeyRequest.getPutKey()); response = client.sendCommand(request); ContainerTestHelper.verifyGetKey(request, response); // Delete Key request = - ContainerTestHelper.getDeleteKeyRequest(putKeyRequest.getPutKey()); + ContainerTestHelper.getDeleteKeyRequest(pipeline, putKeyRequest.getPutKey()); response = client.sendCommand(request); Assert.assertNotNull(response); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); Assert.assertTrue(request.getTraceID().equals(response.getTraceID())); //Delete Chunk - request = ContainerTestHelper.getDeleteChunkRequest(writeChunkRequest + request = ContainerTestHelper.getDeleteChunkRequest(pipeline, writeChunkRequest .getWriteChunk()); response = client.sendCommand(request); @@ -180,15 +181,17 @@ public class TestOzoneContainer { Map<String, String> containerUpdate = new HashMap<String, String>(); containerUpdate.put("container_updated_key", "container_updated_value"); updateRequest1 = ContainerTestHelper.getUpdateContainerRequest( - containerName, containerUpdate); + testContainerID, containerUpdate); updateResponse1 = client.sendCommand(updateRequest1); Assert.assertNotNull(updateResponse1); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); //Update an non-existing container + long nonExistingContinerID = + ContainerTestHelper.getTestContainerID(); updateRequest2 = ContainerTestHelper.getUpdateContainerRequest( - "non_exist_container", containerUpdate); + nonExistingContinerID, containerUpdate); updateResponse2 = client.sendCommand(updateRequest2); Assert.assertEquals(ContainerProtos.Result.CONTAINER_NOT_FOUND, updateResponse2.getResult()); @@ -211,9 +214,8 @@ public class TestOzoneContainer { .setRandomContainerPort(false) .build(); cluster.waitForClusterToBeReady(); - String containerName = client.getPipeline().getContainerName(); - - runTestBothGetandPutSmallFile(containerName, client); + long containerID = ContainerTestHelper.getTestContainerID(); + runTestBothGetandPutSmallFile(containerID, client); } finally { if (cluster != null) { cluster.shutdown(); @@ -222,16 +224,16 @@ public class TestOzoneContainer { } static void runTestBothGetandPutSmallFile( - String containerName, XceiverClientSpi client) throws Exception { + long containerID, XceiverClientSpi client) throws Exception { try { client.connect(); - createContainerForTesting(client, containerName); + createContainerForTesting(client, containerID); - String keyName = OzoneUtils.getRequestID(); + BlockID blockId = ContainerTestHelper.getTestBlockID(containerID); final ContainerProtos.ContainerCommandRequestProto smallFileRequest = ContainerTestHelper.getWriteSmallFileRequest( - client.getPipeline(), containerName, keyName, 1024); + client.getPipeline(), blockId, 1024); ContainerProtos.ContainerCommandResponseProto response = client.sendCommand(smallFileRequest); Assert.assertNotNull(response); @@ -239,7 +241,7 @@ public class TestOzoneContainer { .equals(response.getTraceID())); final ContainerProtos.ContainerCommandRequestProto getSmallFileRequest - = ContainerTestHelper.getReadSmallFileRequest( + = ContainerTestHelper.getReadSmallFileRequest(client.getPipeline(), smallFileRequest.getPutSmallFile().getKey()); response = client.sendCommand(getSmallFileRequest); Assert.assertArrayEquals( @@ -272,13 +274,13 @@ public class TestOzoneContainer { cluster.waitForClusterToBeReady(); client.connect(); - String containerName = client.getPipeline().getContainerName(); - createContainerForTesting(client, containerName); - writeChunkRequest = writeChunkForContainer(client, containerName, 1024); + long containerID = ContainerTestHelper.getTestContainerID(); + createContainerForTesting(client, containerID); + writeChunkRequest = writeChunkForContainer(client, containerID, 1024); - putKeyRequest = ContainerTestHelper.getPutKeyRequest(writeChunkRequest - .getWriteChunk()); + putKeyRequest = ContainerTestHelper.getPutKeyRequest(client.getPipeline(), + writeChunkRequest.getWriteChunk()); // Put key before closing. response = client.sendCommand(putKeyRequest); Assert.assertNotNull(response); @@ -288,7 +290,8 @@ public class TestOzoneContainer { putKeyRequest.getTraceID().equals(response.getTraceID())); // Close the contianer. - request = ContainerTestHelper.getCloseContainer(client.getPipeline()); + request = ContainerTestHelper.getCloseContainer( + client.getPipeline(), containerID); response = client.sendCommand(request); Assert.assertNotNull(response); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); @@ -307,8 +310,8 @@ public class TestOzoneContainer { writeChunkRequest.getTraceID().equals(response.getTraceID())); // Read chunk must work on a closed container. - request = ContainerTestHelper.getReadChunkRequest(writeChunkRequest - .getWriteChunk()); + request = ContainerTestHelper.getReadChunkRequest(client.getPipeline(), + writeChunkRequest.getWriteChunk()); response = client.sendCommand(request); Assert.assertNotNull(response); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); @@ -324,13 +327,15 @@ public class TestOzoneContainer { .assertTrue(putKeyRequest.getTraceID().equals(response.getTraceID())); // Get key must work on the closed container. - request = ContainerTestHelper.getKeyRequest(putKeyRequest.getPutKey()); + request = ContainerTestHelper.getKeyRequest(client.getPipeline(), + putKeyRequest.getPutKey()); response = client.sendCommand(request); ContainerTestHelper.verifyGetKey(request, response); // Delete Key must fail on a closed container. request = - ContainerTestHelper.getDeleteKeyRequest(putKeyRequest.getPutKey()); + ContainerTestHelper.getDeleteKeyRequest(client.getPipeline(), + putKeyRequest.getPutKey()); response = client.sendCommand(request); Assert.assertNotNull(response); Assert.assertEquals(ContainerProtos.Result.CLOSED_CONTAINER_IO, @@ -363,12 +368,12 @@ public class TestOzoneContainer { cluster.waitForClusterToBeReady(); client.connect(); - String containerName = client.getPipeline().getContainerName(); - createContainerForTesting(client, containerName); - writeChunkRequest = writeChunkForContainer(client, containerName, 1024); + long containerID = ContainerTestHelper.getTestContainerID(); + createContainerForTesting(client, containerID); + writeChunkRequest = writeChunkForContainer(client, containerID, 1024); - putKeyRequest = ContainerTestHelper.getPutKeyRequest(writeChunkRequest - .getWriteChunk()); + putKeyRequest = ContainerTestHelper.getPutKeyRequest(client.getPipeline(), + writeChunkRequest.getWriteChunk()); // Put key before deleting. response = client.sendCommand(putKeyRequest); Assert.assertNotNull(response); @@ -380,7 +385,7 @@ public class TestOzoneContainer { // Container cannot be deleted forcibly because // the container is not closed. request = ContainerTestHelper.getDeleteContainer( - client.getPipeline(), true); + client.getPipeline(), containerID, true); response = client.sendCommand(request); Assert.assertNotNull(response); @@ -389,7 +394,8 @@ public class TestOzoneContainer { Assert.assertTrue(request.getTraceID().equals(response.getTraceID())); // Close the container. - request = ContainerTestHelper.getCloseContainer(client.getPipeline()); + request = ContainerTestHelper.getCloseContainer( + client.getPipeline(), containerID); response = client.sendCommand(request); Assert.assertNotNull(response); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); @@ -397,7 +403,7 @@ public class TestOzoneContainer { // Container cannot be deleted because the container is not empty. request = ContainerTestHelper.getDeleteContainer( - client.getPipeline(), false); + client.getPipeline(), containerID, false); response = client.sendCommand(request); Assert.assertNotNull(response); @@ -408,7 +414,7 @@ public class TestOzoneContainer { // Container can be deleted forcibly because // it is closed and non-empty. request = ContainerTestHelper.getDeleteContainer( - client.getPipeline(), true); + client.getPipeline(), containerID, true); response = client.sendCommand(request); Assert.assertNotNull(response); @@ -430,19 +436,19 @@ public class TestOzoneContainer { // Runs a set of commands as Async calls and verifies that calls indeed worked // as expected. static void runAsyncTests( - String containerName, XceiverClientSpi client) throws Exception { + long containerID, XceiverClientSpi client) throws Exception { try { client.connect(); - createContainerForTesting(client, containerName); + createContainerForTesting(client, containerID); final List<CompletableFuture> computeResults = new LinkedList<>(); int requestCount = 1000; // Create a bunch of Async calls from this test. for(int x = 0; x <requestCount; x++) { - String keyName = OzoneUtils.getRequestID(); + BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); final ContainerProtos.ContainerCommandRequestProto smallFileRequest = ContainerTestHelper.getWriteSmallFileRequest( - client.getPipeline(), containerName, keyName, 1024); + client.getPipeline(), blockID, 1024); CompletableFuture<ContainerProtos.ContainerCommandResponseProto> response = client.sendCommandAsync(smallFileRequest); @@ -477,8 +483,8 @@ public class TestOzoneContainer { .setRandomContainerPort(false) .build(); cluster.waitForClusterToBeReady(); - String containerName = client.getPipeline().getContainerName(); - runAsyncTests(containerName, client); + long containerID = ContainerTestHelper.getTestContainerID(); + runAsyncTests(containerID, client); } finally { if (cluster != null) { cluster.shutdown(); @@ -502,8 +508,9 @@ public class TestOzoneContainer { client.connect(); // Send a request without traceId. + long containerID = ContainerTestHelper.getTestContainerID(); request = ContainerTestHelper - .getRequestWithoutTraceId(client.getPipeline()); + .getRequestWithoutTraceId(client.getPipeline(), containerID); client.sendCommand(request); Assert.fail("IllegalArgumentException expected"); } catch(IllegalArgumentException iae){ @@ -515,13 +522,11 @@ public class TestOzoneContainer { } } - private static XceiverClient createClientForTesting(OzoneConfiguration conf) throws Exception { - String containerName = OzoneUtils.getRequestID(); // Start ozone container Via Datanode create. Pipeline pipeline = - ContainerTestHelper.createSingleNodePipeline(containerName); + ContainerTestHelper.createSingleNodePipeline(); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader().getContainerPort()); @@ -530,11 +535,11 @@ public class TestOzoneContainer { } private static void createContainerForTesting(XceiverClientSpi client, - String containerName) throws Exception { + long containerID) throws Exception { // Create container ContainerProtos.ContainerCommandRequestProto request = - ContainerTestHelper.getCreateContainerRequest(containerName, - client.getPipeline()); + ContainerTestHelper.getCreateContainerRequest( + containerID, client.getPipeline()); ContainerProtos.ContainerCommandResponseProto response = client.sendCommand(request); Assert.assertNotNull(response); @@ -543,13 +548,12 @@ public class TestOzoneContainer { private static ContainerProtos.ContainerCommandRequestProto writeChunkForContainer(XceiverClientSpi client, - String containerName, int dataLen) throws Exception { + long containerID, int dataLen) throws Exception { // Write Chunk - final String keyName = OzoneUtils.getRequestID(); + BlockID blockID = ContainerTestHelper.getTestBlockID(containerID);; ContainerProtos.ContainerCommandRequestProto writeChunkRequest = ContainerTestHelper.getWriteChunkRequest(client.getPipeline(), - containerName, keyName, dataLen); - + blockID, dataLen); ContainerProtos.ContainerCommandResponseProto response = client.sendCommand(writeChunkRequest); Assert.assertNotNull(response); @@ -559,24 +563,20 @@ public class TestOzoneContainer { } static void runRequestWithoutTraceId( - String containerName, XceiverClientSpi client) throws Exception { + long containerID, XceiverClientSpi client) throws Exception { try { client.connect(); - - createContainerForTesting(client, containerName); - - String keyName = OzoneUtils.getRequestID(); + createContainerForTesting(client, containerID); + BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); final ContainerProtos.ContainerCommandRequestProto smallFileRequest = ContainerTestHelper.getWriteSmallFileRequest( - client.getPipeline(), containerName, keyName, 1024); + client.getPipeline(), blockID, 1024); ContainerProtos.ContainerCommandResponseProto response = client.sendCommand(smallFileRequest); Assert.assertNotNull(response); Assert.assertTrue(smallFileRequest.getTraceID() .equals(response.getTraceID())); - - } finally { if (client != null) { client.close(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java index 9c10b28..c686b0b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerRatis.java @@ -67,7 +67,7 @@ public class TestOzoneContainerRatis { private static void runTest( String testName, RpcType rpc, int numNodes, - CheckedBiConsumer<String, XceiverClientSpi, Exception> test) + CheckedBiConsumer<Long, XceiverClientSpi, Exception> test) throws Exception { LOG.info(testName + "(rpc=" + rpc + ", numNodes=" + numNodes); @@ -84,7 +84,6 @@ public class TestOzoneContainerRatis { final String containerName = OzoneUtils.getRequestID(); final List<HddsDatanodeService> datanodes = cluster.getHddsDatanodes(); final Pipeline pipeline = ContainerTestHelper.createPipeline( - containerName, CollectionUtils.as(datanodes, HddsDatanodeService::getDatanodeDetails)); LOG.info("pipeline=" + pipeline); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java index 651b10f..b207914 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestContainerServer.java @@ -77,8 +77,9 @@ public class TestContainerServer { channel = new EmbeddedChannel(new XceiverServerHandler( new TestContainerDispatcher())); ContainerCommandRequestProto request = - ContainerTestHelper.getCreateContainerRequest(containerName, - ContainerTestHelper.createSingleNodePipeline(containerName)); + ContainerTestHelper.getCreateContainerRequest( + ContainerTestHelper.getTestContainerID(), + ContainerTestHelper.createSingleNodePipeline()); channel.writeInbound(request); Assert.assertTrue(channel.finish()); @@ -165,8 +166,7 @@ public class TestContainerServer { XceiverClientSpi client = null; String containerName = OzoneUtils.getRequestID(); try { - final Pipeline pipeline = ContainerTestHelper.createPipeline( - containerName, numDatanodes); + final Pipeline pipeline = ContainerTestHelper.createPipeline(numDatanodes); final OzoneConfiguration conf = new OzoneConfiguration(); initConf.accept(pipeline, conf); @@ -182,7 +182,8 @@ public class TestContainerServer { final ContainerCommandRequestProto request = ContainerTestHelper - .getCreateContainerRequest(containerName, pipeline); + .getCreateContainerRequest( + ContainerTestHelper.getTestContainerID(), pipeline); Assert.assertNotNull(request.getTraceID()); ContainerCommandResponseProto response = client.sendCommand(request); @@ -202,8 +203,7 @@ public class TestContainerServer { String containerName = OzoneUtils.getRequestID(); try { - Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline( - containerName); + Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(); OzoneConfiguration conf = new OzoneConfiguration(); conf.setInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT, pipeline.getLeader().getContainerPort()); @@ -219,8 +219,8 @@ public class TestContainerServer { client.connect(); ContainerCommandRequestProto request = - ContainerTestHelper.getCreateContainerRequest(containerName, - pipeline); + ContainerTestHelper.getCreateContainerRequest( + ContainerTestHelper.getTestContainerID(), pipeline); ContainerCommandResponseProto response = client.sendCommand(request); Assert.assertTrue(request.getTraceID().equals(response.getTraceID())); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java index a2a04e0..18482d1 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestContainerReportWithKeys.java @@ -117,24 +117,24 @@ public class TestContainerReportWithKeys { cluster.getKeySpaceManager().lookupKey(keyArgs).getKeyLocationVersions() .get(0).getBlocksLatestVersionOnly().get(0); - ContainerData cd = getContainerData(keyInfo.getContainerName()); + ContainerData cd = getContainerData(keyInfo.getContainerID()); LOG.info("DN Container Data: keyCount: {} used: {} ", cd.getKeyCount(), cd.getBytesUsed()); - ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerName()); + ContainerInfo cinfo = scm.getContainerInfo(keyInfo.getContainerID()); LOG.info("SCM Container Info keyCount: {} usedBytes: {}", cinfo.getNumberOfKeys(), cinfo.getUsedBytes()); } - private static ContainerData getContainerData(String containerName) { + private static ContainerData getContainerData(long containerID) { ContainerData containerData; try { ContainerManager containerManager = cluster.getHddsDatanodes().get(0) .getDatanodeStateMachine().getContainer().getContainerManager(); - containerData = containerManager.readContainer(containerName); + containerData = containerManager.readContainer(containerID); } catch (StorageContainerException e) { throw new AssertionError(e); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java index ae0ffa0..6478e88 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKeySpaceManager.java @@ -641,14 +641,6 @@ public class TestKeySpaceManager { new MetadataKeyFilters.KeyPrefixFilter(DELETING_KEY_PREFIX)); Assert.assertEquals(1, list.size()); - // Check the block key in SCM, make sure it's deleted. - Set<String> keys = new HashSet<>(); - keys.add(keyArgs.getResourceName()); - exception.expect(IOException.class); - exception.expectMessage("Specified block key does not exist"); - cluster.getStorageContainerManager().getBlockProtocolServer() - .getBlockLocations(keys); - // Delete the key again to test deleting non-existing key. exception.expect(IOException.class); exception.expectMessage("KEY_NOT_FOUND"); http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java index 34bbaf6..addd87b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/ksm/TestKsmBlockVersioning.java @@ -164,7 +164,7 @@ public class TestKsmBlockVersioning { // all the blocks from the previous version must present in the next // version for (KsmKeyLocationInfo info2 : version.getLocationList()) { - if (info.getBlockID().equals(info2.getBlockID())) { + if (info.getLocalID() == info2.getLocalID()) { found = true; break; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java index 275fadb..ef6fd5f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestAllocateContainer.java @@ -18,6 +18,7 @@ package org.apache.hadoop.ozone.scm; import org.apache.commons.lang.RandomStringUtils; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -67,12 +68,12 @@ public class TestAllocateContainer { @Test public void testAllocate() throws Exception { - Pipeline pipeline = storageContainerLocationClient.allocateContainer( + ContainerInfo container = storageContainerLocationClient.allocateContainer( xceiverClientManager.getType(), xceiverClientManager.getFactor(), - "container0", containerOwner); - Assert.assertNotNull(pipeline); - Assert.assertNotNull(pipeline.getLeader()); + containerOwner); + Assert.assertNotNull(container); + Assert.assertNotNull(container.getPipeline().getLeader()); } @@ -81,19 +82,6 @@ public class TestAllocateContainer { thrown.expect(NullPointerException.class); storageContainerLocationClient.allocateContainer( xceiverClientManager.getType(), - xceiverClientManager.getFactor(), null, containerOwner); - } - - @Test - public void testAllocateDuplicate() throws Exception { - String containerName = RandomStringUtils.randomAlphanumeric(10); - thrown.expect(IOException.class); - thrown.expectMessage("Specified container already exists"); - storageContainerLocationClient.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), containerName, containerOwner); - storageContainerLocationClient.allocateContainer( - xceiverClientManager.getType(), - xceiverClientManager.getFactor(), containerName, containerOwner); + xceiverClientManager.getFactor(), null); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java index a5d0eac..dabe903 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSQLCli.java @@ -88,10 +88,7 @@ public class TestContainerSQLCli { private NodeManager nodeManager; private BlockManagerImpl blockManager; - private Pipeline pipeline1; - private Pipeline pipeline2; - - private HashMap<String, String> blockContainerMap; + private HashMap<Long, Long> blockContainerMap; private final static long DEFAULT_BLOCK_SIZE = 4 * KB; private static HddsProtos.ReplicationFactor factor; @@ -124,7 +121,7 @@ public class TestContainerSQLCli { nodeManager = cluster.getStorageContainerManager().getScmNodeManager(); mapping = new ContainerMapping(conf, nodeManager, 128); - blockManager = new BlockManagerImpl(conf, nodeManager, mapping, 128); + blockManager = new BlockManagerImpl(conf, nodeManager, mapping); // blockManager.allocateBlock() will create containers if there is none // stored in levelDB. The number of containers to create is the value of @@ -142,8 +139,8 @@ public class TestContainerSQLCli { assertEquals(2, nodeManager.getAllNodes().size()); AllocatedBlock ab1 = blockManager.allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER); - pipeline1 = ab1.getPipeline(); - blockContainerMap.put(ab1.getKey(), pipeline1.getContainerName()); + blockContainerMap.put(ab1.getBlockID().getLocalID(), + ab1.getBlockID().getContainerID()); AllocatedBlock ab2; // we want the two blocks on the two provisioned containers respectively, @@ -155,9 +152,10 @@ public class TestContainerSQLCli { while (true) { ab2 = blockManager .allocateBlock(DEFAULT_BLOCK_SIZE, type, factor, CONTAINER_OWNER); - pipeline2 = ab2.getPipeline(); - blockContainerMap.put(ab2.getKey(), pipeline2.getContainerName()); - if (!pipeline1.getContainerName().equals(pipeline2.getContainerName())) { + blockContainerMap.put(ab2.getBlockID().getLocalID(), + ab2.getBlockID().getContainerID()); + if (ab1.getBlockID().getContainerID() != + ab2.getBlockID().getContainerID()) { break; } } @@ -250,25 +248,26 @@ public class TestContainerSQLCli { conn = connectDB(dbOutPath); sql = "SELECT * FROM containerInfo"; rs = executeQuery(conn, sql); - ArrayList<String> containerNames = new ArrayList<>(); + ArrayList<Long> containerIDs = new ArrayList<>(); while (rs.next()) { - containerNames.add(rs.getString("containerName")); + containerIDs.add(rs.getLong("containerID")); //assertEquals(dnUUID, rs.getString("leaderUUID")); } - assertTrue(containerNames.size() == 2 && - containerNames.contains(pipeline1.getContainerName()) && - containerNames.contains(pipeline2.getContainerName())); + /* TODO: fix this later when the SQLCLI is fixed. + assertTrue(containerIDs.size() == 2 && + containerIDs.contains(pipeline1.getContainerName()) && + containerIDs.contains(pipeline2.getContainerName())); sql = "SELECT * FROM containerMembers"; rs = executeQuery(conn, sql); - containerNames = new ArrayList<>(); + containerIDs = new ArrayList<>(); while (rs.next()) { - containerNames.add(rs.getString("containerName")); + containerIDs.add(rs.getLong("containerID")); //assertEquals(dnUUID, rs.getString("datanodeUUID")); } - assertTrue(containerNames.size() == 2 && - containerNames.contains(pipeline1.getContainerName()) && - containerNames.contains(pipeline2.getContainerName())); + assertTrue(containerIDs.size() == 2 && + containerIDs.contains(pipeline1.getContainerName()) && + containerIDs.contains(pipeline2.getContainerName())); sql = "SELECT * FROM datanodeInfo"; rs = executeQuery(conn, sql); @@ -282,6 +281,7 @@ public class TestContainerSQLCli { int expected = pipeline1.getLeader().getUuid().equals( pipeline2.getLeader().getUuid())? 1 : 2; assertEquals(expected, count); + */ Files.delete(Paths.get(dbOutPath)); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a43ac28/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java index d75b66c..f56d78c 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/TestContainerSmallFile.java @@ -17,7 +17,9 @@ */ package org.apache.hadoop.ozone.scm; +import org.apache.hadoop.hdds.client.BlockID; import org.apache.hadoop.hdds.protocol.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerInfo; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.hdds.conf.OzoneConfiguration; @@ -29,9 +31,9 @@ import org.apache.hadoop.hdds.scm.protocolPB .StorageContainerLocationProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.XceiverClientSpi; -import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; import org.apache.hadoop.hdds.scm.storage.ContainerProtocolCalls; +import org.apache.hadoop.ozone.container.ContainerTestHelper; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; @@ -79,19 +81,21 @@ public class TestContainerSmallFile { @Test public void testAllocateWrite() throws Exception { String traceID = UUID.randomUUID().toString(); - String containerName = "container0"; - Pipeline pipeline = + ContainerInfo container = storageContainerLocationClient.allocateContainer( xceiverClientManager.getType(), - HddsProtos.ReplicationFactor.ONE, containerName, containerOwner); - XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline); - ContainerProtocolCalls.createContainer(client, traceID); - - ContainerProtocolCalls.writeSmallFile(client, containerName, - "key", "data123".getBytes(), traceID); + HddsProtos.ReplicationFactor.ONE, containerOwner); + XceiverClientSpi client = xceiverClientManager.acquireClient( + container.getPipeline(), container.getContainerID()); + ContainerProtocolCalls.createContainer(client, + container.getContainerID(), traceID); + + BlockID blockID = ContainerTestHelper.getTestBlockID( + container.getContainerID()); + ContainerProtocolCalls.writeSmallFile(client, blockID, + "data123".getBytes(), traceID); ContainerProtos.GetSmallFileResponseProto response = - ContainerProtocolCalls.readSmallFile(client, containerName, "key", - traceID); + ContainerProtocolCalls.readSmallFile(client, blockID, traceID); String readData = response.getData().getData().toStringUtf8(); Assert.assertEquals("data123", readData); xceiverClientManager.releaseClient(client); @@ -100,37 +104,42 @@ public class TestContainerSmallFile { @Test public void testInvalidKeyRead() throws Exception { String traceID = UUID.randomUUID().toString(); - String containerName = "container1"; - Pipeline pipeline = + ContainerInfo container = storageContainerLocationClient.allocateContainer( xceiverClientManager.getType(), - HddsProtos.ReplicationFactor.ONE, containerName, containerOwner); - XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline); - ContainerProtocolCalls.createContainer(client, traceID); + HddsProtos.ReplicationFactor.ONE, containerOwner); + XceiverClientSpi client = xceiverClientManager.acquireClient( + container.getPipeline(), container.getContainerID()); + ContainerProtocolCalls.createContainer(client, + container.getContainerID(), traceID); thrown.expect(StorageContainerException.class); thrown.expectMessage("Unable to find the key"); + BlockID blockID = ContainerTestHelper.getTestBlockID( + container.getContainerID()); // Try to read a Key Container Name ContainerProtos.GetSmallFileResponseProto response = - ContainerProtocolCalls.readSmallFile(client, containerName, "key", - traceID); + ContainerProtocolCalls.readSmallFile(client, blockID, traceID); xceiverClientManager.releaseClient(client); } @Test public void testInvalidContainerRead() throws Exception { String traceID = UUID.randomUUID().toString(); - String invalidName = "invalidName"; - String containerName = "container2"; - Pipeline pipeline = + long nonExistContainerID = 8888L; + ContainerInfo container = storageContainerLocationClient.allocateContainer( xceiverClientManager.getType(), - HddsProtos.ReplicationFactor.ONE, containerName, containerOwner); - XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline); - ContainerProtocolCalls.createContainer(client, traceID); - ContainerProtocolCalls.writeSmallFile(client, containerName, - "key", "data123".getBytes(), traceID); + HddsProtos.ReplicationFactor.ONE, containerOwner); + XceiverClientSpi client = xceiverClientManager. + acquireClient(container.getPipeline(), container.getContainerID()); + ContainerProtocolCalls.createContainer(client, + container.getContainerID(), traceID); + BlockID blockID = ContainerTestHelper.getTestBlockID( + container.getContainerID()); + ContainerProtocolCalls.writeSmallFile(client, blockID, + "data123".getBytes(), traceID); thrown.expect(StorageContainerException.class); @@ -138,10 +147,13 @@ public class TestContainerSmallFile { // Try to read a invalid key ContainerProtos.GetSmallFileResponseProto response = - ContainerProtocolCalls.readSmallFile(client, invalidName, "key", - traceID); + ContainerProtocolCalls.readSmallFile(client, + ContainerTestHelper.getTestBlockID( + nonExistContainerID), traceID); xceiverClientManager.releaseClient(client); } + + } --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
