http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java index ed4536f..4f2b3a2 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerCommandResponseProto; import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.utils.MetadataKeyFilters; import org.apache.hadoop.utils.MetadataStore; @@ -116,7 +116,7 @@ public final class KeyValueContainerUtil { File chunksPath = new File(containerData.getChunksPath()); // Close the DB connection and remove the DB handler from cache - KeyUtils.removeDB(containerData, conf); + BlockUtils.removeDB(containerData, conf); // Delete the Container MetaData path. FileUtils.deleteDirectory(containerMetaDataPath); @@ -175,16 +175,16 @@ public final class KeyValueContainerUtil { } kvContainerData.setDbFile(dbFile); - MetadataStore metadata = KeyUtils.getDB(kvContainerData, config); + MetadataStore metadata = BlockUtils.getDB(kvContainerData, config); long bytesUsed = 0; List<Map.Entry<byte[], byte[]>> liveKeys = metadata .getRangeKVs(null, Integer.MAX_VALUE, MetadataKeyFilters.getNormalKeyFilter()); bytesUsed = liveKeys.parallelStream().mapToLong(e-> { - KeyData keyData; + BlockData blockData; try { - keyData = KeyUtils.getKeyData(e.getValue()); - return keyData.getSize(); + blockData = BlockUtils.getBlockData(e.getValue()); + return blockData.getSize(); } catch (IOException ex) { return 0L; }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java index df60c60..3495363 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/SmallFileUtils.java @@ -69,7 +69,7 @@ public final class SmallFileUtils { ContainerProtos.ReadChunkResponseProto.newBuilder(); readChunkresponse.setChunkData(info.getProtoBufMessage()); readChunkresponse.setData(ByteString.copyFrom(data)); - readChunkresponse.setBlockID(msg.getGetSmallFile().getKey().getBlockID()); + readChunkresponse.setBlockID(msg.getGetSmallFile().getBlock().getBlockID()); ContainerProtos.GetSmallFileResponseProto.Builder getSmallFile = ContainerProtos.GetSmallFileResponseProto.newBuilder(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java new file mode 100644 index 0000000..54c15fb --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.keyvalue.impl; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.Longs; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; + +import org.apache.hadoop.ozone.container.common.helpers.BlockData; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; +import org.apache.hadoop.ozone.container.common.interfaces.Container; +import org.apache.hadoop.ozone.container.keyvalue.interfaces.BlockManager; +import org.apache.hadoop.ozone.container.common.utils.ContainerCache; +import org.apache.hadoop.utils.MetadataStore; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_BLOCK; + +/** + * This class is for performing block related operations on the KeyValue + * Container. + */ +public class BlockManagerImpl implements BlockManager { + + static final Logger LOG = LoggerFactory.getLogger(BlockManagerImpl.class); + + private Configuration config; + + /** + * Constructs a Block Manager. + * + * @param conf - Ozone configuration + */ + public BlockManagerImpl(Configuration conf) { + Preconditions.checkNotNull(conf, "Config cannot be null"); + this.config = conf; + } + + /** + * Puts or overwrites a block. + * + * @param container - Container for which block need to be added. + * @param data - BlockData. + * @return length of the block. + * @throws IOException + */ + public long putBlock(Container container, BlockData data) throws IOException { + Preconditions.checkNotNull(data, "BlockData cannot be null for put " + + "operation."); + Preconditions.checkState(data.getContainerID() >= 0, "Container Id " + + "cannot be negative"); + // We are not locking the key manager since LevelDb serializes all actions + // against a single DB. We rely on DB level locking to avoid conflicts. + MetadataStore db = BlockUtils.getDB((KeyValueContainerData) container + .getContainerData(), config); + + // This is a post condition that acts as a hint to the user. + // Should never fail. + Preconditions.checkNotNull(db, "DB cannot be null here"); + db.put(Longs.toByteArray(data.getLocalID()), data.getProtoBufMessage() + .toByteArray()); + + // Increment keycount here + container.getContainerData().incrKeyCount(); + return data.getSize(); + } + + /** + * Gets an existing block. + * + * @param container - Container from which block need to be fetched. + * @param blockID - BlockID of the block. + * @return Key Data. + * @throws IOException + */ + public BlockData getBlock(Container container, BlockID blockID) + throws IOException { + Preconditions.checkNotNull(blockID, + "BlockID cannot be null in GetBlock request"); + Preconditions.checkNotNull(blockID.getContainerID(), + "Container name cannot be null"); + + KeyValueContainerData containerData = (KeyValueContainerData) container + .getContainerData(); + MetadataStore db = BlockUtils.getDB(containerData, config); + // This is a post condition that acts as a hint to the user. + // Should never fail. + Preconditions.checkNotNull(db, "DB cannot be null here"); + byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID())); + if (kData == null) { + throw new StorageContainerException("Unable to find the block.", + NO_SUCH_BLOCK); + } + ContainerProtos.BlockData blockData = + ContainerProtos.BlockData.parseFrom(kData); + return BlockData.getFromProtoBuf(blockData); + } + + /** + * Returns the length of the committed block. + * + * @param container - Container from which block need to be fetched. + * @param blockID - BlockID of the block. + * @return length of the block. + * @throws IOException in case, the block key does not exist in db. + */ + @Override + public long getCommittedBlockLength(Container container, BlockID blockID) + throws IOException { + KeyValueContainerData containerData = (KeyValueContainerData) container + .getContainerData(); + MetadataStore db = BlockUtils.getDB(containerData, config); + // This is a post condition that acts as a hint to the user. + // Should never fail. + Preconditions.checkNotNull(db, "DB cannot be null here"); + byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID())); + if (kData == null) { + throw new StorageContainerException("Unable to find the block.", + NO_SUCH_BLOCK); + } + ContainerProtos.BlockData blockData = + ContainerProtos.BlockData.parseFrom(kData); + return blockData.getSize(); + } + + /** + * Deletes an existing block. + * + * @param container - Container from which block need to be deleted. + * @param blockID - ID of the block. + * @throws StorageContainerException + */ + public void deleteBlock(Container container, BlockID blockID) throws + IOException { + Preconditions.checkNotNull(blockID, "block ID cannot be null."); + Preconditions.checkState(blockID.getContainerID() >= 0, + "Container ID cannot be negative."); + Preconditions.checkState(blockID.getLocalID() >= 0, + "Local ID cannot be negative."); + + KeyValueContainerData cData = (KeyValueContainerData) container + .getContainerData(); + MetadataStore db = BlockUtils.getDB(cData, config); + // This is a post condition that acts as a hint to the user. + // Should never fail. + Preconditions.checkNotNull(db, "DB cannot be null here"); + // Note : There is a race condition here, since get and delete + // are not atomic. Leaving it here since the impact is refusing + // to delete a Block which might have just gotten inserted after + // the get check. + byte[] kKey = Longs.toByteArray(blockID.getLocalID()); + byte[] kData = db.get(kKey); + if (kData == null) { + throw new StorageContainerException("Unable to find the block.", + NO_SUCH_BLOCK); + } + db.delete(kKey); + + // Decrement blockcount here + container.getContainerData().decrKeyCount(); + } + + /** + * List blocks in a container. + * + * @param container - Container from which blocks need to be listed. + * @param startLocalID - Key to start from, 0 to begin. + * @param count - Number of blocks to return. + * @return List of Blocks that match the criteria. + */ + @Override + public List<BlockData> listBlock(Container container, long startLocalID, int + count) throws IOException { + Preconditions.checkNotNull(container, "container cannot be null"); + Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be " + + "negative"); + Preconditions.checkArgument(count > 0, + "Count must be a positive number."); + container.readLock(); + List<BlockData> result = null; + KeyValueContainerData cData = (KeyValueContainerData) container + .getContainerData(); + MetadataStore db = BlockUtils.getDB(cData, config); + result = new ArrayList<>(); + byte[] startKeyInBytes = Longs.toByteArray(startLocalID); + List<Map.Entry<byte[], byte[]>> range = db.getSequentialRangeKVs( + startKeyInBytes, count, null); + for (Map.Entry<byte[], byte[]> entry : range) { + BlockData value = BlockUtils.getBlockData(entry.getValue()); + BlockData data = new BlockData(value.getBlockID()); + result.add(data); + } + return result; + } + + /** + * Shutdown KeyValueContainerManager. + */ + public void shutdown() { + BlockUtils.shutdownCache(ContainerCache.getInstance(config)); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java deleted file mode 100644 index 6370f8e..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/KeyManagerImpl.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue.impl; - -import com.google.common.base.Preconditions; -import com.google.common.primitives.Longs; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; - -import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; -import org.apache.hadoop.ozone.container.common.interfaces.Container; -import org.apache.hadoop.ozone.container.keyvalue.interfaces.KeyManager; -import org.apache.hadoop.ozone.container.common.utils.ContainerCache; -import org.apache.hadoop.utils.MetadataStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import static org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.Result.NO_SUCH_KEY; - -/** - * This class is for performing key related operations on the KeyValue - * Container. - */ -public class KeyManagerImpl implements KeyManager { - - static final Logger LOG = LoggerFactory.getLogger(KeyManagerImpl.class); - - private Configuration config; - - /** - * Constructs a key Manager. - * - * @param conf - Ozone configuration - */ - public KeyManagerImpl(Configuration conf) { - Preconditions.checkNotNull(conf, "Config cannot be null"); - this.config = conf; - } - - /** - * Puts or overwrites a key. - * - * @param container - Container for which key need to be added. - * @param data - Key Data. - * @return length of the key. - * @throws IOException - */ - public long putKey(Container container, KeyData data) throws IOException { - Preconditions.checkNotNull(data, "KeyData cannot be null for put " + - "operation."); - Preconditions.checkState(data.getContainerID() >= 0, "Container Id " + - "cannot be negative"); - // We are not locking the key manager since LevelDb serializes all actions - // against a single DB. We rely on DB level locking to avoid conflicts. - MetadataStore db = KeyUtils.getDB((KeyValueContainerData) container - .getContainerData(), config); - - // This is a post condition that acts as a hint to the user. - // Should never fail. - Preconditions.checkNotNull(db, "DB cannot be null here"); - db.put(Longs.toByteArray(data.getLocalID()), data.getProtoBufMessage() - .toByteArray()); - - // Increment keycount here - container.getContainerData().incrKeyCount(); - return data.getSize(); - } - - /** - * Gets an existing key. - * - * @param container - Container from which key need to be get. - * @param blockID - BlockID of the key. - * @return Key Data. - * @throws IOException - */ - public KeyData getKey(Container container, BlockID blockID) - throws IOException { - Preconditions.checkNotNull(blockID, - "BlockID cannot be null in GetKet request"); - Preconditions.checkNotNull(blockID.getContainerID(), - "Container name cannot be null"); - - KeyValueContainerData containerData = (KeyValueContainerData) container - .getContainerData(); - MetadataStore db = KeyUtils.getDB(containerData, config); - // This is a post condition that acts as a hint to the user. - // Should never fail. - Preconditions.checkNotNull(db, "DB cannot be null here"); - byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID())); - if (kData == null) { - throw new StorageContainerException("Unable to find the key.", - NO_SUCH_KEY); - } - ContainerProtos.KeyData keyData = ContainerProtos.KeyData.parseFrom(kData); - return KeyData.getFromProtoBuf(keyData); - } - - /** - * Returns the length of the committed block. - * - * @param container - Container from which key need to be get. - * @param blockID - BlockID of the key. - * @return length of the block. - * @throws IOException in case, the block key does not exist in db. - */ - @Override - public long getCommittedBlockLength(Container container, BlockID blockID) - throws IOException { - KeyValueContainerData containerData = (KeyValueContainerData) container - .getContainerData(); - MetadataStore db = KeyUtils.getDB(containerData, config); - // This is a post condition that acts as a hint to the user. - // Should never fail. - Preconditions.checkNotNull(db, "DB cannot be null here"); - byte[] kData = db.get(Longs.toByteArray(blockID.getLocalID())); - if (kData == null) { - throw new StorageContainerException("Unable to find the key.", - NO_SUCH_KEY); - } - ContainerProtos.KeyData keyData = ContainerProtos.KeyData.parseFrom(kData); - return keyData.getSize(); - } - - /** - * Deletes an existing Key. - * - * @param container - Container from which key need to be deleted. - * @param blockID - ID of the block. - * @throws StorageContainerException - */ - public void deleteKey(Container container, BlockID blockID) throws - IOException { - Preconditions.checkNotNull(blockID, "block ID cannot be null."); - Preconditions.checkState(blockID.getContainerID() >= 0, - "Container ID cannot be negative."); - Preconditions.checkState(blockID.getLocalID() >= 0, - "Local ID cannot be negative."); - - KeyValueContainerData cData = (KeyValueContainerData) container - .getContainerData(); - MetadataStore db = KeyUtils.getDB(cData, config); - // This is a post condition that acts as a hint to the user. - // Should never fail. - Preconditions.checkNotNull(db, "DB cannot be null here"); - // Note : There is a race condition here, since get and delete - // are not atomic. Leaving it here since the impact is refusing - // to delete a key which might have just gotten inserted after - // the get check. - byte[] kKey = Longs.toByteArray(blockID.getLocalID()); - byte[] kData = db.get(kKey); - if (kData == null) { - throw new StorageContainerException("Unable to find the key.", - NO_SUCH_KEY); - } - db.delete(kKey); - - // Decrement keycount here - container.getContainerData().decrKeyCount(); - } - - /** - * List keys in a container. - * - * @param container - Container from which keys need to be listed. - * @param startLocalID - Key to start from, 0 to begin. - * @param count - Number of keys to return. - * @return List of Keys that match the criteria. - */ - @Override - public List<KeyData> listKey(Container container, long startLocalID, int - count) throws IOException { - Preconditions.checkNotNull(container, "container cannot be null"); - Preconditions.checkState(startLocalID >= 0, "startLocal ID cannot be " + - "negative"); - Preconditions.checkArgument(count > 0, - "Count must be a positive number."); - container.readLock(); - List<KeyData> result = null; - KeyValueContainerData cData = (KeyValueContainerData) container - .getContainerData(); - MetadataStore db = KeyUtils.getDB(cData, config); - result = new ArrayList<>(); - byte[] startKeyInBytes = Longs.toByteArray(startLocalID); - List<Map.Entry<byte[], byte[]>> range = db.getSequentialRangeKVs( - startKeyInBytes, count, null); - for (Map.Entry<byte[], byte[]> entry : range) { - KeyData value = KeyUtils.getKeyData(entry.getValue()); - KeyData data = new KeyData(value.getBlockID()); - result.add(data); - } - return result; - } - - /** - * Shutdown KeyValueContainerManager. - */ - public void shutdown() { - KeyUtils.shutdownCache(ContainerCache.getInstance(config)); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java index 525d51b..564b50e 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/package-info.java @@ -17,6 +17,5 @@ */ package org.apache.hadoop.ozone.container.keyvalue.impl; /** - This package contains chunk manager and key manager implementation for - keyvalue container type. - **/ \ No newline at end of file + * Chunk manager and block manager implementations for keyvalue container type. + */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java new file mode 100644 index 0000000..35ed22a --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/BlockManager.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.apache.hadoop.ozone.container.keyvalue.interfaces; + +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; +import org.apache.hadoop.ozone.container.common.interfaces.Container; + +import java.io.IOException; +import java.util.List; + +/** + * BlockManager is for performing key related operations on the container. + */ +public interface BlockManager { + + /** + * Puts or overwrites a block. + * + * @param container - Container for which block need to be added. + * @param data - Block Data. + * @return length of the Block. + * @throws IOException + */ + long putBlock(Container container, BlockData data) throws IOException; + + /** + * Gets an existing block. + * + * @param container - Container from which block need to be get. + * @param blockID - BlockID of the Block. + * @return Block Data. + * @throws IOException + */ + BlockData getBlock(Container container, BlockID blockID) throws IOException; + + /** + * Deletes an existing block. + * + * @param container - Container from which block need to be deleted. + * @param blockID - ID of the block. + * @throws StorageContainerException + */ + void deleteBlock(Container container, BlockID blockID) throws IOException; + + /** + * List blocks in a container. + * + * @param container - Container from which blocks need to be listed. + * @param startLocalID - Block to start from, 0 to begin. + * @param count - Number of blocks to return. + * @return List of Blocks that match the criteria. + */ + List<BlockData> listBlock(Container container, long startLocalID, int count) + throws IOException; + + /** + * Returns the last committed block length for the block. + * @param blockID blockId + */ + long getCommittedBlockLength(Container container, BlockID blockID) + throws IOException; + + /** + * Shutdown ContainerManager. + */ + void shutdown(); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java deleted file mode 100644 index 84f771a..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/KeyManager.java +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue.interfaces; - -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; -import org.apache.hadoop.ozone.container.common.interfaces.Container; - -import java.io.IOException; -import java.util.List; - -/** - * KeyManager is for performing key related operations on the container. - */ -public interface KeyManager { - - /** - * Puts or overwrites a key. - * - * @param container - Container for which key need to be added. - * @param data - Key Data. - * @return length of the Key. - * @throws IOException - */ - long putKey(Container container, KeyData data) throws IOException; - - /** - * Gets an existing key. - * - * @param container - Container from which key need to be get. - * @param blockID - BlockID of the Key. - * @return Key Data. - * @throws IOException - */ - KeyData getKey(Container container, BlockID blockID) throws IOException; - - /** - * Deletes an existing Key. - * - * @param container - Container from which key need to be deleted. - * @param blockID - ID of the block. - * @throws StorageContainerException - */ - void deleteKey(Container container, BlockID blockID) throws IOException; - - /** - * List keys in a container. - * - * @param container - Container from which keys need to be listed. - * @param startLocalID - Key to start from, 0 to begin. - * @param count - Number of keys to return. - * @return List of Keys that match the criteria. - */ - List<KeyData> listKey(Container container, long startLocalID, int count) - throws IOException; - - /** - * Returns the last committed block length for the block. - * @param blockID blockId - */ - long getCommittedBlockLength(Container container, BlockID blockID) - throws IOException; - - /** - * Shutdown ContainerManager. - */ - void shutdown(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java new file mode 100644 index 0000000..5129094 --- /dev/null +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/interfaces/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.container.keyvalue.interfaces; +/** + * Chunk manager and block manager interfaces for keyvalue container type. + */ http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java index 51eed7f..d96fbfa 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java @@ -25,7 +25,7 @@ import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy; import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.util.ReflectionUtils; import org.apache.ratis.shaded.com.google.protobuf .InvalidProtocolBufferException; @@ -72,7 +72,7 @@ public class BlockDeletingService extends BackgroundService{ private static final Logger LOG = LoggerFactory.getLogger(BlockDeletingService.class); - ContainerSet containerSet; + private ContainerSet containerSet; private ContainerDeletionChoosingPolicy containerDeletionPolicy; private final Configuration conf; @@ -185,7 +185,7 @@ public class BlockDeletingService extends BackgroundService{ ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult(); long startTime = Time.monotonicNow(); // Scan container's db and get list of under deletion blocks - MetadataStore meta = KeyUtils.getDB( + MetadataStore meta = BlockUtils.getDB( (KeyValueContainerData) containerData, conf); // # of blocks to delete is throttled KeyPrefixFilter filter = @@ -211,8 +211,8 @@ public class BlockDeletingService extends BackgroundService{ String blockName = DFSUtil.bytes2String(entry.getKey()); LOG.debug("Deleting block {}", blockName); try { - ContainerProtos.KeyData data = - ContainerProtos.KeyData.parseFrom(entry.getValue()); + ContainerProtos.BlockData data = + ContainerProtos.BlockData.parseFrom(entry.getValue()); for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) { File chunkFile = dataDir.toPath() .resolve(chunkInfo.getChunkName()).toFile(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java new file mode 100644 index 0000000..6fe6d81 --- /dev/null +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestBlockManagerImpl.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.container.keyvalue; + +import org.apache.hadoop.conf.StorageUnit; +import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; +import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; +import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; +import org.apache.hadoop.ozone.container.common.volume + .RoundRobinVolumeChoosingPolicy; +import org.apache.hadoop.ozone.container.common.volume.VolumeSet; +import org.apache.hadoop.ozone.container.common.volume.HddsVolume; +import org.apache.hadoop.ozone.container.keyvalue.impl.BlockManagerImpl; +import org.apache.hadoop.test.GenericTestUtils; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; + +import static org.junit.Assert.*; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; + +/** + * This class is used to test key related operations on the container. + */ +public class TestBlockManagerImpl { + + private OzoneConfiguration config; + private String scmId = UUID.randomUUID().toString(); + private VolumeSet volumeSet; + private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; + private KeyValueContainerData keyValueContainerData; + private KeyValueContainer keyValueContainer; + private BlockData blockData; + private BlockManagerImpl blockManager; + private BlockID blockID; + + @Rule + public TemporaryFolder folder = new TemporaryFolder(); + + + @Before + public void setUp() throws Exception { + config = new OzoneConfiguration(); + + HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot() + .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID() + .toString()).build(); + + volumeSet = mock(VolumeSet.class); + + volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); + Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) + .thenReturn(hddsVolume); + + keyValueContainerData = new KeyValueContainerData(1L, + (long) StorageUnit.GB.toBytes(5)); + + keyValueContainer = new KeyValueContainer( + keyValueContainerData, config); + + keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); + + // Creating BlockData + blockID = new BlockID(1L, 1L); + blockData = new BlockData(blockID); + blockData.addMetadata("VOLUME", "ozone"); + blockData.addMetadata("OWNER", "hdfs"); + List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); + ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID + .getLocalID(), 0), 0, 1024); + chunkList.add(info.getProtoBufMessage()); + blockData.setChunks(chunkList); + + // Create KeyValueContainerManager + blockManager = new BlockManagerImpl(config); + + } + + @Test + public void testPutAndGetBlock() throws Exception { + assertEquals(0, keyValueContainer.getContainerData().getKeyCount()); + //Put Block + blockManager.putBlock(keyValueContainer, blockData); + + assertEquals(1, keyValueContainer.getContainerData().getKeyCount()); + //Get Block + BlockData fromGetBlockData = blockManager.getBlock(keyValueContainer, + blockData.getBlockID()); + + assertEquals(blockData.getContainerID(), fromGetBlockData.getContainerID()); + assertEquals(blockData.getLocalID(), fromGetBlockData.getLocalID()); + assertEquals(blockData.getChunks().size(), + fromGetBlockData.getChunks().size()); + assertEquals(blockData.getMetadata().size(), fromGetBlockData.getMetadata() + .size()); + + } + + + @Test + public void testDeleteBlock() throws Exception { + try { + assertEquals(0, + keyValueContainer.getContainerData().getKeyCount()); + //Put Block + blockManager.putBlock(keyValueContainer, blockData); + assertEquals(1, + keyValueContainer.getContainerData().getKeyCount()); + //Delete Block + blockManager.deleteBlock(keyValueContainer, blockID); + assertEquals(0, + keyValueContainer.getContainerData().getKeyCount()); + try { + blockManager.getBlock(keyValueContainer, blockID); + fail("testDeleteBlock"); + } catch (StorageContainerException ex) { + GenericTestUtils.assertExceptionContains( + "Unable to find the block", ex); + } + } catch (IOException ex) { + fail("testDeleteBlock failed"); + } + } + + @Test + public void testListBlock() throws Exception { + try { + blockManager.putBlock(keyValueContainer, blockData); + List<BlockData> listBlockData = blockManager.listBlock( + keyValueContainer, 1, 10); + assertNotNull(listBlockData); + assertTrue(listBlockData.size() == 1); + + for (long i = 2; i <= 10; i++) { + blockID = new BlockID(1L, i); + blockData = new BlockData(blockID); + blockData.addMetadata("VOLUME", "ozone"); + blockData.addMetadata("OWNER", "hdfs"); + List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); + ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID + .getLocalID(), 0), 0, 1024); + chunkList.add(info.getProtoBufMessage()); + blockData.setChunks(chunkList); + blockManager.putBlock(keyValueContainer, blockData); + } + + listBlockData = blockManager.listBlock( + keyValueContainer, 1, 10); + assertNotNull(listBlockData); + assertTrue(listBlockData.size() == 10); + + } catch (IOException ex) { + fail("testListBlock failed"); + } + } + + @Test + public void testGetNoSuchBlock() throws Exception { + try { + assertEquals(0, + keyValueContainer.getContainerData().getKeyCount()); + //Put Block + blockManager.putBlock(keyValueContainer, blockData); + assertEquals(1, + keyValueContainer.getContainerData().getKeyCount()); + //Delete Block + blockManager.deleteBlock(keyValueContainer, blockID); + assertEquals(0, + keyValueContainer.getContainerData().getKeyCount()); + try { + //Since the block has been deleted, we should not be able to find it + blockManager.getBlock(keyValueContainer, blockID); + fail("testGetNoSuchBlock failed"); + } catch (StorageContainerException ex) { + GenericTestUtils.assertExceptionContains( + "Unable to find the block", ex); + assertEquals(ContainerProtos.Result.NO_SUCH_BLOCK, ex.getResult()); + } + } catch (IOException ex) { + fail("testGetNoSuchBlock failed"); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java index 9664052..3c0876b 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestChunkManagerImpl.java @@ -88,7 +88,7 @@ public class TestChunkManagerImpl { keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); data = "testing write chunks".getBytes(); - // Creating KeyData + // Creating BlockData blockID = new BlockID(1L, 1L); chunkInfo = new ChunkInfo(String.format("%d.data.%d", blockID .getLocalID(), 0), 0, data.length); http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java deleted file mode 100644 index b05dbca..0000000 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyManagerImpl.java +++ /dev/null @@ -1,191 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.keyvalue; - -import org.apache.hadoop.conf.StorageUnit; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; -import org.apache.hadoop.ozone.container.common.volume - .RoundRobinVolumeChoosingPolicy; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.common.volume.HddsVolume; -import org.apache.hadoop.ozone.container.keyvalue.impl.KeyManagerImpl; -import org.apache.hadoop.test.GenericTestUtils; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import java.io.IOException; -import java.util.LinkedList; -import java.util.List; -import java.util.UUID; - -import static org.junit.Assert.*; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; - -/** - * This class is used to test key related operations on the container. - */ -public class TestKeyManagerImpl { - - private OzoneConfiguration config; - private String scmId = UUID.randomUUID().toString(); - private VolumeSet volumeSet; - private RoundRobinVolumeChoosingPolicy volumeChoosingPolicy; - private KeyValueContainerData keyValueContainerData; - private KeyValueContainer keyValueContainer; - private KeyData keyData; - private KeyManagerImpl keyManager; - private BlockID blockID; - - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - - @Before - public void setUp() throws Exception { - config = new OzoneConfiguration(); - - HddsVolume hddsVolume = new HddsVolume.Builder(folder.getRoot() - .getAbsolutePath()).conf(config).datanodeUuid(UUID.randomUUID() - .toString()).build(); - - volumeSet = mock(VolumeSet.class); - - volumeChoosingPolicy = mock(RoundRobinVolumeChoosingPolicy.class); - Mockito.when(volumeChoosingPolicy.chooseVolume(anyList(), anyLong())) - .thenReturn(hddsVolume); - - keyValueContainerData = new KeyValueContainerData(1L, - (long) StorageUnit.GB.toBytes(5)); - - keyValueContainer = new KeyValueContainer( - keyValueContainerData, config); - - keyValueContainer.create(volumeSet, volumeChoosingPolicy, scmId); - - // Creating KeyData - blockID = new BlockID(1L, 1L); - keyData = new KeyData(blockID); - keyData.addMetadata("VOLUME", "ozone"); - keyData.addMetadata("OWNER", "hdfs"); - List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, 1024); - chunkList.add(info.getProtoBufMessage()); - keyData.setChunks(chunkList); - - // Create KeyValueContainerManager - keyManager = new KeyManagerImpl(config); - - } - - @Test - public void testPutAndGetKey() throws Exception { - assertEquals(0, keyValueContainer.getContainerData().getKeyCount()); - //Put Key - keyManager.putKey(keyValueContainer, keyData); - - assertEquals(1, keyValueContainer.getContainerData().getKeyCount()); - //Get Key - KeyData fromGetKeyData = keyManager.getKey(keyValueContainer, - keyData.getBlockID()); - - assertEquals(keyData.getContainerID(), fromGetKeyData.getContainerID()); - assertEquals(keyData.getLocalID(), fromGetKeyData.getLocalID()); - assertEquals(keyData.getChunks().size(), fromGetKeyData.getChunks().size()); - assertEquals(keyData.getMetadata().size(), fromGetKeyData.getMetadata() - .size()); - - } - - - @Test - public void testDeleteKey() throws Exception { - try { - assertEquals(0, keyValueContainer.getContainerData().getKeyCount()); - //Put Key - keyManager.putKey(keyValueContainer, keyData); - assertEquals(1, keyValueContainer.getContainerData().getKeyCount()); - //Delete Key - keyManager.deleteKey(keyValueContainer, blockID); - assertEquals(0, keyValueContainer.getContainerData().getKeyCount()); - try { - keyManager.getKey(keyValueContainer, blockID); - fail("testDeleteKey"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Unable to find the key", ex); - } - } catch (IOException ex) { - fail("testDeleteKey failed"); - } - } - - @Test - public void testListKey() throws Exception { - try { - keyManager.putKey(keyValueContainer, keyData); - List<KeyData> listKeyData = keyManager.listKey( - keyValueContainer, 1, 10); - assertNotNull(listKeyData); - assertTrue(listKeyData.size() == 1); - - for (long i = 2; i <= 10; i++) { - blockID = new BlockID(1L, i); - keyData = new KeyData(blockID); - keyData.addMetadata("VOLUME", "ozone"); - keyData.addMetadata("OWNER", "hdfs"); - List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); - ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID - .getLocalID(), 0), 0, 1024); - chunkList.add(info.getProtoBufMessage()); - keyData.setChunks(chunkList); - keyManager.putKey(keyValueContainer, keyData); - } - - listKeyData = keyManager.listKey( - keyValueContainer, 1, 10); - assertNotNull(listKeyData); - assertTrue(listKeyData.size() == 10); - - } catch (IOException ex) { - fail("testListKey failed"); - } - } - - @Test - public void testGetNoSuchKey() throws Exception { - try { - keyData = new KeyData(new BlockID(1L, 2L)); - keyManager.getKey(keyValueContainer, new BlockID(1L, 2L)); - fail("testGetNoSuchKey failed"); - } catch (StorageContainerException ex) { - GenericTestUtils.assertExceptionContains("Unable to find the key.", ex); - assertEquals(ContainerProtos.Result.NO_SUCH_KEY, ex.getResult()); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java index f1fe88e..fbc5ad0 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java @@ -27,11 +27,11 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.utils.MetadataKeyFilters; import org.apache.hadoop.utils.MetadataStore; @@ -114,8 +114,8 @@ public class TestKeyValueBlockIterator { int counter = 0; while(keyValueBlockIterator.hasNext()) { - KeyData keyData = keyValueBlockIterator.nextBlock(); - assertEquals(keyData.getLocalID(), counter++); + BlockData blockData = keyValueBlockIterator.nextBlock(); + assertEquals(blockData.getLocalID(), counter++); } assertFalse(keyValueBlockIterator.hasNext()); @@ -123,8 +123,8 @@ public class TestKeyValueBlockIterator { keyValueBlockIterator.seekToFirst(); counter = 0; while(keyValueBlockIterator.hasNext()) { - KeyData keyData = keyValueBlockIterator.nextBlock(); - assertEquals(keyData.getLocalID(), counter++); + BlockData blockData = keyValueBlockIterator.nextBlock(); + assertEquals(blockData.getLocalID(), counter++); } assertFalse(keyValueBlockIterator.hasNext()); @@ -214,8 +214,8 @@ public class TestKeyValueBlockIterator { int counter = 5; while(keyValueBlockIterator.hasNext()) { - KeyData keyData = keyValueBlockIterator.nextBlock(); - assertEquals(keyData.getLocalID(), counter++); + BlockData blockData = keyValueBlockIterator.nextBlock(); + assertEquals(blockData.getLocalID(), counter++); } } @@ -250,7 +250,7 @@ public class TestKeyValueBlockIterator { container = new KeyValueContainer(containerData, conf); container.create(volumeSet, new RoundRobinVolumeChoosingPolicy(), UUID .randomUUID().toString()); - MetadataStore metadataStore = KeyUtils.getDB(containerData, conf); + MetadataStore metadataStore = BlockUtils.getDB(containerData, conf); List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); ChunkInfo info = new ChunkInfo("chunkfile", 0, 1024); @@ -258,18 +258,18 @@ public class TestKeyValueBlockIterator { for (int i=0; i<normalBlocks; i++) { BlockID blockID = new BlockID(containerId, i); - KeyData keyData = new KeyData(blockID); - keyData.setChunks(chunkList); - metadataStore.put(Longs.toByteArray(blockID.getLocalID()), keyData + BlockData blockData = new BlockData(blockID); + blockData.setChunks(chunkList); + metadataStore.put(Longs.toByteArray(blockID.getLocalID()), blockData .getProtoBufMessage().toByteArray()); } for (int i=normalBlocks; i<deletedBlocks; i++) { BlockID blockID = new BlockID(containerId, i); - KeyData keyData = new KeyData(blockID); - keyData.setChunks(chunkList); + BlockData blockData = new BlockData(blockID); + blockData.setChunks(chunkList); metadataStore.put(DFSUtil.string2Bytes(OzoneConsts - .DELETING_KEY_PREFIX + blockID.getLocalID()), keyData + .DELETING_KEY_PREFIX + blockID.getLocalID()), blockData .getProtoBufMessage().toByteArray()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java index f84ba7d..bf6b8b0 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java @@ -28,14 +28,14 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos .ContainerLifeCycleState; import org.apache.hadoop.hdds.scm.container.common.helpers .StorageContainerException; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; import org.apache.hadoop.ozone.container.common.volume.HddsVolume; import org.apache.hadoop.ozone.container.common.volume .RoundRobinVolumeChoosingPolicy; import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.DiskChecker; import org.apache.hadoop.utils.MetadataStore; @@ -117,11 +117,11 @@ public class TestKeyValueContainer { addBlocks(blockCount); blockIterator = keyValueContainer.blockIterator(); assertTrue(blockIterator.hasNext()); - KeyData keyData; + BlockData blockData; int blockCounter = 0; while(blockIterator.hasNext()) { - keyData = blockIterator.nextBlock(); - assertEquals(blockCounter++, keyData.getBlockID().getLocalID()); + blockData = blockIterator.nextBlock(); + assertEquals(blockCounter++, blockData.getBlockID().getLocalID()); } assertEquals(blockCount, blockCounter); } @@ -129,20 +129,20 @@ public class TestKeyValueContainer { private void addBlocks(int count) throws Exception { long containerId = keyValueContainerData.getContainerID(); - MetadataStore metadataStore = KeyUtils.getDB(keyValueContainer + MetadataStore metadataStore = BlockUtils.getDB(keyValueContainer .getContainerData(), conf); for (int i=0; i < count; i++) { - // Creating KeyData + // Creating BlockData BlockID blockID = new BlockID(containerId, i); - KeyData keyData = new KeyData(blockID); - keyData.addMetadata("VOLUME", "ozone"); - keyData.addMetadata("OWNER", "hdfs"); + BlockData blockData = new BlockData(blockID); + blockData.addMetadata("VOLUME", "ozone"); + blockData.addMetadata("OWNER", "hdfs"); List<ContainerProtos.ChunkInfo> chunkList = new LinkedList<>(); ChunkInfo info = new ChunkInfo(String.format("%d.data.%d", blockID .getLocalID(), 0), 0, 1024); chunkList.add(info.getProtoBufMessage()); - keyData.setChunks(chunkList); - metadataStore.put(Longs.toByteArray(blockID.getLocalID()), keyData + blockData.setChunks(chunkList); + metadataStore.put(Longs.toByteArray(blockID.getLocalID()), blockData .getProtoBufMessage().toByteArray()); } @@ -189,7 +189,7 @@ public class TestKeyValueContainer { int numberOfKeysToWrite = 12; //write one few keys to check the key count after import - MetadataStore metadataStore = KeyUtils.getDB(keyValueContainerData, conf); + MetadataStore metadataStore = BlockUtils.getDB(keyValueContainerData, conf); for (int i = 0; i < numberOfKeysToWrite; i++) { metadataStore.put(("test" + i).getBytes(), "test".getBytes()); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java index d91bbf7..e1904c1 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueHandler.java @@ -142,31 +142,31 @@ public class TestKeyValueHandler { Mockito.verify(handler, times(1)).handleCloseContainer( any(ContainerCommandRequestProto.class), any()); - // Test Put Key Request handling - ContainerCommandRequestProto putKeyRequest = - getDummyCommandRequestProto(ContainerProtos.Type.PutKey); - dispatcher.dispatch(putKeyRequest); - Mockito.verify(handler, times(1)).handlePutKey( + // Test Put Block Request handling + ContainerCommandRequestProto putBlockRequest = + getDummyCommandRequestProto(ContainerProtos.Type.PutBlock); + dispatcher.dispatch(putBlockRequest); + Mockito.verify(handler, times(1)).handlePutBlock( any(ContainerCommandRequestProto.class), any()); - // Test Get Key Request handling - ContainerCommandRequestProto getKeyRequest = - getDummyCommandRequestProto(ContainerProtos.Type.GetKey); - dispatcher.dispatch(getKeyRequest); - Mockito.verify(handler, times(1)).handleGetKey( + // Test Get Block Request handling + ContainerCommandRequestProto getBlockRequest = + getDummyCommandRequestProto(ContainerProtos.Type.GetBlock); + dispatcher.dispatch(getBlockRequest); + Mockito.verify(handler, times(1)).handleGetBlock( any(ContainerCommandRequestProto.class), any()); - // Test Delete Key Request handling - ContainerCommandRequestProto deleteKeyRequest = - getDummyCommandRequestProto(ContainerProtos.Type.DeleteKey); - dispatcher.dispatch(deleteKeyRequest); - Mockito.verify(handler, times(1)).handleDeleteKey( + // Test Delete Block Request handling + ContainerCommandRequestProto deleteBlockRequest = + getDummyCommandRequestProto(ContainerProtos.Type.DeleteBlock); + dispatcher.dispatch(deleteBlockRequest); + Mockito.verify(handler, times(1)).handleDeleteBlock( any(ContainerCommandRequestProto.class), any()); - // Test List Key Request handling - ContainerCommandRequestProto listKeyRequest = - getDummyCommandRequestProto(ContainerProtos.Type.ListKey); - dispatcher.dispatch(listKeyRequest); + // Test List Block Request handling + ContainerCommandRequestProto listBlockRequest = + getDummyCommandRequestProto(ContainerProtos.Type.ListBlock); + dispatcher.dispatch(listBlockRequest); Mockito.verify(handler, times(2)).handleUnsupportedOp( any(ContainerCommandRequestProto.class)); http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java index 94966f6..2b10578 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java @@ -281,10 +281,10 @@ public class ChunkGroupInputStream extends InputStream implements Seekable { groupInputStream.streamOffset[i] = length; ContainerProtos.DatanodeBlockID datanodeBlockID = blockID .getDatanodeBlockIDProtobuf(); - ContainerProtos.GetKeyResponseProto response = ContainerProtocolCalls - .getKey(xceiverClient, datanodeBlockID, requestId); + ContainerProtos.GetBlockResponseProto response = ContainerProtocolCalls + .getBlock(xceiverClient, datanodeBlockID, requestId); List<ContainerProtos.ChunkInfo> chunks = - response.getKeyData().getChunksList(); + response.getBlockData().getChunksList(); for (ContainerProtos.ChunkInfo chunk : chunks) { length += chunk.getLen(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java index d0e173c..0537f8a 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java @@ -27,7 +27,7 @@ import org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler; import org.apache.hadoop.hdds.protocol.DatanodeDetails; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -165,10 +165,10 @@ public class TestStorageContainerManagerHelper { DatanodeDetails leadDN = containerWithPipeline.getPipeline().getLeader(); OzoneContainer containerServer = getContainerServerByDatanodeUuid(leadDN.getUuidString()); - KeyValueContainerData containerData = (KeyValueContainerData) containerServer - .getContainerSet() + KeyValueContainerData containerData = + (KeyValueContainerData) containerServer.getContainerSet() .getContainer(containerID).getContainerData(); - return KeyUtils.getDB(containerData, conf); + return BlockUtils.getDB(containerData, conf); } private OzoneContainer getContainerServerByDatanodeUuid(String dnUUID) http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java index ddff0c5..b4a0ba7 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rest/TestOzoneRestClient.java @@ -31,7 +31,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.junit.AfterClass; @@ -469,9 +469,9 @@ public class TestOzoneRestClient { containerID, new File(containerPath)); long valueLength = 0; while (keyValueBlockIterator.hasNext()) { - KeyData keyData = keyValueBlockIterator.nextBlock(); - if (keyData.getBlockID().getLocalID() == localID) { - List<ContainerProtos.ChunkInfo> chunks = keyData.getChunks(); + BlockData blockData = keyValueBlockIterator.nextBlock(); + if (blockData.getBlockID().getLocalID() == localID) { + List<ContainerProtos.ChunkInfo> chunks = blockData.getChunks(); for (ContainerProtos.ChunkInfo chunk : chunks) { valueLength += chunk.getLen(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java index bf1eba6..cc045d0 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClient.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueBlockIterator; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.om.OzoneManager; @@ -603,10 +603,10 @@ public class TestOzoneRpcClient { KeyValueBlockIterator keyValueBlockIterator = new KeyValueBlockIterator( containerID, new File(containerPath)); while (keyValueBlockIterator.hasNext()) { - KeyData keyData = keyValueBlockIterator.nextBlock(); - if (keyData.getBlockID().getLocalID() == localID) { + BlockData blockData = keyValueBlockIterator.nextBlock(); + if (blockData.getBlockID().getLocalID() == localID) { long length = 0; - List<ContainerProtos.ChunkInfo> chunks = keyData.getChunks(); + List<ContainerProtos.ChunkInfo> chunks = blockData.getChunks(); for (ContainerProtos.ChunkInfo chunk : chunks) { length += chunk.getLen(); } http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java index 0c86828..f278479 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ContainerTestHelper.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationType; import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.KeyValue; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.util.Time; import org.junit.Assert; @@ -241,18 +241,18 @@ public final class ContainerTestHelper { setDataChecksum(info, data); - ContainerProtos.PutKeyRequestProto.Builder putRequest = - ContainerProtos.PutKeyRequestProto.newBuilder(); + ContainerProtos.PutBlockRequestProto.Builder putRequest = + ContainerProtos.PutBlockRequestProto.newBuilder(); - KeyData keyData = new KeyData(blockID); + BlockData blockData = new BlockData(blockID); List<ContainerProtos.ChunkInfo> newList = new LinkedList<>(); newList.add(info.getProtoBufMessage()); - keyData.setChunks(newList); - putRequest.setKeyData(keyData.getProtoBufMessage()); + blockData.setChunks(newList); + putRequest.setBlockData(blockData.getProtoBufMessage()); smallFileRequest.setChunkInfo(info.getProtoBufMessage()); smallFileRequest.setData(ByteString.copyFrom(data)); - smallFileRequest.setKey(putRequest); + smallFileRequest.setBlock(putRequest); ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); @@ -266,17 +266,17 @@ public final class ContainerTestHelper { public static ContainerCommandRequestProto getReadSmallFileRequest( - Pipeline pipeline, ContainerProtos.PutKeyRequestProto putKey) + Pipeline pipeline, ContainerProtos.PutBlockRequestProto putKey) throws Exception { ContainerProtos.GetSmallFileRequestProto.Builder smallFileRequest = ContainerProtos.GetSmallFileRequestProto.newBuilder(); - ContainerCommandRequestProto getKey = getKeyRequest(pipeline, putKey); - smallFileRequest.setKey(getKey.getGetKey()); + ContainerCommandRequestProto getKey = getBlockRequest(pipeline, putKey); + smallFileRequest.setBlock(getKey.getGetBlock()); ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); request.setCmdType(ContainerProtos.Type.GetSmallFile); - request.setContainerID(getKey.getGetKey().getBlockID().getContainerID()); + request.setContainerID(getKey.getGetBlock().getBlockID().getContainerID()); request.setGetSmallFile(smallFileRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); @@ -421,58 +421,58 @@ public final class ContainerTestHelper { } /** - * Returns the PutKeyRequest for test purpose. + * Returns the PutBlockRequest for test purpose. * @param pipeline - pipeline. * @param writeRequest - Write Chunk Request. * @return - Request */ - public static ContainerCommandRequestProto getPutKeyRequest( + public static ContainerCommandRequestProto getPutBlockRequest( Pipeline pipeline, ContainerProtos.WriteChunkRequestProto writeRequest) { - LOG.trace("putKey: {} to pipeline={}", + LOG.trace("putBlock: {} to pipeline={}", writeRequest.getBlockID()); - ContainerProtos.PutKeyRequestProto.Builder putRequest = - ContainerProtos.PutKeyRequestProto.newBuilder(); + ContainerProtos.PutBlockRequestProto.Builder putRequest = + ContainerProtos.PutBlockRequestProto.newBuilder(); - KeyData keyData = new KeyData( + BlockData blockData = new BlockData( BlockID.getFromProtobuf(writeRequest.getBlockID())); List<ContainerProtos.ChunkInfo> newList = new LinkedList<>(); newList.add(writeRequest.getChunkData()); - keyData.setChunks(newList); - putRequest.setKeyData(keyData.getProtoBufMessage()); + blockData.setChunks(newList); + putRequest.setBlockData(blockData.getProtoBufMessage()); ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); - request.setCmdType(ContainerProtos.Type.PutKey); - request.setContainerID(keyData.getContainerID()); - request.setPutKey(putRequest); + request.setCmdType(ContainerProtos.Type.PutBlock); + request.setContainerID(blockData.getContainerID()); + request.setPutBlock(putRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); return request.build(); } /** - * Gets a GetKeyRequest for test purpose. + * Gets a GetBlockRequest for test purpose. * @param pipeline - pipeline - * @param putKeyRequest - putKeyRequest. + * @param putBlockRequest - putBlockRequest. * @return - Request * immediately. */ - public static ContainerCommandRequestProto getKeyRequest( - Pipeline pipeline, ContainerProtos.PutKeyRequestProto putKeyRequest) { + public static ContainerCommandRequestProto getBlockRequest( + Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest) { ContainerProtos.DatanodeBlockID blockID = - putKeyRequest.getKeyData().getBlockID(); + putBlockRequest.getBlockData().getBlockID(); LOG.trace("getKey: blockID={}", blockID); - ContainerProtos.GetKeyRequestProto.Builder getRequest = - ContainerProtos.GetKeyRequestProto.newBuilder(); + ContainerProtos.GetBlockRequestProto.Builder getRequest = + ContainerProtos.GetBlockRequestProto.newBuilder(); getRequest.setBlockID(blockID); ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); - request.setCmdType(ContainerProtos.Type.GetKey); + request.setCmdType(ContainerProtos.Type.GetBlock); request.setContainerID(blockID.getContainerID()); - request.setGetKey(getRequest); + request.setGetBlock(getRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); return request.build(); @@ -484,32 +484,32 @@ public final class ContainerTestHelper { * @param request - Request * @param response - Response */ - public static void verifyGetKey(ContainerCommandRequestProto request, + public static void verifyGetBlock(ContainerCommandRequestProto request, ContainerCommandResponseProto response, int expectedChunksCount) { Assert.assertEquals(request.getTraceID(), response.getTraceID()); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); Assert.assertEquals(expectedChunksCount, - response.getGetKey().getKeyData().getChunksCount()); + response.getGetBlock().getBlockData().getChunksCount()); } /** * @param pipeline - pipeline. - * @param putKeyRequest - putKeyRequest. + * @param putBlockRequest - putBlockRequest. * @return - Request */ - public static ContainerCommandRequestProto getDeleteKeyRequest( - Pipeline pipeline, ContainerProtos.PutKeyRequestProto putKeyRequest) { - ContainerProtos.DatanodeBlockID blockID = putKeyRequest.getKeyData() + public static ContainerCommandRequestProto getDeleteBlockRequest( + Pipeline pipeline, ContainerProtos.PutBlockRequestProto putBlockRequest) { + ContainerProtos.DatanodeBlockID blockID = putBlockRequest.getBlockData() .getBlockID(); - LOG.trace("deleteKey: name={}", blockID); - ContainerProtos.DeleteKeyRequestProto.Builder delRequest = - ContainerProtos.DeleteKeyRequestProto.newBuilder(); + LOG.trace("deleteBlock: name={}", blockID); + ContainerProtos.DeleteBlockRequestProto.Builder delRequest = + ContainerProtos.DeleteBlockRequestProto.newBuilder(); delRequest.setBlockID(blockID); ContainerCommandRequestProto.Builder request = ContainerCommandRequestProto.newBuilder(); - request.setCmdType(ContainerProtos.Type.DeleteKey); + request.setCmdType(ContainerProtos.Type.DeleteBlock); request.setContainerID(blockID.getContainerID()); - request.setDeleteKey(delRequest); + request.setDeleteBlock(delRequest); request.setTraceID(UUID.randomUUID().toString()); request.setDatanodeUuid(pipeline.getLeader().getUuidString()); return request.build(); http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java index 7391b25..52cebb3 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/TestContainerReplication.java @@ -39,7 +39,7 @@ import org.apache.hadoop.hdds.scm.XceiverClientSpi; import org.apache.hadoop.hdds.scm.container.common.helpers.Pipeline; import org.apache.hadoop.ozone.HddsDatanodeService; import org.apache.hadoop.ozone.MiniOzoneCluster; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.interfaces.Container; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; import org.apache.hadoop.ozone.container.keyvalue.KeyValueHandler; @@ -100,17 +100,20 @@ public class TestContainerReplication { DatanodeBlockID blockID = requestProto.getWriteChunk().getBlockID(); - // Put Key to the test container - ContainerCommandRequestProto putKeyRequest = ContainerTestHelper - .getPutKeyRequest(sourcePipelines, requestProto.getWriteChunk()); + // Put Block to the test container + ContainerCommandRequestProto putBlockRequest = ContainerTestHelper + .getPutBlockRequest(sourcePipelines, requestProto.getWriteChunk()); - ContainerProtos.KeyData keyData = putKeyRequest.getPutKey().getKeyData(); + ContainerProtos.BlockData blockData = + putBlockRequest.getPutBlock().getBlockData(); - ContainerCommandResponseProto response = client.sendCommand(putKeyRequest); + ContainerCommandResponseProto response = + client.sendCommand(putBlockRequest); Assert.assertNotNull(response); Assert.assertEquals(ContainerProtos.Result.SUCCESS, response.getResult()); - Assert.assertTrue(putKeyRequest.getTraceID().equals(response.getTraceID())); + Assert.assertTrue( + putBlockRequest.getTraceID().equals(response.getTraceID())); HddsDatanodeService destinationDatanode = chooseDatanodeWithoutContainer(sourcePipelines, @@ -147,8 +150,8 @@ public class TestContainerReplication { KeyValueHandler handler = (KeyValueHandler) ozoneContainer.getDispatcher() .getHandler(ContainerType.KeyValueContainer); - KeyData key = handler.getKeyManager() - .getKey(container, BlockID.getFromProtobuf(blockID)); + BlockData key = handler.getBlockManager() + .getBlock(container, BlockID.getFromProtobuf(blockID)); Assert.assertNotNull(key); Assert.assertEquals(1, key.getChunks().size()); @@ -164,7 +167,8 @@ public class TestContainerReplication { return datanode; } } - throw new AssertionError("No datanode outside of the pipeline"); + throw new AssertionError( + "No datanode outside of the pipeline"); } static OzoneConfiguration newOzoneConfiguration() { http://git-wip-us.apache.org/repos/asf/hadoop/blob/096a7160/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java index 25c8c6b..7e30c5f 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java @@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.container.ContainerTestHelper; +import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; import org.apache.hadoop.ozone.container.common.interfaces.Container; @@ -33,9 +34,8 @@ import org.apache.hadoop.ozone.container.common.volume.RoundRobinVolumeChoosingP import org.apache.hadoop.ozone.container.common.volume.VolumeSet; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainer; import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import org.apache.hadoop.ozone.container.keyvalue.helpers.KeyUtils; +import org.apache.hadoop.ozone.container.keyvalue.helpers.BlockUtils; import org.apache.hadoop.ozone.container.testutils.BlockDeletingServiceTestImpl; -import org.apache.hadoop.ozone.container.common.helpers.KeyData; import org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy; import org.apache.hadoop.ozone.container.keyvalue.statemachine.background .BlockDeletingService; @@ -117,13 +117,13 @@ public class TestBlockDeletingService { containerSet.addContainer(container); data = (KeyValueContainerData) containerSet.getContainer( containerID).getContainerData(); - MetadataStore metadata = KeyUtils.getDB(data, conf); + MetadataStore metadata = BlockUtils.getDB(data, conf); for (int j = 0; j<numOfBlocksPerContainer; j++) { BlockID blockID = ContainerTestHelper.getTestBlockID(containerID); String deleteStateName = OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID(); - KeyData kd = new KeyData(blockID); + BlockData kd = new BlockData(blockID); List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList(); for (int k = 0; k<numOfChunksPerBlock; k++) { // offset doesn't matter here @@ -200,7 +200,7 @@ public class TestBlockDeletingService { containerSet.listContainer(0L, 1, containerData); Assert.assertEquals(1, containerData.size()); - MetadataStore meta = KeyUtils.getDB( + MetadataStore meta = BlockUtils.getDB( (KeyValueContainerData) containerData.get(0), conf); Map<Long, Container> containerMap = containerSet.getContainerMap(); // NOTE: this test assumes that all the container is KetValueContainer and @@ -309,7 +309,7 @@ public class TestBlockDeletingService { // get container meta data List<ContainerData> containerData = Lists.newArrayList(); containerSet.listContainer(0L, 1, containerData); - MetadataStore meta = KeyUtils.getDB( + MetadataStore meta = BlockUtils.getDB( (KeyValueContainerData) containerData.get(0), conf); LogCapturer newLog = LogCapturer.captureLogs(BackgroundService.LOG); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
