http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java deleted file mode 100644 index 21f31e1..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.helpers; -/** - Contains protocol buffer helper classes and utilites used in - impl. - **/ \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java deleted file mode 100644 index d1b1bd6..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkLayOutVersion.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.impl; - - -import com.google.common.base.Preconditions; - -/** - * Defines layout versions for the Chunks. - */ - -public final class ChunkLayOutVersion { - - private final static ChunkLayOutVersion[] CHUNK_LAYOUT_VERSION_INFOS = - {new ChunkLayOutVersion(1, "Data without checksums.")}; - - private int version; - private String description; - - - /** - * Never created outside this class. - * - * @param description -- description - * @param version -- version number - */ - private ChunkLayOutVersion(int version, String description) { - this.version = version; - this.description = description; - } - - /** - * Return ChunkLayOutVersion object for the chunkVersion. - * @param chunkVersion - * @return ChunkLayOutVersion - */ - public static ChunkLayOutVersion getChunkLayOutVersion(int chunkVersion) { - Preconditions.checkArgument((chunkVersion <= ChunkLayOutVersion - .getLatestVersion().getVersion())); - for(ChunkLayOutVersion chunkLayOutVersion : CHUNK_LAYOUT_VERSION_INFOS) { - if(chunkLayOutVersion.getVersion() == chunkVersion) { - return chunkLayOutVersion; - } - } - return null; - } - - /** - * Returns all versions. - * - * @return Version info array. - */ - public static ChunkLayOutVersion[] getAllVersions() { - return CHUNK_LAYOUT_VERSION_INFOS.clone(); - } - - /** - * Returns the latest version. - * - * @return versionInfo - */ - public static ChunkLayOutVersion getLatestVersion() { - return CHUNK_LAYOUT_VERSION_INFOS[CHUNK_LAYOUT_VERSION_INFOS.length - 1]; - } - - /** - * Return version. - * - * @return int - */ - public int getVersion() { - return version; - } - - /** - * Returns description. - * @return String - */ - public String getDescription() { - return description; - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java deleted file mode 100644 index efea20b..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerData.java +++ /dev/null @@ -1,435 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.impl; - -import com.google.common.base.Preconditions; -import com.google.common.collect.Lists; -import java.io.IOException; -import java.nio.charset.Charset; -import java.util.List; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos. - ContainerType; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos. - ContainerLifeCycleState; -import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.volume.HddsVolume; - -import java.util.Collections; -import java.util.Map; -import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; -import org.yaml.snakeyaml.Yaml; - -import static org.apache.hadoop.ozone.OzoneConsts.CHECKSUM; -import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ID; -import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_TYPE; -import static org.apache.hadoop.ozone.OzoneConsts.LAYOUTVERSION; -import static org.apache.hadoop.ozone.OzoneConsts.MAX_SIZE; -import static org.apache.hadoop.ozone.OzoneConsts.METADATA; -import static org.apache.hadoop.ozone.OzoneConsts.STATE; - -/** - * ContainerData is the in-memory representation of container metadata and is - * represented on disk by the .container file. - */ -public abstract class ContainerData { - - //Type of the container. - // For now, we support only KeyValueContainer. - private final ContainerType containerType; - - // Unique identifier for the container - private final long containerID; - - // Layout version of the container data - private final int layOutVersion; - - // Metadata of the container will be a key value pair. - // This can hold information like volume name, owner etc., - private final Map<String, String> metadata; - - // State of the Container - private ContainerLifeCycleState state; - - private final long maxSize; - - /** parameters for read/write statistics on the container. **/ - private final AtomicLong readBytes; - private final AtomicLong writeBytes; - private final AtomicLong readCount; - private final AtomicLong writeCount; - private final AtomicLong bytesUsed; - private final AtomicLong keyCount; - - private HddsVolume volume; - - private String checksum; - public static final Charset CHARSET_ENCODING = Charset.forName("UTF-8"); - private static final String DUMMY_CHECKSUM = new String(new byte[64], - CHARSET_ENCODING); - - // Common Fields need to be stored in .container file. - protected static final List<String> YAML_FIELDS = - Collections.unmodifiableList(Lists.newArrayList( - CONTAINER_TYPE, - CONTAINER_ID, - LAYOUTVERSION, - STATE, - METADATA, - MAX_SIZE, - CHECKSUM)); - - /** - * Creates a ContainerData Object, which holds metadata of the container. - * @param type - ContainerType - * @param containerId - ContainerId - * @param size - container maximum size in bytes - */ - protected ContainerData(ContainerType type, long containerId, long size) { - this(type, containerId, - ChunkLayOutVersion.getLatestVersion().getVersion(), size); - } - - /** - * Creates a ContainerData Object, which holds metadata of the container. - * @param type - ContainerType - * @param containerId - ContainerId - * @param layOutVersion - Container layOutVersion - * @param size - Container maximum size in bytes - */ - protected ContainerData(ContainerType type, long containerId, - int layOutVersion, long size) { - Preconditions.checkNotNull(type); - - this.containerType = type; - this.containerID = containerId; - this.layOutVersion = layOutVersion; - this.metadata = new TreeMap<>(); - this.state = ContainerLifeCycleState.OPEN; - this.readCount = new AtomicLong(0L); - this.readBytes = new AtomicLong(0L); - this.writeCount = new AtomicLong(0L); - this.writeBytes = new AtomicLong(0L); - this.bytesUsed = new AtomicLong(0L); - this.keyCount = new AtomicLong(0L); - this.maxSize = size; - setChecksumTo0ByteArray(); - } - - /** - * Returns the containerID. - */ - public long getContainerID() { - return containerID; - } - - /** - * Returns the path to base dir of the container. - * @return Path to base dir. - */ - public abstract String getContainerPath(); - - /** - * Returns the type of the container. - * @return ContainerType - */ - public ContainerType getContainerType() { - return containerType; - } - - - /** - * Returns the state of the container. - * @return ContainerLifeCycleState - */ - public synchronized ContainerLifeCycleState getState() { - return state; - } - - /** - * Set the state of the container. - * @param state - */ - public synchronized void setState(ContainerLifeCycleState state) { - this.state = state; - } - - /** - * Return's maximum size of the container in bytes. - * @return maxSize in bytes - */ - public long getMaxSize() { - return maxSize; - } - - /** - * Returns the layOutVersion of the actual container data format. - * @return layOutVersion - */ - public int getLayOutVersion() { - return ChunkLayOutVersion.getChunkLayOutVersion(layOutVersion).getVersion(); - } - - /** - * Add/Update metadata. - * We should hold the container lock before updating the metadata as this - * will be persisted on disk. Unless, we are reconstructing ContainerData - * from protoBuf or from on disk .container file in which case lock is not - * required. - */ - public void addMetadata(String key, String value) { - metadata.put(key, value); - } - - /** - * Retuns metadata of the container. - * @return metadata - */ - public Map<String, String> getMetadata() { - return Collections.unmodifiableMap(this.metadata); - } - - /** - * Set metadata. - * We should hold the container lock before updating the metadata as this - * will be persisted on disk. Unless, we are reconstructing ContainerData - * from protoBuf or from on disk .container file in which case lock is not - * required. - */ - public void setMetadata(Map<String, String> metadataMap) { - metadata.clear(); - metadata.putAll(metadataMap); - } - - /** - * checks if the container is open. - * @return - boolean - */ - public synchronized boolean isOpen() { - return ContainerLifeCycleState.OPEN == state; - } - - /** - * checks if the container is invalid. - * @return - boolean - */ - public synchronized boolean isValid() { - return !(ContainerLifeCycleState.INVALID == state); - } - - /** - * checks if the container is closed. - * @return - boolean - */ - public synchronized boolean isClosed() { - return ContainerLifeCycleState.CLOSED == state; - } - - /** - * Marks this container as closed. - */ - public synchronized void closeContainer() { - setState(ContainerLifeCycleState.CLOSED); - } - - /** - * Get the number of bytes read from the container. - * @return the number of bytes read from the container. - */ - public long getReadBytes() { - return readBytes.get(); - } - - /** - * Increase the number of bytes read from the container. - * @param bytes number of bytes read. - */ - public void incrReadBytes(long bytes) { - this.readBytes.addAndGet(bytes); - } - - /** - * Get the number of times the container is read. - * @return the number of times the container is read. - */ - public long getReadCount() { - return readCount.get(); - } - - /** - * Increase the number of container read count by 1. - */ - public void incrReadCount() { - this.readCount.incrementAndGet(); - } - - /** - * Get the number of bytes write into the container. - * @return the number of bytes write into the container. - */ - public long getWriteBytes() { - return writeBytes.get(); - } - - /** - * Increase the number of bytes write into the container. - * @param bytes the number of bytes write into the container. - */ - public void incrWriteBytes(long bytes) { - this.writeBytes.addAndGet(bytes); - } - - /** - * Get the number of writes into the container. - * @return the number of writes into the container. - */ - public long getWriteCount() { - return writeCount.get(); - } - - /** - * Increase the number of writes into the container by 1. - */ - public void incrWriteCount() { - this.writeCount.incrementAndGet(); - } - - /** - * Sets the number of bytes used by the container. - * @param used - */ - public void setBytesUsed(long used) { - this.bytesUsed.set(used); - } - - /** - * Get the number of bytes used by the container. - * @return the number of bytes used by the container. - */ - public long getBytesUsed() { - return bytesUsed.get(); - } - - /** - * Increase the number of bytes used by the container. - * @param used number of bytes used by the container. - * @return the current number of bytes used by the container afert increase. - */ - public long incrBytesUsed(long used) { - return this.bytesUsed.addAndGet(used); - } - - /** - * Decrease the number of bytes used by the container. - * @param reclaimed the number of bytes reclaimed from the container. - * @return the current number of bytes used by the container after decrease. - */ - public long decrBytesUsed(long reclaimed) { - return this.bytesUsed.addAndGet(-1L * reclaimed); - } - - /** - * Set the Volume for the Container. - * This should be called only from the createContainer. - * @param hddsVolume - */ - public void setVolume(HddsVolume hddsVolume) { - this.volume = hddsVolume; - } - - /** - * Returns the volume of the Container. - * @return HddsVolume - */ - public HddsVolume getVolume() { - return volume; - } - - /** - * Increments the number of keys in the container. - */ - public void incrKeyCount() { - this.keyCount.incrementAndGet(); - } - - /** - * Decrements number of keys in the container. - */ - public void decrKeyCount() { - this.keyCount.decrementAndGet(); - } - - /** - * Returns number of keys in the container. - * @return key count - */ - public long getKeyCount() { - return this.keyCount.get(); - } - - /** - * Set's number of keys in the container. - * @param count - */ - public void setKeyCount(long count) { - this.keyCount.set(count); - } - - public void setChecksumTo0ByteArray() { - this.checksum = DUMMY_CHECKSUM; - } - - public void setChecksum(String checkSum) { - this.checksum = checkSum; - } - - public String getChecksum() { - return this.checksum; - } - - /** - * Compute the checksum for ContainerData using the specified Yaml (based - * on ContainerType) and set the checksum. - * - * Checksum of ContainerData is calculated by setting the - * {@link ContainerData#checksum} field to a 64-byte array with all 0's - - * {@link ContainerData#DUMMY_CHECKSUM}. After the checksum is calculated, - * the checksum field is updated with this value. - * - * @param yaml Yaml for ContainerType to get the ContainerData as Yaml String - * @throws IOException - */ - public void computeAndSetChecksum(Yaml yaml) throws IOException { - // Set checksum to dummy value - 0 byte array, to calculate the checksum - // of rest of the data. - setChecksumTo0ByteArray(); - - // Dump yaml data into a string to compute its checksum - String containerDataYamlStr = yaml.dump(this); - - this.checksum = ContainerUtils.getChecksum(containerDataYamlStr); - } - - /** - * Returns a ProtoBuf Message from ContainerData. - * - * @return Protocol Buffer Message - */ - public abstract ContainerProtos.ContainerData getProtoBufMessage(); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java deleted file mode 100644 index 65262d4..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerDataYaml.java +++ /dev/null @@ -1,326 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.impl; - -import java.beans.IntrospectionException; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStreamWriter; -import java.io.Writer; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerType; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; - -import com.google.common.base.Preconditions; -import static org.apache.hadoop.ozone.container.keyvalue - .KeyValueContainerData.KEYVALUE_YAML_TAG; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.yaml.snakeyaml.Yaml; -import org.yaml.snakeyaml.constructor.AbstractConstruct; -import org.yaml.snakeyaml.constructor.Constructor; -import org.yaml.snakeyaml.introspector.BeanAccess; -import org.yaml.snakeyaml.introspector.Property; -import org.yaml.snakeyaml.introspector.PropertyUtils; -import org.yaml.snakeyaml.nodes.MappingNode; -import org.yaml.snakeyaml.nodes.Node; -import org.yaml.snakeyaml.nodes.ScalarNode; -import org.yaml.snakeyaml.nodes.Tag; -import org.yaml.snakeyaml.representer.Representer; - -/** - * Class for creating and reading .container files. - */ - -public final class ContainerDataYaml { - - private static final Logger LOG = - LoggerFactory.getLogger(ContainerDataYaml.class); - - private ContainerDataYaml() { - - } - - /** - * Creates a .container file in yaml format. - * - * @param containerFile - * @param containerData - * @throws IOException - */ - public static void createContainerFile(ContainerType containerType, - ContainerData containerData, File containerFile) throws IOException { - Writer writer = null; - try { - // Create Yaml for given container type - Yaml yaml = getYamlForContainerType(containerType); - // Compute Checksum and update ContainerData - containerData.computeAndSetChecksum(yaml); - - // Write the ContainerData with checksum to Yaml file. - writer = new OutputStreamWriter(new FileOutputStream( - containerFile), "UTF-8"); - yaml.dump(containerData, writer); - - } finally { - try { - if (writer != null) { - writer.close(); - } - } catch (IOException ex) { - LOG.warn("Error occurred during closing the writer. ContainerID: " + - containerData.getContainerID()); - } - } - } - - /** - * Read the yaml file, and return containerData. - * - * @throws IOException - */ - public static ContainerData readContainerFile(File containerFile) - throws IOException { - Preconditions.checkNotNull(containerFile, "containerFile cannot be null"); - try (FileInputStream inputFileStream = new FileInputStream(containerFile)) { - return readContainer(inputFileStream); - } - - } - - /** - * Read the yaml file content, and return containerData. - * - * @throws IOException - */ - public static ContainerData readContainer(byte[] containerFileContent) - throws IOException { - return readContainer( - new ByteArrayInputStream(containerFileContent)); - } - - /** - * Read the yaml content, and return containerData. - * - * @throws IOException - */ - public static ContainerData readContainer(InputStream input) - throws IOException { - - ContainerData containerData; - PropertyUtils propertyUtils = new PropertyUtils(); - propertyUtils.setBeanAccess(BeanAccess.FIELD); - propertyUtils.setAllowReadOnlyProperties(true); - - Representer representer = new ContainerDataRepresenter(); - representer.setPropertyUtils(propertyUtils); - - Constructor containerDataConstructor = new ContainerDataConstructor(); - - Yaml yaml = new Yaml(containerDataConstructor, representer); - yaml.setBeanAccess(BeanAccess.FIELD); - - containerData = (ContainerData) - yaml.load(input); - - return containerData; - } - - /** - * Given a ContainerType this method returns a Yaml representation of - * the container properties. - * - * @param containerType type of container - * @return Yamal representation of container properties - * - * @throws StorageContainerException if the type is unrecognized - */ - public static Yaml getYamlForContainerType(ContainerType containerType) - throws StorageContainerException { - PropertyUtils propertyUtils = new PropertyUtils(); - propertyUtils.setBeanAccess(BeanAccess.FIELD); - propertyUtils.setAllowReadOnlyProperties(true); - - switch (containerType) { - case KeyValueContainer: - Representer representer = new ContainerDataRepresenter(); - representer.setPropertyUtils(propertyUtils); - representer.addClassTag( - KeyValueContainerData.class, - KeyValueContainerData.KEYVALUE_YAML_TAG); - - Constructor keyValueDataConstructor = new ContainerDataConstructor(); - - return new Yaml(keyValueDataConstructor, representer); - default: - throw new StorageContainerException("Unrecognized container Type " + - "format " + containerType, ContainerProtos.Result - .UNKNOWN_CONTAINER_TYPE); - } - } - - /** - * Representer class to define which fields need to be stored in yaml file. - */ - private static class ContainerDataRepresenter extends Representer { - @Override - protected Set<Property> getProperties(Class<? extends Object> type) - throws IntrospectionException { - Set<Property> set = super.getProperties(type); - Set<Property> filtered = new TreeSet<Property>(); - - // When a new Container type is added, we need to add what fields need - // to be filtered here - if (type.equals(KeyValueContainerData.class)) { - List<String> yamlFields = KeyValueContainerData.getYamlFields(); - // filter properties - for (Property prop : set) { - String name = prop.getName(); - if (yamlFields.contains(name)) { - filtered.add(prop); - } - } - } - return filtered; - } - } - - /** - * Constructor class for KeyValueData, which will be used by Yaml. - */ - private static class ContainerDataConstructor extends Constructor { - ContainerDataConstructor() { - //Adding our own specific constructors for tags. - // When a new Container type is added, we need to add yamlConstructor - // for that - this.yamlConstructors.put( - KEYVALUE_YAML_TAG, new ConstructKeyValueContainerData()); - this.yamlConstructors.put(Tag.INT, new ConstructLong()); - } - - private class ConstructKeyValueContainerData extends AbstractConstruct { - public Object construct(Node node) { - MappingNode mnode = (MappingNode) node; - Map<Object, Object> nodes = constructMapping(mnode); - - //Needed this, as TAG.INT type is by default converted to Long. - long layOutVersion = (long) nodes.get(OzoneConsts.LAYOUTVERSION); - int lv = (int) layOutVersion; - - long size = (long) nodes.get(OzoneConsts.MAX_SIZE); - - //When a new field is added, it needs to be added here. - KeyValueContainerData kvData = new KeyValueContainerData( - (long) nodes.get(OzoneConsts.CONTAINER_ID), lv, size); - - kvData.setContainerDBType((String)nodes.get( - OzoneConsts.CONTAINER_DB_TYPE)); - kvData.setMetadataPath((String) nodes.get( - OzoneConsts.METADATA_PATH)); - kvData.setChunksPath((String) nodes.get(OzoneConsts.CHUNKS_PATH)); - Map<String, String> meta = (Map) nodes.get(OzoneConsts.METADATA); - kvData.setMetadata(meta); - kvData.setChecksum((String) nodes.get(OzoneConsts.CHECKSUM)); - String state = (String) nodes.get(OzoneConsts.STATE); - switch (state) { - case "OPEN": - kvData.setState(ContainerProtos.ContainerLifeCycleState.OPEN); - break; - case "CLOSING": - kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSING); - break; - case "CLOSED": - kvData.setState(ContainerProtos.ContainerLifeCycleState.CLOSED); - break; - default: - throw new IllegalStateException("Unexpected " + - "ContainerLifeCycleState " + state + " for the containerId " + - nodes.get(OzoneConsts.CONTAINER_ID)); - } - return kvData; - } - } - - //Below code is taken from snake yaml, as snakeyaml tries to fit the - // number if it fits in integer, otherwise returns long. So, slightly - // modified the code to return long in all cases. - private class ConstructLong extends AbstractConstruct { - public Object construct(Node node) { - String value = constructScalar((ScalarNode) node).toString() - .replaceAll("_", ""); - int sign = +1; - char first = value.charAt(0); - if (first == '-') { - sign = -1; - value = value.substring(1); - } else if (first == '+') { - value = value.substring(1); - } - int base = 10; - if ("0".equals(value)) { - return Long.valueOf(0); - } else if (value.startsWith("0b")) { - value = value.substring(2); - base = 2; - } else if (value.startsWith("0x")) { - value = value.substring(2); - base = 16; - } else if (value.startsWith("0")) { - value = value.substring(1); - base = 8; - } else if (value.indexOf(':') != -1) { - String[] digits = value.split(":"); - int bes = 1; - int val = 0; - for (int i = 0, j = digits.length; i < j; i++) { - val += (Long.parseLong(digits[(j - i) - 1]) * bes); - bes *= 60; - } - return createNumber(sign, String.valueOf(val), 10); - } else { - return createNumber(sign, value, 10); - } - return createNumber(sign, value, base); - } - } - - private Number createNumber(int sign, String number, int radix) { - Number result; - if (sign < 0) { - number = "-" + number; - } - result = Long.valueOf(number, radix); - return result; - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java deleted file mode 100644 index 7f4f147..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerSet.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.impl; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerReportsProto; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.ozone.container.common.interfaces.Container; -import org.apache.hadoop.ozone.container.common - .interfaces.ContainerDeletionChoosingPolicy; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentNavigableMap; -import java.util.concurrent.ConcurrentSkipListMap; -import java.util.stream.Collectors; - - -/** - * Class that manages Containers created on the datanode. - */ -public class ContainerSet { - - private static final Logger LOG = LoggerFactory.getLogger(ContainerSet.class); - - private final ConcurrentSkipListMap<Long, Container> containerMap = new - ConcurrentSkipListMap<>(); - - /** - * Add Container to container map. - * @param container - * @return If container is added to containerMap returns true, otherwise - * false - */ - public boolean addContainer(Container container) throws - StorageContainerException { - Preconditions.checkNotNull(container, "container cannot be null"); - - long containerId = container.getContainerData().getContainerID(); - if(containerMap.putIfAbsent(containerId, container) == null) { - LOG.debug("Container with container Id {} is added to containerMap", - containerId); - return true; - } else { - LOG.warn("Container already exists with container Id {}", containerId); - throw new StorageContainerException("Container already exists with " + - "container Id " + containerId, - ContainerProtos.Result.CONTAINER_EXISTS); - } - } - - /** - * Returns the Container with specified containerId. - * @param containerId - * @return Container - */ - public Container getContainer(long containerId) { - Preconditions.checkState(containerId >= 0, - "Container Id cannot be negative."); - return containerMap.get(containerId); - } - - /** - * Removes the Container matching with specified containerId. - * @param containerId - * @return If container is removed from containerMap returns true, otherwise - * false - */ - public boolean removeContainer(long containerId) { - Preconditions.checkState(containerId >= 0, - "Container Id cannot be negative."); - Container removed = containerMap.remove(containerId); - if(removed == null) { - LOG.debug("Container with containerId {} is not present in " + - "containerMap", containerId); - return false; - } else { - LOG.debug("Container with containerId {} is removed from containerMap", - containerId); - return true; - } - } - - /** - * Return number of containers in container map. - * @return container count - */ - @VisibleForTesting - public int containerCount() { - return containerMap.size(); - } - - /** - * Return an container Iterator over {@link ContainerSet#containerMap}. - * @return Iterator<Container> - */ - public Iterator<Container> getContainerIterator() { - return containerMap.values().iterator(); - } - - /** - * Return an containerMap iterator over {@link ContainerSet#containerMap}. - * @return containerMap Iterator - */ - public Iterator<Map.Entry<Long, Container>> getContainerMapIterator() { - return containerMap.entrySet().iterator(); - } - - /** - * Return a copy of the containerMap. - * @return containerMap - */ - public Map<Long, Container> getContainerMap() { - return ImmutableMap.copyOf(containerMap); - } - - /** - * A simple interface for container Iterations. - * <p/> - * This call make no guarantees about consistency of the data between - * different list calls. It just returns the best known data at that point of - * time. It is possible that using this iteration you can miss certain - * container from the listing. - * - * @param startContainerId - Return containers with Id >= startContainerId. - * @param count - how many to return - * @param data - Actual containerData - * @throws StorageContainerException - */ - public void listContainer(long startContainerId, long count, - List<ContainerData> data) throws - StorageContainerException { - Preconditions.checkNotNull(data, - "Internal assertion: data cannot be null"); - Preconditions.checkState(startContainerId >= 0, - "Start container Id cannot be negative"); - Preconditions.checkState(count > 0, - "max number of containers returned " + - "must be positive"); - LOG.debug("listContainer returns containerData starting from {} of count " + - "{}", startContainerId, count); - ConcurrentNavigableMap<Long, Container> map; - if (startContainerId == 0) { - map = containerMap.tailMap(containerMap.firstKey(), true); - } else { - map = containerMap.tailMap(startContainerId, true); - } - int currentCount = 0; - for (Container entry : map.values()) { - if (currentCount < count) { - data.add(entry.getContainerData()); - currentCount++; - } else { - return; - } - } - } - - /** - * Get container report. - * - * @return The container report. - * @throws IOException - */ - public ContainerReportsProto getContainerReport() throws IOException { - LOG.debug("Starting container report iteration."); - - // No need for locking since containerMap is a ConcurrentSkipListMap - // And we can never get the exact state since close might happen - // after we iterate a point. - List<Container> containers = containerMap.values().stream().collect( - Collectors.toList()); - - ContainerReportsProto.Builder crBuilder = - ContainerReportsProto.newBuilder(); - - for (Container container: containers) { - crBuilder.addReports(container.getContainerReport()); - } - - return crBuilder.build(); - } - - public List<ContainerData> chooseContainerForBlockDeletion(int count, - ContainerDeletionChoosingPolicy deletionPolicy) - throws StorageContainerException { - Map<Long, ContainerData> containerDataMap = containerMap.entrySet().stream() - .filter(e -> deletionPolicy.isValidContainerType( - e.getValue().getContainerType())) - .collect(Collectors.toMap(Map.Entry::getKey, - e -> e.getValue().getContainerData())); - return deletionPolicy - .chooseContainerForBlockDeletion(count, containerDataMap); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java deleted file mode 100644 index bb5002a..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/HddsDispatcher.java +++ /dev/null @@ -1,213 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.impl; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; -import org.apache.hadoop.hdds.HddsConfigKeys; -import org.apache.hadoop.hdds.HddsUtils; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.proto - .StorageContainerDatanodeProtocolProtos.ContainerAction; -import org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics; -import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils; -import org.apache.hadoop.ozone.container.common.interfaces.Container; -import org.apache.hadoop.ozone.container.common.interfaces.Handler; -import org.apache.hadoop.ozone.container.common.statemachine.StateContext; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandResponseProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerType; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerLifeCycleState; -import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Map; -import java.util.Optional; - -/** - * Ozone Container dispatcher takes a call from the netty server and routes it - * to the right handler function. - */ -public class HddsDispatcher implements ContainerDispatcher { - - static final Logger LOG = LoggerFactory.getLogger(HddsDispatcher.class); - - private final Map<ContainerType, Handler> handlers; - private final Configuration conf; - private final ContainerSet containerSet; - private final VolumeSet volumeSet; - private final StateContext context; - private final float containerCloseThreshold; - private String scmID; - private ContainerMetrics metrics; - - /** - * Constructs an OzoneContainer that receives calls from - * XceiverServerHandler. - */ - public HddsDispatcher(Configuration config, ContainerSet contSet, - VolumeSet volumes, StateContext context) { - this.conf = config; - this.containerSet = contSet; - this.volumeSet = volumes; - this.context = context; - this.handlers = Maps.newHashMap(); - this.metrics = ContainerMetrics.create(conf); - for (ContainerType containerType : ContainerType.values()) { - handlers.put(containerType, - Handler.getHandlerForContainerType( - containerType, conf, containerSet, volumeSet, metrics)); - } - this.containerCloseThreshold = conf.getFloat( - HddsConfigKeys.HDDS_CONTAINER_CLOSE_THRESHOLD, - HddsConfigKeys.HDDS_CONTAINER_CLOSE_THRESHOLD_DEFAULT); - - } - - @Override - public void init() { - } - - @Override - public void shutdown() { - // Shutdown the volumes - volumeSet.shutdown(); - } - - @Override - public ContainerCommandResponseProto dispatch( - ContainerCommandRequestProto msg) { - LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(), - msg.getTraceID()); - Preconditions.checkNotNull(msg); - - Container container = null; - ContainerType containerType = null; - ContainerCommandResponseProto responseProto = null; - long startTime = System.nanoTime(); - ContainerProtos.Type cmdType = msg.getCmdType(); - try { - long containerID = msg.getContainerID(); - - metrics.incContainerOpsMetrics(cmdType); - if (cmdType != ContainerProtos.Type.CreateContainer) { - container = getContainer(containerID); - containerType = getContainerType(container); - } else { - if (!msg.hasCreateContainer()) { - return ContainerUtils.malformedRequest(msg); - } - containerType = msg.getCreateContainer().getContainerType(); - } - } catch (StorageContainerException ex) { - return ContainerUtils.logAndReturnError(LOG, ex, msg); - } - // Small performance optimization. We check if the operation is of type - // write before trying to send CloseContainerAction. - if (!HddsUtils.isReadOnly(msg)) { - sendCloseContainerActionIfNeeded(container); - } - Handler handler = getHandler(containerType); - if (handler == null) { - StorageContainerException ex = new StorageContainerException("Invalid " + - "ContainerType " + containerType, - ContainerProtos.Result.CONTAINER_INTERNAL_ERROR); - return ContainerUtils.logAndReturnError(LOG, ex, msg); - } - responseProto = handler.handle(msg, container); - if (responseProto != null) { - metrics.incContainerOpsLatencies(cmdType, System.nanoTime() - startTime); - return responseProto; - } else { - return ContainerUtils.unsupportedRequest(msg); - } - } - - /** - * If the container usage reaches the close threshold we send Close - * ContainerAction to SCM. - * - * @param container current state of container - */ - private void sendCloseContainerActionIfNeeded(Container container) { - // We have to find a more efficient way to close a container. - Boolean isOpen = Optional.ofNullable(container) - .map(cont -> cont.getContainerState() == ContainerLifeCycleState.OPEN) - .orElse(Boolean.FALSE); - if (isOpen) { - ContainerData containerData = container.getContainerData(); - double containerUsedPercentage = 1.0f * containerData.getBytesUsed() / - containerData.getMaxSize(); - if (containerUsedPercentage >= containerCloseThreshold) { - ContainerAction action = ContainerAction.newBuilder() - .setContainerID(containerData.getContainerID()) - .setAction(ContainerAction.Action.CLOSE) - .setReason(ContainerAction.Reason.CONTAINER_FULL) - .build(); - context.addContainerActionIfAbsent(action); - } - } - } - - @Override - public Handler getHandler(ContainerProtos.ContainerType containerType) { - return handlers.get(containerType); - } - - @Override - public void setScmId(String scmId) { - Preconditions.checkNotNull(scmId, "scmId Cannot be null"); - if (this.scmID == null) { - this.scmID = scmId; - for (Map.Entry<ContainerType, Handler> handlerMap : handlers.entrySet()) { - handlerMap.getValue().setScmID(scmID); - } - } - } - - @VisibleForTesting - public Container getContainer(long containerID) - throws StorageContainerException { - Container container = containerSet.getContainer(containerID); - if (container == null) { - throw new StorageContainerException( - "ContainerID " + containerID + " does not exist", - ContainerProtos.Result.CONTAINER_NOT_FOUND); - } - return container; - } - - private ContainerType getContainerType(Container container) { - return container.getContainerType(); - } - - @VisibleForTesting - public void setMetricsForTesting(ContainerMetrics containerMetrics) { - this.metrics = containerMetrics; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java deleted file mode 100644 index b736eb5..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/OpenContainerBlockMap.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.impl; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.client.BlockID; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo; -import org.apache.hadoop.ozone.container.common.helpers.BlockData; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.function.Function; - -/** - * Map: containerId {@literal ->} (localId {@literal ->} {@link BlockData}). - * The outer container map does not entail locking for a better performance. - * The inner {@link BlockDataMap} is synchronized. - * - * This class will maintain list of open keys per container when closeContainer - * command comes, it should autocommit all open keys of a open container before - * marking the container as closed. - */ -public class OpenContainerBlockMap { - /** - * Map: localId {@literal ->} BlockData. - * - * In order to support {@link #getAll()}, the update operations are - * synchronized. - */ - static class BlockDataMap { - private final ConcurrentMap<Long, BlockData> blocks = - new ConcurrentHashMap<>(); - - BlockData get(long localId) { - return blocks.get(localId); - } - - synchronized int removeAndGetSize(long localId) { - blocks.remove(localId); - return blocks.size(); - } - - synchronized BlockData computeIfAbsent( - long localId, Function<Long, BlockData> f) { - return blocks.computeIfAbsent(localId, f); - } - - synchronized List<BlockData> getAll() { - return new ArrayList<>(blocks.values()); - } - } - - /** - * TODO : We may construct the openBlockMap by reading the Block Layout - * for each block inside a container listing all chunk files and reading the - * sizes. This will help to recreate the openKeys Map once the DataNode - * restarts. - * - * For now, we will track all open blocks of a container in the blockMap. - */ - private final ConcurrentMap<Long, BlockDataMap> containers = - new ConcurrentHashMap<>(); - - /** - * Removes the Container matching with specified containerId. - * @param containerId containerId - */ - public void removeContainer(long containerId) { - Preconditions - .checkState(containerId >= 0, "Container Id cannot be negative."); - containers.remove(containerId); - } - - public void addChunk(BlockID blockID, ChunkInfo info) { - Preconditions.checkNotNull(info); - containers.computeIfAbsent(blockID.getContainerID(), - id -> new BlockDataMap()).computeIfAbsent(blockID.getLocalID(), - id -> new BlockData(blockID)).addChunk(info); - } - - /** - * Removes the chunk from the chunkInfo list for the given block. - * @param blockID id of the block - * @param chunkInfo chunk info. - */ - public void removeChunk(BlockID blockID, ChunkInfo chunkInfo) { - Preconditions.checkNotNull(chunkInfo); - Preconditions.checkNotNull(blockID); - Optional.ofNullable(containers.get(blockID.getContainerID())) - .map(blocks -> blocks.get(blockID.getLocalID())) - .ifPresent(keyData -> keyData.removeChunk(chunkInfo)); - } - - /** - * Returns the list of open blocks to the openContainerBlockMap. - * @param containerId container id - * @return List of open blocks - */ - public List<BlockData> getOpenBlocks(long containerId) { - return Optional.ofNullable(containers.get(containerId)) - .map(BlockDataMap::getAll) - .orElseGet(Collections::emptyList); - } - - /** - * removes the block from the block map. - * @param blockID - block ID - */ - public void removeFromBlockMap(BlockID blockID) { - Preconditions.checkNotNull(blockID); - containers.computeIfPresent(blockID.getContainerID(), (containerId, blocks) - -> blocks.removeAndGetSize(blockID.getLocalID()) == 0? null: blocks); - } - - /** - * Returns true if the block exists in the map, false otherwise. - * - * @param blockID - Block ID. - * @return True, if it exists, false otherwise - */ - public boolean checkIfBlockExists(BlockID blockID) { - BlockDataMap keyDataMap = containers.get(blockID.getContainerID()); - return keyDataMap != null && keyDataMap.get(blockID.getLocalID()) != null; - } - - @VisibleForTesting - BlockDataMap getBlockDataMap(long containerId) { - return containers.get(containerId); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java deleted file mode 100644 index 5c6c319..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.impl; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdfs.DFSUtil; -import org.apache.hadoop.ozone.container.common.interfaces - .ContainerDeletionChoosingPolicy; -import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * Randomly choosing containers for block deletion. - */ -public class RandomContainerDeletionChoosingPolicy - implements ContainerDeletionChoosingPolicy { - private static final Logger LOG = - LoggerFactory.getLogger(RandomContainerDeletionChoosingPolicy.class); - - @Override - public List<ContainerData> chooseContainerForBlockDeletion(int count, - Map<Long, ContainerData> candidateContainers) - throws StorageContainerException { - Preconditions.checkNotNull(candidateContainers, - "Internal assertion: candidate containers cannot be null"); - - int currentCount = 0; - List<ContainerData> result = new LinkedList<>(); - ContainerData[] values = new ContainerData[candidateContainers.size()]; - // to get a shuffle list - for (ContainerData entry : DFSUtil.shuffle( - candidateContainers.values().toArray(values))) { - if (currentCount < count) { - result.add(entry); - currentCount++; - - LOG.debug("Select container {} for block deletion, " - + "pending deletion blocks num: {}.", - entry.getContainerID(), - ((KeyValueContainerData)entry).getNumPendingDeletionBlocks()); - } else { - break; - } - } - - return result; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java deleted file mode 100644 index 061d09b..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java +++ /dev/null @@ -1,300 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.impl; - -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdds.protocol.proto. - StorageContainerDatanodeProtocolProtos.StorageReportProto; -import org.apache.hadoop.hdds.protocol.proto. - StorageContainerDatanodeProtocolProtos.StorageTypeProto; -import org.apache.hadoop.ozone.container.common.interfaces - .StorageLocationReportMXBean; - -import java.io.IOException; - -/** - * Storage location stats of datanodes that provide back store for containers. - * - */ -public final class StorageLocationReport implements - StorageLocationReportMXBean { - - private final String id; - private final boolean failed; - private final long capacity; - private final long scmUsed; - private final long remaining; - private final StorageType storageType; - private final String storageLocation; - - private StorageLocationReport(String id, boolean failed, long capacity, - long scmUsed, long remaining, StorageType storageType, - String storageLocation) { - this.id = id; - this.failed = failed; - this.capacity = capacity; - this.scmUsed = scmUsed; - this.remaining = remaining; - this.storageType = storageType; - this.storageLocation = storageLocation; - } - - public String getId() { - return id; - } - - public boolean isFailed() { - return failed; - } - - public long getCapacity() { - return capacity; - } - - public long getScmUsed() { - return scmUsed; - } - - public long getRemaining() { - return remaining; - } - - public String getStorageLocation() { - return storageLocation; - } - - @Override - public String getStorageTypeName() { - return storageType.name(); - } - - public StorageType getStorageType() { - return storageType; - } - - - private StorageTypeProto getStorageTypeProto() throws - IOException { - StorageTypeProto storageTypeProto; - switch (getStorageType()) { - case SSD: - storageTypeProto = StorageTypeProto.SSD; - break; - case DISK: - storageTypeProto = StorageTypeProto.DISK; - break; - case ARCHIVE: - storageTypeProto = StorageTypeProto.ARCHIVE; - break; - case PROVIDED: - storageTypeProto = StorageTypeProto.PROVIDED; - break; - case RAM_DISK: - storageTypeProto = StorageTypeProto.RAM_DISK; - break; - default: - throw new IOException("Illegal Storage Type specified"); - } - return storageTypeProto; - } - - private static StorageType getStorageType(StorageTypeProto proto) throws - IOException { - StorageType storageType; - switch (proto) { - case SSD: - storageType = StorageType.SSD; - break; - case DISK: - storageType = StorageType.DISK; - break; - case ARCHIVE: - storageType = StorageType.ARCHIVE; - break; - case PROVIDED: - storageType = StorageType.PROVIDED; - break; - case RAM_DISK: - storageType = StorageType.RAM_DISK; - break; - default: - throw new IOException("Illegal Storage Type specified"); - } - return storageType; - } - - /** - * Returns the SCMStorageReport protoBuf message for the Storage Location - * report. - * @return SCMStorageReport - * @throws IOException In case, the storage type specified is invalid. - */ - public StorageReportProto getProtoBufMessage() throws IOException{ - StorageReportProto.Builder srb = StorageReportProto.newBuilder(); - return srb.setStorageUuid(getId()) - .setCapacity(getCapacity()) - .setScmUsed(getScmUsed()) - .setRemaining(getRemaining()) - .setStorageType(getStorageTypeProto()) - .setStorageLocation(getStorageLocation()) - .setFailed(isFailed()) - .build(); - } - - /** - * Returns the StorageLocationReport from the protoBuf message. - * @param report SCMStorageReport - * @return StorageLocationReport - * @throws IOException in case of invalid storage type - */ - - public static StorageLocationReport getFromProtobuf(StorageReportProto report) - throws IOException { - StorageLocationReport.Builder builder = StorageLocationReport.newBuilder(); - builder.setId(report.getStorageUuid()) - .setStorageLocation(report.getStorageLocation()); - if (report.hasCapacity()) { - builder.setCapacity(report.getCapacity()); - } - if (report.hasScmUsed()) { - builder.setScmUsed(report.getScmUsed()); - } - if (report.hasStorageType()) { - builder.setStorageType(getStorageType(report.getStorageType())); - } - if (report.hasRemaining()) { - builder.setRemaining(report.getRemaining()); - } - - if (report.hasFailed()) { - builder.setFailed(report.getFailed()); - } - return builder.build(); - } - - /** - * Returns StorageLocation.Builder instance. - * - * @return StorageLocation.Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder class for building StorageLocationReport. - */ - public static class Builder { - private String id; - private boolean failed; - private long capacity; - private long scmUsed; - private long remaining; - private StorageType storageType; - private String storageLocation; - - /** - * Sets the storageId. - * - * @param idValue storageId - * @return StorageLocationReport.Builder - */ - public Builder setId(String idValue) { - this.id = idValue; - return this; - } - - /** - * Sets whether the volume failed or not. - * - * @param failedValue whether volume failed or not - * @return StorageLocationReport.Builder - */ - public Builder setFailed(boolean failedValue) { - this.failed = failedValue; - return this; - } - - /** - * Sets the capacity of volume. - * - * @param capacityValue capacity - * @return StorageLocationReport.Builder - */ - public Builder setCapacity(long capacityValue) { - this.capacity = capacityValue; - return this; - } - /** - * Sets the scmUsed Value. - * - * @param scmUsedValue storage space used by scm - * @return StorageLocationReport.Builder - */ - public Builder setScmUsed(long scmUsedValue) { - this.scmUsed = scmUsedValue; - return this; - } - - /** - * Sets the remaining free space value. - * - * @param remainingValue remaining free space - * @return StorageLocationReport.Builder - */ - public Builder setRemaining(long remainingValue) { - this.remaining = remainingValue; - return this; - } - - /** - * Sets the storageType. - * - * @param storageTypeValue type of the storage used - * @return StorageLocationReport.Builder - */ - public Builder setStorageType(StorageType storageTypeValue) { - this.storageType = storageTypeValue; - return this; - } - - /** - * Sets the storageLocation. - * - * @param storageLocationValue location of the volume - * @return StorageLocationReport.Builder - */ - public Builder setStorageLocation(String storageLocationValue) { - this.storageLocation = storageLocationValue; - return this; - } - - /** - * Builds and returns StorageLocationReport instance. - * - * @return StorageLocationReport - */ - public StorageLocationReport build() { - return new StorageLocationReport(id, failed, capacity, scmUsed, - remaining, storageType, storageLocation); - } - - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java deleted file mode 100644 index b17680c..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java +++ /dev/null @@ -1,90 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.impl; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.ozone.container.common.interfaces - .ContainerDeletionChoosingPolicy; -import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -/** - * TopN Ordered choosing policy that choosing containers based on pending - * deletion blocks' number. - */ -public class TopNOrderedContainerDeletionChoosingPolicy - implements ContainerDeletionChoosingPolicy { - private static final Logger LOG = - LoggerFactory.getLogger(TopNOrderedContainerDeletionChoosingPolicy.class); - - /** customized comparator used to compare differentiate container data. **/ - private static final Comparator<KeyValueContainerData> - KEY_VALUE_CONTAINER_DATA_COMPARATOR = (KeyValueContainerData c1, - KeyValueContainerData c2) -> - Integer.compare(c2.getNumPendingDeletionBlocks(), - c1.getNumPendingDeletionBlocks()); - - @Override - public List<ContainerData> chooseContainerForBlockDeletion(int count, - Map<Long, ContainerData> candidateContainers) - throws StorageContainerException { - Preconditions.checkNotNull(candidateContainers, - "Internal assertion: candidate containers cannot be null"); - - List<ContainerData> result = new LinkedList<>(); - List<KeyValueContainerData> orderedList = new LinkedList<>(); - for (ContainerData entry : candidateContainers.values()) { - orderedList.add((KeyValueContainerData)entry); - } - Collections.sort(orderedList, KEY_VALUE_CONTAINER_DATA_COMPARATOR); - - // get top N list ordered by pending deletion blocks' number - int currentCount = 0; - for (KeyValueContainerData entry : orderedList) { - if (currentCount < count) { - if (entry.getNumPendingDeletionBlocks() > 0) { - result.add(entry); - currentCount++; - - LOG.debug( - "Select container {} for block deletion, " - + "pending deletion blocks num: {}.", - entry.getContainerID(), - entry.getNumPendingDeletionBlocks()); - } else { - LOG.debug("Stop looking for next container, there is no" - + " pending deletion block contained in remaining containers."); - break; - } - } else { - break; - } - } - - return result; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java deleted file mode 100644 index 16da5d9..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.impl; - -/** - This package is contains Ozone container implementation. -**/ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BlockIterator.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BlockIterator.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BlockIterator.java deleted file mode 100644 index f6931e3..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/BlockIterator.java +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.interfaces; - - -import java.io.IOException; -import java.util.NoSuchElementException; - -/** - * Block Iterator for container. Each container type need to implement this - * interface. - * @param <T> - */ -public interface BlockIterator<T> { - - /** - * This checks if iterator has next element. If it has returns true, - * otherwise false. - * @return boolean - */ - boolean hasNext() throws IOException; - - /** - * Seek to first entry. - */ - void seekToFirst(); - - /** - * Seek to last entry. - */ - void seekToLast(); - - /** - * Get next block in the container. - * @return next block or null if there are no blocks - * @throws IOException - */ - T nextBlock() throws IOException, NoSuchElementException; - - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java deleted file mode 100644 index 9380f0c..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/Container.java +++ /dev/null @@ -1,135 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.interfaces; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.Map; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerLifeCycleState; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos; - -import org.apache.hadoop.hdfs.util.RwLock; -import org.apache.hadoop.ozone.container.common.impl.ContainerData; -import org.apache.hadoop.ozone.container.common.volume.VolumeSet; - -/** - * Interface for Container Operations. - */ -public interface Container<CONTAINERDATA extends ContainerData> extends RwLock { - - /** - * Creates a container. - * - * @throws StorageContainerException - */ - void create(VolumeSet volumeSet, VolumeChoosingPolicy volumeChoosingPolicy, - String scmId) throws StorageContainerException; - - /** - * Deletes the container. - * - * @param forceDelete - whether this container should be deleted forcibly. - * @throws StorageContainerException - */ - void delete(boolean forceDelete) throws StorageContainerException; - - /** - * Update the container. - * - * @param metaData - * @param forceUpdate if true, update container forcibly. - * @throws StorageContainerException - */ - void update(Map<String, String> metaData, boolean forceUpdate) - throws StorageContainerException; - - /** - * Get metadata about the container. - * - * @return ContainerData - Container Data. - * @throws StorageContainerException - */ - CONTAINERDATA getContainerData(); - - /** - * Get the Container Lifecycle state. - * - * @return ContainerLifeCycleState - Container State. - * @throws StorageContainerException - */ - ContainerLifeCycleState getContainerState(); - - /** - * Closes a open container, if it is already closed or does not exist a - * StorageContainerException is thrown. - * - * @throws StorageContainerException - */ - void close() throws StorageContainerException; - - /** - * Return the ContainerType for the container. - */ - ContainerProtos.ContainerType getContainerType(); - - /** - * Returns containerFile. - */ - File getContainerFile(); - - /** - * updates the DeleteTransactionId. - * @param deleteTransactionId - */ - void updateDeleteTransactionId(long deleteTransactionId); - - /** - * Returns blockIterator for the container. - * @return BlockIterator - * @throws IOException - */ - BlockIterator blockIterator() throws IOException; - - /** - * Import the container from an external archive. - */ - void importContainerData(InputStream stream, - ContainerPacker<CONTAINERDATA> packer) throws IOException; - - /** - * Export all the data of the container to one output archive with the help - * of the packer. - * - */ - void exportContainerData(OutputStream stream, - ContainerPacker<CONTAINERDATA> packer) throws IOException; - - /** - * Returns containerReport for the container. - */ - StorageContainerDatanodeProtocolProtos.ContainerInfo getContainerReport() - throws StorageContainerException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java deleted file mode 100644 index 84c4f90..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.interfaces; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.scm.container.common.helpers - .StorageContainerException; -import org.apache.hadoop.ozone.container.common.impl.ContainerData; - -import java.util.List; -import java.util.Map; - -/** - * This interface is used for choosing desired containers for - * block deletion. - */ -public interface ContainerDeletionChoosingPolicy { - - /** - * Chooses desired containers for block deletion. - * @param count - * how many to return - * @param candidateContainers - * candidate containers collection - * @return container data list - * @throws StorageContainerException - */ - List<ContainerData> chooseContainerForBlockDeletion(int count, - Map<Long, ContainerData> candidateContainers) - throws StorageContainerException; - - /** - * Determine if the container has suitable type for this policy. - * @param type type of the container - * @return whether the container type suitable for this policy. - */ - default boolean isValidContainerType(ContainerProtos.ContainerType type) { - if (type == ContainerProtos.ContainerType.KeyValueContainer) { - return true; - } - return false; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java deleted file mode 100644 index 18644bb..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.container.common.interfaces; - -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandRequestProto; -import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos - .ContainerCommandResponseProto; - -/** - * Dispatcher acts as the bridge between the transport layer and - * the actual container layer. This layer is capable of transforming - * protobuf objects into corresponding class and issue the function call - * into the lower layers. - * - * The reply from the request is dispatched to the client. - */ -public interface ContainerDispatcher { - /** - * Dispatches commands to container layer. - * @param msg - Command Request - * @return Command Response - */ - ContainerCommandResponseProto dispatch(ContainerCommandRequestProto msg); - - /** - * Initialize the Dispatcher. - */ - void init(); - - /** - * Shutdown Dispatcher services. - */ - void shutdown(); - - /** - * Returns the handler for the specified containerType. - * @param containerType - * @return - */ - Handler getHandler(ContainerProtos.ContainerType containerType); - - /** - * If scmId is not set, this will set scmId, otherwise it is a no-op. - * @param scmId - */ - void setScmId(String scmId); -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c392da8/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java ---------------------------------------------------------------------- diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java deleted file mode 100644 index 9c5fcea..0000000 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.container.common.interfaces; - -import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport; - -import java.io.IOException; -import java.nio.file.Path; - -/** - * Returns physical path locations, where the containers will be created. - */ -public interface ContainerLocationManager { - /** - * Returns the path where the container should be placed from a set of - * locations. - * - * @return A path where we should place this container and metadata. - * @throws IOException - */ - Path getContainerPath() throws IOException; - - /** - * Returns the path where the container Data file are stored. - * - * @return a path where we place the LevelDB and data files of a container. - * @throws IOException - */ - Path getDataPath(String containerName) throws IOException; - - /** - * Returns an array of storage location usage report. - * @return storage location usage report. - */ - StorageLocationReport[] getLocationReport() throws IOException; - - /** - * Supports clean shutdown of container. - * - * @throws IOException - */ - void shutdown() throws IOException; -} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
