HDFS-12259. Ozone: OzoneClient: Refactor move ozone client from hadoop-hdfs to hadoop-hdfs-client. Contributed by Nandakumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43d38114 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43d38114 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43d38114 Branch: refs/heads/HDFS-7240 Commit: 43d38114e695a2dd1d2da21f848ba8142749faa3 Parents: b153dbb Author: Xiaoyu Yao <x...@apache.org> Authored: Tue Aug 8 13:36:05 2017 -0700 Committer: Xiaoyu Yao <x...@apache.org> Committed: Tue Aug 8 13:36:05 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/cblock/CBlockConfigKeys.java | 178 +++++ .../hadoop/ksm/helpers/KsmBucketArgs.java | 233 ------ .../hadoop/ksm/helpers/KsmBucketInfo.java | 212 ------ .../apache/hadoop/ksm/helpers/KsmKeyArgs.java | 88 --- .../apache/hadoop/ksm/helpers/KsmKeyInfo.java | 161 ----- .../hadoop/ksm/helpers/KsmKeyLocationInfo.java | 137 ---- .../hadoop/ksm/helpers/KsmOzoneAclMap.java | 110 --- .../hadoop/ksm/helpers/KsmVolumeArgs.java | 223 ------ .../apache/hadoop/ksm/helpers/VolumeArgs.java | 140 ---- .../apache/hadoop/ksm/helpers/package-info.java | 18 - .../org/apache/hadoop/ksm/package-info.java | 21 - .../ksm/protocol/KeySpaceManagerProtocol.java | 212 ------ .../hadoop/ksm/protocol/package-info.java | 19 - ...ceManagerProtocolClientSideTranslatorPB.java | 651 ----------------- .../protocolPB/KeySpaceManagerProtocolPB.java | 34 - .../hadoop/ksm/protocolPB/package-info.java | 19 - .../java/org/apache/hadoop/ozone/OzoneAcl.java | 5 + .../apache/hadoop/ozone/OzoneConfigKeys.java | 119 ++++ .../apache/hadoop/ozone/OzoneConfiguration.java | 43 ++ .../apache/hadoop/ozone/client/OzoneBucket.java | 118 ++++ .../apache/hadoop/ozone/client/OzoneClient.java | 415 +++++++++++ .../hadoop/ozone/client/OzoneClientFactory.java | 95 +++ .../hadoop/ozone/client/OzoneClientUtils.java | 707 +++++++++++++++++++ .../apache/hadoop/ozone/client/OzoneKey.java | 109 +++ .../apache/hadoop/ozone/client/OzoneVolume.java | 107 +++ .../ozone/client/io/ChunkGroupInputStream.java | 210 ++++++ .../ozone/client/io/ChunkGroupOutputStream.java | 327 +++++++++ .../ozone/client/io/LengthInputStream.java | 49 ++ .../client/io/OzoneContainerTranslation.java | 51 ++ .../ozone/client/io/OzoneInputStream.java | 52 ++ .../ozone/client/io/OzoneOutputStream.java | 60 ++ .../hadoop/ozone/client/io/package-info.java | 23 + .../hadoop/ozone/client/package-info.java | 23 + .../ozone/client/rest/OzoneRestClient.java | 510 +++++++++++++ .../ozone/client/rest/headers/Header.java | 70 ++ .../ozone/client/rest/headers/package-info.java | 26 + .../hadoop/ozone/client/rest/package-info.java | 23 + .../hadoop/ozone/client/rpc/OzoneRpcClient.java | 578 +++++++++++++++ .../hadoop/ozone/client/rpc/package-info.java | 23 + .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 77 ++ .../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 ++++++ .../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 212 ++++++ .../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 88 +++ .../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 161 +++++ .../ozone/ksm/helpers/KsmKeyLocationInfo.java | 137 ++++ .../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 +++ .../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 ++++++ .../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 ++++ .../hadoop/ozone/ksm/helpers/package-info.java | 18 + .../apache/hadoop/ozone/ksm/package-info.java | 21 + .../ksm/protocol/KeySpaceManagerProtocol.java | 212 ++++++ .../hadoop/ozone/ksm/protocol/package-info.java | 19 + ...ceManagerProtocolClientSideTranslatorPB.java | 651 +++++++++++++++++ .../protocolPB/KeySpaceManagerProtocolPB.java | 34 + .../ozone/ksm/protocolPB/package-info.java | 19 + .../java/org/apache/hadoop/scm/TestArchive.java | 106 --- .../org/apache/hadoop/scm/package-info.java | 21 - .../java/org/apache/hadoop/scm/TestArchive.java | 106 +++ .../org/apache/hadoop/scm/package-info.java | 21 + .../apache/hadoop/cblock/CBlockConfigKeys.java | 178 ----- .../org/apache/hadoop/cblock/CBlockManager.java | 2 +- .../cblock/client/CBlockVolumeClient.java | 2 +- .../server/datanode/ObjectStoreHandler.java | 12 +- .../hdfs/server/datanode/web/URLDispatcher.java | 2 +- .../org/apache/hadoop/hdfs/tools/GetConf.java | 2 +- .../org/apache/hadoop/ozone/OzoneBucket.java | 117 --- .../org/apache/hadoop/ozone/OzoneClient.java | 414 ----------- .../apache/hadoop/ozone/OzoneClientFactory.java | 69 -- .../apache/hadoop/ozone/OzoneClientImpl.java | 570 --------------- .../apache/hadoop/ozone/OzoneClientUtils.java | 705 ------------------ .../apache/hadoop/ozone/OzoneConfigKeys.java | 119 ---- .../apache/hadoop/ozone/OzoneConfiguration.java | 43 -- .../java/org/apache/hadoop/ozone/OzoneKey.java | 109 --- .../org/apache/hadoop/ozone/OzoneVolume.java | 107 --- .../statemachine/DatanodeStateMachine.java | 2 +- .../statemachine/EndpointStateMachine.java | 2 +- .../statemachine/SCMConnectionManager.java | 2 +- .../states/datanode/InitDatanodeState.java | 2 +- .../hadoop/ozone/io/OzoneInputStream.java | 53 -- .../hadoop/ozone/io/OzoneOutputStream.java | 62 -- .../apache/hadoop/ozone/io/package-info.java | 23 - .../apache/hadoop/ozone/ksm/BucketManager.java | 4 +- .../hadoop/ozone/ksm/BucketManagerImpl.java | 4 +- .../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 77 -- .../org/apache/hadoop/ozone/ksm/KeyManager.java | 4 +- .../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 6 +- .../hadoop/ozone/ksm/KeySpaceManager.java | 16 +- .../hadoop/ozone/ksm/MetadataManager.java | 6 +- .../hadoop/ozone/ksm/MetadataManagerImpl.java | 6 +- .../apache/hadoop/ozone/ksm/VolumeManager.java | 2 +- .../hadoop/ozone/ksm/VolumeManagerImpl.java | 8 +- ...ceManagerProtocolServerSideTranslatorPB.java | 18 +- .../ozone/scm/StorageContainerManager.java | 2 +- .../hadoop/ozone/scm/node/SCMNodeManager.java | 2 +- .../hadoop/ozone/web/OzoneHttpServer.java | 2 +- .../hadoop/ozone/web/client/OzoneBucket.java | 4 +- .../ozone/web/client/OzoneRestClient.java | 4 +- .../hadoop/ozone/web/client/OzoneVolume.java | 4 +- .../ozone/web/handlers/BucketHandler.java | 2 +- .../web/handlers/BucketProcessTemplate.java | 2 +- .../hadoop/ozone/web/handlers/KeyHandler.java | 4 +- .../ozone/web/handlers/KeyProcessTemplate.java | 2 +- .../ozone/web/handlers/ServiceFilter.java | 2 +- .../ozone/web/handlers/VolumeHandler.java | 2 +- .../apache/hadoop/ozone/web/headers/Header.java | 70 -- .../hadoop/ozone/web/headers/package-info.java | 26 - .../hadoop/ozone/web/interfaces/Bucket.java | 2 +- .../hadoop/ozone/web/interfaces/Keys.java | 2 +- .../ozone/web/interfaces/StorageHandler.java | 2 +- .../hadoop/ozone/web/interfaces/Volume.java | 2 +- .../web/localstorage/LocalStorageHandler.java | 2 +- .../web/localstorage/OzoneMetadataManager.java | 2 +- .../LengthInputStreamMessageBodyWriter.java | 2 +- .../ozone/web/ozShell/keys/ListKeyHandler.java | 2 - .../hadoop/ozone/web/request/OzoneQuota.java | 2 +- .../web/storage/ChunkGroupInputStream.java | 211 ------ .../web/storage/ChunkGroupOutputStream.java | 325 --------- .../web/storage/DistributedStorageHandler.java | 19 +- .../web/storage/OzoneContainerTranslation.java | 261 ------- .../hadoop/ozone/web/userauth/Simple.java | 2 +- .../hadoop/ozone/web/utils/OzoneUtils.java | 4 +- .../java/org/apache/hadoop/ozone/Corona.java | 6 +- .../hadoop/ozone/TestOzoneClientImpl.java | 357 ---------- .../hadoop/ozone/TestOzoneClientUtils.java | 350 --------- .../ozone/client/TestOzoneClientUtils.java | 362 ++++++++++ .../hadoop/ozone/client/package-info.java | 23 + .../ozone/client/rpc/TestOzoneRpcClient.java | 368 ++++++++++ .../hadoop/ozone/client/rpc/package-info.java | 23 + .../container/ozoneimpl/TestOzoneContainer.java | 1 - .../hadoop/ozone/ksm/TestBucketManagerImpl.java | 4 +- .../hadoop/ozone/ksm/TestChunkStreams.java | 4 +- .../hadoop/ozone/web/TestOzoneHelper.java | 2 +- .../hadoop/ozone/web/TestOzoneWebAccess.java | 2 +- .../hadoop/ozone/web/client/TestVolume.java | 2 +- 134 files changed, 7278 insertions(+), 6750 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java new file mode 100644 index 0000000..35b8961 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.cblock; + +import static java.lang.Thread.NORM_PRIORITY; + +/** + * This class contains constants for configuration keys used in CBlock. + */ +public final class CBlockConfigKeys { + public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY = + "dfs.cblock.servicerpc-address"; + public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT = + 9810; + public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT = + "0.0.0.0"; + + public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY = + "dfs.cblock.jscsi-address"; + + //The port on CBlockManager node for jSCSI to ask + public static final String DFS_CBLOCK_JSCSI_PORT_KEY = + "dfs.cblock.jscsi.port"; + public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT = + 9811; + + public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY = + "dfs.cblock.service.rpc-bind-host"; + public static final String DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY = + "dfs.cblock.jscsi.rpc-bind-host"; + + // default block size is 4KB + public static final int DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT = + 4096; + + public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY = + "dfs.storage.service.handler.count"; + public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10; + + public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY = + "dfs.cblock.service.leveldb.path"; + //TODO : find a better place + public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT = + "/tmp/cblock_levelDB.dat"; + + + public static final String DFS_CBLOCK_DISK_CACHE_PATH_KEY = + "dfs.cblock.disk.cache.path"; + public static final String DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT = + "/tmp/cblockCacheDB"; + /** + * Setting this flag to true makes the block layer compute a sha256 hash of + * the data and log that information along with block ID. This is very + * useful for doing trace based simulation of various workloads. Since it is + * computing a hash for each block this could be expensive, hence default + * is false. + */ + public static final String DFS_CBLOCK_TRACE_IO = "dfs.cblock.trace.io"; + public static final boolean DFS_CBLOCK_TRACE_IO_DEFAULT = false; + + public static final String DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO = + "dfs.cblock.short.circuit.io"; + public static final boolean DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT = + false; + + /** + * Cache size in 1000s of entries. 256 indicates 256 * 1024. + */ + public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB = + "dfs.cblock.cache.cache.size.in.kb"; + public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256; + + /** + * Minimum Number of threads that cache pool will use for background I/O. + */ + public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE = + "dfs.cblock.cache.core.min.pool.size"; + public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16; + + /** + * Maximum Number of threads that cache pool will use for background I/O. + */ + + public static final String DFS_CBLOCK_CACHE_MAX_POOL_SIZE = + "dfs.cblock.cache.max.pool.size"; + public static final int DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT = 256; + + /** + * Number of seconds to keep the Thread alive when it is idle. + */ + public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE_SECONDS = + "dfs.cblock.cache.keep.alive.seconds"; + public static final long DFS_CBLOCK_CACHE_KEEP_ALIVE_SECONDS_DEFAULT = 60; + + /** + * Priority of cache flusher thread, affecting the relative performance of + * write and read. + */ + public static final String DFS_CBLOCK_CACHE_THREAD_PRIORITY = + "dfs.cblock.cache.thread.priority"; + public static final int DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT = + NORM_PRIORITY; + + /** + * Block Buffer size in terms of blockID entries, 512 means 512 blockIDs. + */ + public static final String DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE = + "dfs.cblock.cache.block.buffer.size"; + public static final int DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT = 512; + + public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_SECONDS = + "dfs.cblock.block.buffer.flush.interval.seconds"; + public static final int + DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_SECONDS_DEFAULT = 60; + + // jscsi server settings + public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY = + "dfs.cblock.jscsi.server.address"; + public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT = + "127.0.0.1"; + public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY = + "dfs.cblock.jscsi.cblock.server.address"; + public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT = + "127.0.0.1"; + + // to what address cblock server should talk to scm? + public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY = + "dfs.cblock.scm.ipaddress"; + public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT = + "127.0.0.1"; + public static final String DFS_CBLOCK_SCM_PORT_KEY = + "dfs.cblock.scm.port"; + public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860; + + public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY = + "dfs.cblock.container.size"; + public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT = + 5; + + // LevelDB cache file uses an off-heap cache in LevelDB of 256 MB. + public static final String DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY = + "dfs.cblock.cache.leveldb.cache.size.mb"; + public static final int DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT = 256; + + /** + * Cache does an best case attempt to write a block to a container. + * At some point of time, we will need to handle the case where we did try + * 64K times and is till not able to write to the container. + * + * TODO: We will need cBlock Server to allow us to do a remapping of the + * block location in case of failures, at that point we should reduce the + * retry count to a more normal number. This is approximately 18 hours of + * retry. + */ + public static final String DFS_CBLOCK_CACHE_MAX_RETRY_KEY = + "dfs.cblock.cache.max.retry"; + public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT = + 64 * 1024; + + private CBlockConfigKeys() { + + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketArgs.java deleted file mode 100644 index a682f5f..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketArgs.java +++ /dev/null @@ -1,233 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; - -import java.util.List; -import java.util.stream.Collectors; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.BucketArgs; -import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; - -/** - * A class that encapsulates Bucket Arguments. - */ -public final class KsmBucketArgs { - /** - * Name of the volume in which the bucket belongs to. - */ - private final String volumeName; - /** - * Name of the bucket. - */ - private final String bucketName; - /** - * ACL's that are to be added for the bucket. - */ - private List<OzoneAcl> addAcls; - /** - * ACL's that are to be removed from the bucket. - */ - private List<OzoneAcl> removeAcls; - /** - * Bucket Version flag. - */ - private Boolean isVersionEnabled; - /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] - */ - private StorageType storageType; - - /** - * Private constructor, constructed via builder. - * @param volumeName - Volume name. - * @param bucketName - Bucket name. - * @param addAcls - ACL's to be added. - * @param removeAcls - ACL's to be removed. - * @param isVersionEnabled - Bucket version flag. - * @param storageType - Storage type to be used. - */ - private KsmBucketArgs(String volumeName, String bucketName, - List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls, - Boolean isVersionEnabled, StorageType storageType) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.addAcls = addAcls; - this.removeAcls = removeAcls; - this.isVersionEnabled = isVersionEnabled; - this.storageType = storageType; - } - - /** - * Returns the Volume Name. - * @return String. - */ - public String getVolumeName() { - return volumeName; - } - - /** - * Returns the Bucket Name. - * @return String - */ - public String getBucketName() { - return bucketName; - } - - /** - * Returns the ACL's that are to be added. - * @return List<OzoneAclInfo> - */ - public List<OzoneAcl> getAddAcls() { - return addAcls; - } - - /** - * Returns the ACL's that are to be removed. - * @return List<OzoneAclInfo> - */ - public List<OzoneAcl> getRemoveAcls() { - return removeAcls; - } - - /** - * Returns true if bucket version is enabled, else false. - * @return isVersionEnabled - */ - public Boolean getIsVersionEnabled() { - return isVersionEnabled; - } - - /** - * Returns the type of storage to be used. - * @return StorageType - */ - public StorageType getStorageType() { - return storageType; - } - - /** - * Returns new builder class that builds a KsmBucketArgs. - * - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder for KsmBucketArgs. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private List<OzoneAcl> addAcls; - private List<OzoneAcl> removeAcls; - private Boolean isVersionEnabled; - private StorageType storageType; - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setAddAcls(List<OzoneAcl> acls) { - this.addAcls = acls; - return this; - } - - public Builder setRemoveAcls(List<OzoneAcl> acls) { - this.removeAcls = acls; - return this; - } - - public Builder setIsVersionEnabled(Boolean versionFlag) { - this.isVersionEnabled = versionFlag; - return this; - } - - public Builder setStorageType(StorageType storage) { - this.storageType = storage; - return this; - } - - /** - * Constructs the KsmBucketArgs. - * @return instance of KsmBucketArgs. - */ - public KsmBucketArgs build() { - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - return new KsmBucketArgs(volumeName, bucketName, addAcls, - removeAcls, isVersionEnabled, storageType); - } - } - - /** - * Creates BucketArgs protobuf from KsmBucketArgs. - */ - public BucketArgs getProtobuf() { - BucketArgs.Builder builder = BucketArgs.newBuilder(); - builder.setVolumeName(volumeName) - .setBucketName(bucketName); - if(addAcls != null && !addAcls.isEmpty()) { - builder.addAllAddAcls(addAcls.stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); - } - if(removeAcls != null && !removeAcls.isEmpty()) { - builder.addAllRemoveAcls(removeAcls.stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); - } - if(isVersionEnabled != null) { - builder.setIsVersionEnabled(isVersionEnabled); - } - if(storageType != null) { - builder.setStorageType( - PBHelperClient.convertStorageType(storageType)); - } - return builder.build(); - } - - /** - * Parses BucketInfo protobuf and creates KsmBucketArgs. - * @param bucketArgs - * @return instance of KsmBucketArgs - */ - public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) { - return new KsmBucketArgs(bucketArgs.getVolumeName(), - bucketArgs.getBucketName(), - bucketArgs.getAddAclsList().stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), - bucketArgs.getRemoveAclsList().stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), - bucketArgs.hasIsVersionEnabled() ? - bucketArgs.getIsVersionEnabled() : null, - bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType( - bucketArgs.getStorageType()) : null); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketInfo.java deleted file mode 100644 index effbb11..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmBucketInfo.java +++ /dev/null @@ -1,212 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; -import org.apache.hadoop.ozone.OzoneAcl; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.BucketInfo; -import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; - -import java.util.LinkedList; -import java.util.List; -import java.util.stream.Collectors; - -/** - * A class that encapsulates Bucket Info. - */ -public final class KsmBucketInfo { - /** - * Name of the volume in which the bucket belongs to. - */ - private final String volumeName; - /** - * Name of the bucket. - */ - private final String bucketName; - /** - * ACL Information. - */ - private List<OzoneAcl> acls; - /** - * Bucket Version flag. - */ - private Boolean isVersionEnabled; - /** - * Type of storage to be used for this bucket. - * [RAM_DISK, SSD, DISK, ARCHIVE] - */ - private StorageType storageType; - - /** - * Private constructor, constructed via builder. - * @param volumeName - Volume name. - * @param bucketName - Bucket name. - * @param acls - list of ACLs. - * @param isVersionEnabled - Bucket version flag. - * @param storageType - Storage type to be used. - */ - private KsmBucketInfo(String volumeName, String bucketName, - List<OzoneAcl> acls, boolean isVersionEnabled, - StorageType storageType) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.acls = acls; - this.isVersionEnabled = isVersionEnabled; - this.storageType = storageType; - } - - /** - * Returns the Volume Name. - * @return String. - */ - public String getVolumeName() { - return volumeName; - } - - /** - * Returns the Bucket Name. - * @return String - */ - public String getBucketName() { - return bucketName; - } - - /** - * Returns the ACL's associated with this bucket. - * @return List<OzoneAcl> - */ - public List<OzoneAcl> getAcls() { - return acls; - } - - /** - * Returns true if bucket version is enabled, else false. - * @return isVersionEnabled - */ - public boolean getIsVersionEnabled() { - return isVersionEnabled; - } - - /** - * Returns the type of storage to be used. - * @return StorageType - */ - public StorageType getStorageType() { - return storageType; - } - - /** - * Returns new builder class that builds a KsmBucketInfo. - * - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder for KsmBucketInfo. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private List<OzoneAcl> acls; - private Boolean isVersionEnabled; - private StorageType storageType; - - Builder() { - //Default values - this.acls = new LinkedList<>(); - this.isVersionEnabled = false; - this.storageType = StorageType.DISK; - } - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setAcls(List<OzoneAcl> listOfAcls) { - this.acls = listOfAcls; - return this; - } - - public Builder setIsVersionEnabled(Boolean versionFlag) { - this.isVersionEnabled = versionFlag; - return this; - } - - public Builder setStorageType(StorageType storage) { - this.storageType = storage; - return this; - } - - /** - * Constructs the KsmBucketInfo. - * @return instance of KsmBucketInfo. - */ - public KsmBucketInfo build() { - Preconditions.checkNotNull(volumeName); - Preconditions.checkNotNull(bucketName); - Preconditions.checkNotNull(acls); - Preconditions.checkNotNull(isVersionEnabled); - Preconditions.checkNotNull(storageType); - return new KsmBucketInfo(volumeName, bucketName, acls, - isVersionEnabled, storageType); - } - } - - /** - * Creates BucketInfo protobuf from KsmBucketInfo. - */ - public BucketInfo getProtobuf() { - return BucketInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .addAllAcls(acls.stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())) - .setIsVersionEnabled(isVersionEnabled) - .setStorageType(PBHelperClient.convertStorageType( - storageType)) - .build(); - } - - /** - * Parses BucketInfo protobuf and creates KsmBucketInfo. - * @param bucketInfo - * @return instance of KsmBucketInfo - */ - public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) { - return new KsmBucketInfo( - bucketInfo.getVolumeName(), - bucketInfo.getBucketName(), - bucketInfo.getAclsList().stream().map( - KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), - bucketInfo.getIsVersionEnabled(), - PBHelperClient.convertStorageType( - bucketInfo.getStorageType())); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyArgs.java deleted file mode 100644 index a034ed3..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyArgs.java +++ /dev/null @@ -1,88 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; - -/** - * Args for key. Client use this to specify key's attributes on key creation - * (putKey()). - */ -public final class KsmKeyArgs { - private final String volumeName; - private final String bucketName; - private final String keyName; - - private final long dataSize; - - private KsmKeyArgs(String volumeName, String bucketName, String keyName, - long dataSize) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.keyName = keyName; - this.dataSize = dataSize; - } - - public String getVolumeName() { - return volumeName; - } - - public String getBucketName() { - return bucketName; - } - - public String getKeyName() { - return keyName; - } - - public long getDataSize() { - return dataSize; - } - - /** - * Builder class of KsmKeyArgs. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private String keyName; - private long dataSize; - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setKeyName(String key) { - this.keyName = key; - return this; - } - - public Builder setDataSize(long size) { - this.dataSize = size; - return this; - } - - public KsmKeyArgs build() { - return new KsmKeyArgs(volumeName, bucketName, keyName, dataSize); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java deleted file mode 100644 index f46ec89..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyInfo.java +++ /dev/null @@ -1,161 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; - -import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo; - -import java.util.List; -import java.util.stream.Collectors; - -/** - * Args for key block. The block instance for the key requested in putKey. - * This is returned from KSM to client, and client use class to talk to - * datanode. Also, this is the metadata written to ksm.db on server side. - */ -public final class KsmKeyInfo { - private final String volumeName; - private final String bucketName; - // name of key client specified - private final String keyName; - private final long dataSize; - private List<KsmKeyLocationInfo> keyLocationList; - private final long creationTime; - private final long modificationTime; - - private KsmKeyInfo(String volumeName, String bucketName, String keyName, - List<KsmKeyLocationInfo> locationInfos, long dataSize, long creationTime, - long modificationTime) { - this.volumeName = volumeName; - this.bucketName = bucketName; - this.keyName = keyName; - this.dataSize = dataSize; - this.keyLocationList = locationInfos; - this.creationTime = creationTime; - this.modificationTime = modificationTime; - } - - public String getVolumeName() { - return volumeName; - } - - public String getBucketName() { - return bucketName; - } - - public String getKeyName() { - return keyName; - } - - public long getDataSize() { - return dataSize; - } - - public List<KsmKeyLocationInfo> getKeyLocationList() { - return keyLocationList; - } - - public long getCreationTime() { - return creationTime; - } - - public long getModificationTime() { - return modificationTime; - } - - /** - * Builder of KsmKeyInfo. - */ - public static class Builder { - private String volumeName; - private String bucketName; - private String keyName; - private long dataSize; - private List<KsmKeyLocationInfo> ksmKeyLocationInfos; - private long creationTime; - private long modificationTime; - - public Builder setVolumeName(String volume) { - this.volumeName = volume; - return this; - } - - public Builder setBucketName(String bucket) { - this.bucketName = bucket; - return this; - } - - public Builder setKeyName(String key) { - this.keyName = key; - return this; - } - - public Builder setKsmKeyLocationInfos( - List<KsmKeyLocationInfo> ksmKeyLocationInfoList) { - this.ksmKeyLocationInfos = ksmKeyLocationInfoList; - return this; - } - - public Builder setDataSize(long size) { - this.dataSize = size; - return this; - } - - public Builder setCreationTime(long creationTime) { - this.creationTime = creationTime; - return this; - } - - public Builder setModificationTime(long modificationTime) { - this.modificationTime = modificationTime; - return this; - } - - public KsmKeyInfo build() { - return new KsmKeyInfo( - volumeName, bucketName, keyName, ksmKeyLocationInfos, - dataSize, creationTime, modificationTime); - } - } - - public KeyInfo getProtobuf() { - return KeyInfo.newBuilder() - .setVolumeName(volumeName) - .setBucketName(bucketName) - .setKeyName(keyName) - .setDataSize(dataSize) - .addAllKeyLocationList(keyLocationList.stream() - .map(KsmKeyLocationInfo::getProtobuf).collect(Collectors.toList())) - .setCreationTime(creationTime) - .setModificationTime(modificationTime) - .build(); - } - - public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) { - return new KsmKeyInfo( - keyInfo.getVolumeName(), - keyInfo.getBucketName(), - keyInfo.getKeyName(), - keyInfo.getKeyLocationListList().stream() - .map(KsmKeyLocationInfo::getFromProtobuf) - .collect(Collectors.toList()), - keyInfo.getDataSize(), - keyInfo.getCreationTime(), - keyInfo.getModificationTime()); - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyLocationInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyLocationInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyLocationInfo.java deleted file mode 100644 index 62d20f6..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmKeyLocationInfo.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package org.apache.hadoop.ksm.helpers; - -import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocation; - -/** - * One key can be too huge to fit in one container. In which case it gets split - * into a number of subkeys. This class represents one such subkey instance. - */ -public final class KsmKeyLocationInfo { - private final String containerName; - // name of the block id SCM assigned for the key - private final String blockID; - private final boolean shouldCreateContainer; - // the id of this subkey in all the subkeys. - private final int index; - private final long length; - private final long offset; - - private KsmKeyLocationInfo(String containerName, - String blockID, boolean shouldCreateContainer, int index, - long length, long offset) { - this.containerName = containerName; - this.blockID = blockID; - this.shouldCreateContainer = shouldCreateContainer; - this.index = index; - this.length = length; - this.offset = offset; - } - - public String getContainerName() { - return containerName; - } - - public String getBlockID() { - return blockID; - } - - public boolean getShouldCreateContainer() { - return shouldCreateContainer; - } - - public int getIndex() { - return index; - } - - public long getLength() { - return length; - } - - public long getOffset() { - return offset; - } - - /** - * Builder of KsmKeyLocationInfo. - */ - public static class Builder { - private String containerName; - private String blockID; - private boolean shouldCreateContainer; - // the id of this subkey in all the subkeys. - private int index; - private long length; - private long offset; - public Builder setContainerName(String container) { - this.containerName = container; - return this; - } - - public Builder setBlockID(String block) { - this.blockID = block; - return this; - } - - public Builder setShouldCreateContainer(boolean create) { - this.shouldCreateContainer = create; - return this; - } - - public Builder setIndex(int id) { - this.index = id; - return this; - } - - public Builder setLength(long len) { - this.length = len; - return this; - } - - public Builder setOffset(long off) { - this.offset = off; - return this; - } - - public KsmKeyLocationInfo build() { - return new KsmKeyLocationInfo(containerName, blockID, - shouldCreateContainer, index, length, offset); - } - } - - public KeyLocation getProtobuf() { - return KeyLocation.newBuilder() - .setContainerName(containerName) - .setBlockID(blockID) - .setShouldCreateContainer(shouldCreateContainer) - .setIndex(index) - .setLength(length) - .setOffset(offset) - .build(); - } - - public static KsmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) { - return new KsmKeyLocationInfo( - keyLocation.getContainerName(), - keyLocation.getBlockID(), - keyLocation.getShouldCreateContainer(), - keyLocation.getIndex(), - keyLocation.getLength(), - keyLocation.getOffset()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmOzoneAclMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmOzoneAclMap.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmOzoneAclMap.java deleted file mode 100644 index 1c4c9cb..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmOzoneAclMap.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ksm.helpers; - -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType; - -import java.util.List; -import java.util.LinkedList; -import java.util.Map; -import java.util.ArrayList; -import java.util.HashMap; - -/** - * This helper class keeps a map of all user and their permissions. - */ -public class KsmOzoneAclMap { - // per Acl Type user:rights map - private ArrayList<Map<String, OzoneAclRights>> aclMaps; - - KsmOzoneAclMap() { - aclMaps = new ArrayList<>(); - for (OzoneAclType aclType : OzoneAclType.values()) { - aclMaps.add(aclType.ordinal(), new HashMap<>()); - } - } - - private Map<String, OzoneAclRights> getMap(OzoneAclType type) { - return aclMaps.get(type.ordinal()); - } - - // For a given acl type and user, get the stored acl - private OzoneAclRights getAcl(OzoneAclType type, String user) { - return getMap(type).get(user); - } - - // Add a new acl to the map - public void addAcl(OzoneAclInfo acl) { - getMap(acl.getType()).put(acl.getName(), acl.getRights()); - } - - // for a given acl, check if the user has access rights - public boolean hasAccess(OzoneAclInfo acl) { - OzoneAclRights storedRights = getAcl(acl.getType(), acl.getName()); - if (storedRights != null) { - switch (acl.getRights()) { - case READ: - return (storedRights == OzoneAclRights.READ) - || (storedRights == OzoneAclRights.READ_WRITE); - case WRITE: - return (storedRights == OzoneAclRights.WRITE) - || (storedRights == OzoneAclRights.READ_WRITE); - case READ_WRITE: - return (storedRights == OzoneAclRights.READ_WRITE); - default: - return false; - } - } else { - return false; - } - } - - // Convert this map to OzoneAclInfo Protobuf List - public List<OzoneAclInfo> ozoneAclGetProtobuf() { - List<OzoneAclInfo> aclList = new LinkedList<>(); - for (OzoneAclType type: OzoneAclType.values()) { - for (Map.Entry<String, OzoneAclRights> entry : - aclMaps.get(type.ordinal()).entrySet()) { - OzoneAclInfo aclInfo = OzoneAclInfo.newBuilder() - .setName(entry.getKey()) - .setType(type) - .setRights(entry.getValue()) - .build(); - aclList.add(aclInfo); - } - } - - return aclList; - } - - // Create map from list of OzoneAclInfos - public static KsmOzoneAclMap ozoneAclGetFromProtobuf( - List<OzoneAclInfo> aclList) { - KsmOzoneAclMap aclMap = new KsmOzoneAclMap(); - for (OzoneAclInfo acl : aclList) { - aclMap.addAcl(acl); - } - return aclMap; - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java deleted file mode 100644 index f7c6593..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/KsmVolumeArgs.java +++ /dev/null @@ -1,223 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; - -import com.google.common.base.Preconditions; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.VolumeInfo; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; - -import java.io.IOException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - - -/** - * A class that encapsulates the KsmVolumeArgs Args. - */ -public final class KsmVolumeArgs { - private final String adminName; - private final String ownerName; - private final String volume; - private final long creationTime; - private final long quotaInBytes; - private final Map<String, String> keyValueMap; - private final KsmOzoneAclMap aclMap; - - /** - * Private constructor, constructed via builder. - * @param adminName - Administrator's name. - * @param ownerName - Volume owner's name - * @param volume - volume name - * @param quotaInBytes - Volume Quota in bytes. - * @param keyValueMap - keyValue map. - * @param aclMap - User to access rights map. - * @param creationTime - Volume creation time. - */ - private KsmVolumeArgs(String adminName, String ownerName, String volume, - long quotaInBytes, Map<String, String> keyValueMap, - KsmOzoneAclMap aclMap, long creationTime) { - this.adminName = adminName; - this.ownerName = ownerName; - this.volume = volume; - this.quotaInBytes = quotaInBytes; - this.keyValueMap = keyValueMap; - this.aclMap = aclMap; - this.creationTime = creationTime; - } - - /** - * Returns the Admin Name. - * @return String. - */ - public String getAdminName() { - return adminName; - } - - /** - * Returns the owner Name. - * @return String - */ - public String getOwnerName() { - return ownerName; - } - - /** - * Returns the volume Name. - * @return String - */ - public String getVolume() { - return volume; - } - - /** - * Returns creation time. - * @return long - */ - public long getCreationTime() { - return creationTime; - } - - /** - * Returns Quota in Bytes. - * @return long, Quota in bytes. - */ - public long getQuotaInBytes() { - return quotaInBytes; - } - - public Map<String, String> getKeyValueMap() { - return keyValueMap; - } - - public KsmOzoneAclMap getAclMap() { - return aclMap; - } - /** - * Returns new builder class that builds a KsmVolumeArgs. - * - * @return Builder - */ - public static Builder newBuilder() { - return new Builder(); - } - - /** - * Builder for KsmVolumeArgs. - */ - public static class Builder { - private String adminName; - private String ownerName; - private String volume; - private long creationTime; - private long quotaInBytes; - private Map<String, String> keyValueMap; - private KsmOzoneAclMap aclMap; - - /** - * Constructs a builder. - */ - Builder() { - keyValueMap = new HashMap<>(); - aclMap = new KsmOzoneAclMap(); - } - - public Builder setAdminName(String adminName) { - this.adminName = adminName; - return this; - } - - public Builder setOwnerName(String ownerName) { - this.ownerName = ownerName; - return this; - } - - public Builder setVolume(String volume) { - this.volume = volume; - return this; - } - - public Builder setCreationTime(long createdOn) { - this.creationTime = createdOn; - return this; - } - - public Builder setQuotaInBytes(long quotaInBytes) { - this.quotaInBytes = quotaInBytes; - return this; - } - - public Builder addMetadata(String key, String value) { - keyValueMap.put(key, value); // overwrite if present. - return this; - } - - public Builder addOzoneAcls(OzoneAclInfo acl) throws IOException { - aclMap.addAcl(acl); - return this; - } - - /** - * Constructs a CreateVolumeArgument. - * @return CreateVolumeArgs. - */ - public KsmVolumeArgs build() { - Preconditions.checkNotNull(adminName); - Preconditions.checkNotNull(ownerName); - Preconditions.checkNotNull(volume); - return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes, - keyValueMap, aclMap, creationTime); - } - } - - public VolumeInfo getProtobuf() { - List<KeyValue> metadataList = new LinkedList<>(); - for (Map.Entry<String, String> entry : keyValueMap.entrySet()) { - metadataList.add(KeyValue.newBuilder().setKey(entry.getKey()). - setValue(entry.getValue()).build()); - } - List<OzoneAclInfo> aclList = aclMap.ozoneAclGetProtobuf(); - - return VolumeInfo.newBuilder() - .setAdminName(adminName) - .setOwnerName(ownerName) - .setVolume(volume) - .setQuotaInBytes(quotaInBytes) - .addAllMetadata(metadataList) - .addAllVolumeAcls(aclList) - .setCreationTime(creationTime) - .build(); - } - - public static KsmVolumeArgs getFromProtobuf(VolumeInfo volInfo) { - Map<String, String> kvMap = volInfo.getMetadataList().stream() - .collect(Collectors.toMap(KeyValue::getKey, - KeyValue::getValue)); - KsmOzoneAclMap aclMap = - KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList()); - - return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(), - volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap, - volInfo.getCreationTime()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java deleted file mode 100644 index 472af65..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/VolumeArgs.java +++ /dev/null @@ -1,140 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; - -import com.google.common.base.Preconditions; - -import java.util.HashMap; -import java.util.Map; - -/** - * A class that encapsulates the createVolume Args. - */ -public final class VolumeArgs { - private final String adminName; - private final String ownerName; - private final String volume; - private final long quotaInBytes; - private final Map<String, String> extendedAttributes; - - /** - * Private constructor, constructed via builder. - * - * @param adminName - Administrator name. - * @param ownerName - Volume owner's name - * @param volume - volume name - * @param quotaInBytes - Volume Quota in bytes. - * @param keyValueMap - keyValue map. - */ - private VolumeArgs(String adminName, String ownerName, String volume, - long quotaInBytes, Map<String, String> keyValueMap) { - this.adminName = adminName; - this.ownerName = ownerName; - this.volume = volume; - this.quotaInBytes = quotaInBytes; - this.extendedAttributes = keyValueMap; - } - - /** - * Returns the Admin Name. - * - * @return String. - */ - public String getAdminName() { - return adminName; - } - - /** - * Returns the owner Name. - * - * @return String - */ - public String getOwnerName() { - return ownerName; - } - - /** - * Returns the volume Name. - * - * @return String - */ - public String getVolume() { - return volume; - } - - /** - * Returns Quota in Bytes. - * - * @return long, Quota in bytes. - */ - public long getQuotaInBytes() { - return quotaInBytes; - } - - public Map<String, String> getExtendedAttributes() { - return extendedAttributes; - } - - static class Builder { - private String adminName; - private String ownerName; - private String volume; - private long quotaInBytes; - private Map<String, String> extendedAttributes; - - /** - * Constructs a builder. - */ - Builder() { - extendedAttributes = new HashMap<>(); - } - - public void setAdminName(String adminName) { - this.adminName = adminName; - } - - public void setOwnerName(String ownerName) { - this.ownerName = ownerName; - } - - public void setVolume(String volume) { - this.volume = volume; - } - - public void setQuotaInBytes(long quotaInBytes) { - this.quotaInBytes = quotaInBytes; - } - - public void addMetadata(String key, String value) { - extendedAttributes.put(key, value); // overwrite if present. - } - - /** - * Constructs a CreateVolumeArgument. - * - * @return CreateVolumeArgs. - */ - public VolumeArgs build() { - Preconditions.checkNotNull(adminName); - Preconditions.checkNotNull(ownerName); - Preconditions.checkNotNull(volume); - return new VolumeArgs(adminName, ownerName, volume, quotaInBytes, - extendedAttributes); - } - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java deleted file mode 100644 index 8e48773..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/helpers/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.helpers; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java deleted file mode 100644 index 5b208fa..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm; -/** - This package contains client side protocol library to communicate with KSM. - */ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeySpaceManagerProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeySpaceManagerProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeySpaceManagerProtocol.java deleted file mode 100644 index 2ff8e36..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/KeySpaceManagerProtocol.java +++ /dev/null @@ -1,212 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ksm.protocol; - -import org.apache.hadoop.ksm.helpers.KsmBucketArgs; -import org.apache.hadoop.ksm.helpers.KsmBucketInfo; -import org.apache.hadoop.ksm.helpers.KsmKeyArgs; -import org.apache.hadoop.ksm.helpers.KsmKeyInfo; -import org.apache.hadoop.ksm.helpers.KsmVolumeArgs; -import org.apache.hadoop.ozone.protocol.proto - .KeySpaceManagerProtocolProtos.OzoneAclInfo; -import java.io.IOException; -import java.util.List; - -/** - * Protocol to talk to KSM. - */ -public interface KeySpaceManagerProtocol { - - /** - * Creates a volume. - * @param args - Arguments to create Volume. - * @throws IOException - */ - void createVolume(KsmVolumeArgs args) throws IOException; - - /** - * Changes the owner of a volume. - * @param volume - Name of the volume. - * @param owner - Name of the owner. - * @throws IOException - */ - void setOwner(String volume, String owner) throws IOException; - - /** - * Changes the Quota on a volume. - * @param volume - Name of the volume. - * @param quota - Quota in bytes. - * @throws IOException - */ - void setQuota(String volume, long quota) throws IOException; - - /** - * Checks if the specified user can access this volume. - * @param volume - volume - * @param userAcl - user acls which needs to be checked for access - * @return true if the user has required access for the volume, - * false otherwise - * @throws IOException - */ - boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl) - throws IOException; - - /** - * Gets the volume information. - * @param volume - Volume name. - * @return VolumeArgs or exception is thrown. - * @throws IOException - */ - KsmVolumeArgs getVolumeInfo(String volume) throws IOException; - - /** - * Deletes an existing empty volume. - * @param volume - Name of the volume. - * @throws IOException - */ - void deleteVolume(String volume) throws IOException; - - /** - * Lists volume owned by a specific user. - * @param userName - user name - * @param prefix - Filter prefix -- Return only entries that match this. - * @param prevKey - Previous key -- List starts from the next from the prevkey - * @param maxKeys - Max number of keys to return. - * @return List of Volumes. - * @throws IOException - */ - List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix, String - prevKey, int maxKeys) throws IOException; - - /** - * Lists volume all volumes in the cluster. - * @param prefix - Filter prefix -- Return only entries that match this. - * @param prevKey - Previous key -- List starts from the next from the prevkey - * @param maxKeys - Max number of keys to return. - * @return List of Volumes. - * @throws IOException - */ - List<KsmVolumeArgs> listAllVolumes(String prefix, String - prevKey, int maxKeys) throws IOException; - - /** - * Creates a bucket. - * @param bucketInfo - BucketInfo to create Bucket. - * @throws IOException - */ - void createBucket(KsmBucketInfo bucketInfo) throws IOException; - - /** - * Gets the bucket information. - * @param volumeName - Volume name. - * @param bucketName - Bucket name. - * @return KsmBucketInfo or exception is thrown. - * @throws IOException - */ - KsmBucketInfo getBucketInfo(String volumeName, String bucketName) - throws IOException; - - /** - * Sets bucket property from args. - * @param args - BucketArgs. - * @throws IOException - */ - void setBucketProperty(KsmBucketArgs args) throws IOException; - - /** - * Allocate a block to a container, the block is returned to the client. - * - * @param args the args of the key. - * @return KsmKeyInfo isntacne that client uses to talk to container. - * @throws IOException - */ - KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException; - - /** - * Look up for the container of an existing key. - * - * @param args the args of the key. - * @return KsmKeyInfo isntacne that client uses to talk to container. - * @throws IOException - */ - KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException; - - /** - * Deletes an existing key. - * - * @param args the args of the key. - * @throws IOException - */ - void deleteKey(KsmKeyArgs args) throws IOException; - - /** - * Deletes an existing empty bucket from volume. - * @param volume - Name of the volume. - * @param bucket - Name of the bucket. - * @throws IOException - */ - void deleteBucket(String volume, String bucket) throws IOException; - - /** - * Returns a list of buckets represented by {@link KsmBucketInfo} - * in the given volume. Argument volumeName is required, others - * are optional. - * - * @param volumeName - * the name of the volume. - * @param startBucketName - * the start bucket name, only the buckets whose name is - * after this value will be included in the result. - * @param bucketPrefix - * bucket name prefix, only the buckets whose name has - * this prefix will be included in the result. - * @param maxNumOfBuckets - * the maximum number of buckets to return. It ensures - * the size of the result will not exceed this limit. - * @return a list of buckets. - * @throws IOException - */ - List<KsmBucketInfo> listBuckets(String volumeName, - String startBucketName, String bucketPrefix, int maxNumOfBuckets) - throws IOException; - - /** - * Returns a list of keys represented by {@link KsmKeyInfo} - * in the given bucket. Argument volumeName, bucketName is required, - * others are optional. - * - * @param volumeName - * the name of the volume. - * @param bucketName - * the name of the bucket. - * @param startKeyName - * the start key name, only the keys whose name is - * after this value will be included in the result. - * @param keyPrefix - * key name prefix, only the keys whose name has - * this prefix will be included in the result. - * @param maxKeys - * the maximum number of keys to return. It ensures - * the size of the result will not exceed this limit. - * @return a list of keys. - * @throws IOException - */ - List<KsmKeyInfo> listKeys(String volumeName, - String bucketName, String startKeyName, String keyPrefix, int maxKeys) - throws IOException; -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java deleted file mode 100644 index d1d12dc..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ksm/protocol/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ksm.protocol; \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
