http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java new file mode 100644 index 0000000..1211b50 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java @@ -0,0 +1,233 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +import java.util.List; +import java.util.stream.Collectors; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.BucketArgs; +import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; + +/** + * A class that encapsulates Bucket Arguments. + */ +public final class KsmBucketArgs { + /** + * Name of the volume in which the bucket belongs to. + */ + private final String volumeName; + /** + * Name of the bucket. + */ + private final String bucketName; + /** + * ACL's that are to be added for the bucket. + */ + private List<OzoneAcl> addAcls; + /** + * ACL's that are to be removed from the bucket. + */ + private List<OzoneAcl> removeAcls; + /** + * Bucket Version flag. + */ + private Boolean isVersionEnabled; + /** + * Type of storage to be used for this bucket. + * [RAM_DISK, SSD, DISK, ARCHIVE] + */ + private StorageType storageType; + + /** + * Private constructor, constructed via builder. + * @param volumeName - Volume name. + * @param bucketName - Bucket name. + * @param addAcls - ACL's to be added. + * @param removeAcls - ACL's to be removed. + * @param isVersionEnabled - Bucket version flag. + * @param storageType - Storage type to be used. + */ + private KsmBucketArgs(String volumeName, String bucketName, + List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls, + Boolean isVersionEnabled, StorageType storageType) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.addAcls = addAcls; + this.removeAcls = removeAcls; + this.isVersionEnabled = isVersionEnabled; + this.storageType = storageType; + } + + /** + * Returns the Volume Name. + * @return String. + */ + public String getVolumeName() { + return volumeName; + } + + /** + * Returns the Bucket Name. + * @return String + */ + public String getBucketName() { + return bucketName; + } + + /** + * Returns the ACL's that are to be added. + * @return List<OzoneAclInfo> + */ + public List<OzoneAcl> getAddAcls() { + return addAcls; + } + + /** + * Returns the ACL's that are to be removed. + * @return List<OzoneAclInfo> + */ + public List<OzoneAcl> getRemoveAcls() { + return removeAcls; + } + + /** + * Returns true if bucket version is enabled, else false. + * @return isVersionEnabled + */ + public Boolean getIsVersionEnabled() { + return isVersionEnabled; + } + + /** + * Returns the type of storage to be used. + * @return StorageType + */ + public StorageType getStorageType() { + return storageType; + } + + /** + * Returns new builder class that builds a KsmBucketArgs. + * + * @return Builder + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder for KsmBucketArgs. + */ + public static class Builder { + private String volumeName; + private String bucketName; + private List<OzoneAcl> addAcls; + private List<OzoneAcl> removeAcls; + private Boolean isVersionEnabled; + private StorageType storageType; + + public Builder setVolumeName(String volume) { + this.volumeName = volume; + return this; + } + + public Builder setBucketName(String bucket) { + this.bucketName = bucket; + return this; + } + + public Builder setAddAcls(List<OzoneAcl> acls) { + this.addAcls = acls; + return this; + } + + public Builder setRemoveAcls(List<OzoneAcl> acls) { + this.removeAcls = acls; + return this; + } + + public Builder setIsVersionEnabled(Boolean versionFlag) { + this.isVersionEnabled = versionFlag; + return this; + } + + public Builder setStorageType(StorageType storage) { + this.storageType = storage; + return this; + } + + /** + * Constructs the KsmBucketArgs. + * @return instance of KsmBucketArgs. + */ + public KsmBucketArgs build() { + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + return new KsmBucketArgs(volumeName, bucketName, addAcls, + removeAcls, isVersionEnabled, storageType); + } + } + + /** + * Creates BucketArgs protobuf from KsmBucketArgs. + */ + public BucketArgs getProtobuf() { + BucketArgs.Builder builder = BucketArgs.newBuilder(); + builder.setVolumeName(volumeName) + .setBucketName(bucketName); + if(addAcls != null && !addAcls.isEmpty()) { + builder.addAllAddAcls(addAcls.stream().map( + KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); + } + if(removeAcls != null && !removeAcls.isEmpty()) { + builder.addAllRemoveAcls(removeAcls.stream().map( + KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())); + } + if(isVersionEnabled != null) { + builder.setIsVersionEnabled(isVersionEnabled); + } + if(storageType != null) { + builder.setStorageType( + PBHelperClient.convertStorageType(storageType)); + } + return builder.build(); + } + + /** + * Parses BucketInfo protobuf and creates KsmBucketArgs. + * @param bucketArgs + * @return instance of KsmBucketArgs + */ + public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) { + return new KsmBucketArgs(bucketArgs.getVolumeName(), + bucketArgs.getBucketName(), + bucketArgs.getAddAclsList().stream().map( + KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), + bucketArgs.getRemoveAclsList().stream().map( + KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), + bucketArgs.hasIsVersionEnabled() ? + bucketArgs.getIsVersionEnabled() : null, + bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType( + bucketArgs.getStorageType()) : null); + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java new file mode 100644 index 0000000..4de07aa --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.StorageType; +import org.apache.hadoop.hdfs.protocolPB.PBHelperClient; +import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.BucketInfo; +import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; + +import java.util.LinkedList; +import java.util.List; +import java.util.stream.Collectors; + +/** + * A class that encapsulates Bucket Info. + */ +public final class KsmBucketInfo { + /** + * Name of the volume in which the bucket belongs to. + */ + private final String volumeName; + /** + * Name of the bucket. + */ + private final String bucketName; + /** + * ACL Information. + */ + private List<OzoneAcl> acls; + /** + * Bucket Version flag. + */ + private Boolean isVersionEnabled; + /** + * Type of storage to be used for this bucket. + * [RAM_DISK, SSD, DISK, ARCHIVE] + */ + private StorageType storageType; + + /** + * Private constructor, constructed via builder. + * @param volumeName - Volume name. + * @param bucketName - Bucket name. + * @param acls - list of ACLs. + * @param isVersionEnabled - Bucket version flag. + * @param storageType - Storage type to be used. + */ + private KsmBucketInfo(String volumeName, String bucketName, + List<OzoneAcl> acls, boolean isVersionEnabled, + StorageType storageType) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.acls = acls; + this.isVersionEnabled = isVersionEnabled; + this.storageType = storageType; + } + + /** + * Returns the Volume Name. + * @return String. + */ + public String getVolumeName() { + return volumeName; + } + + /** + * Returns the Bucket Name. + * @return String + */ + public String getBucketName() { + return bucketName; + } + + /** + * Returns the ACL's associated with this bucket. + * @return List<OzoneAcl> + */ + public List<OzoneAcl> getAcls() { + return acls; + } + + /** + * Returns true if bucket version is enabled, else false. + * @return isVersionEnabled + */ + public boolean getIsVersionEnabled() { + return isVersionEnabled; + } + + /** + * Returns the type of storage to be used. + * @return StorageType + */ + public StorageType getStorageType() { + return storageType; + } + + /** + * Returns new builder class that builds a KsmBucketInfo. + * + * @return Builder + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder for KsmBucketInfo. + */ + public static class Builder { + private String volumeName; + private String bucketName; + private List<OzoneAcl> acls; + private Boolean isVersionEnabled; + private StorageType storageType; + + Builder() { + //Default values + this.acls = new LinkedList<>(); + this.isVersionEnabled = false; + this.storageType = StorageType.DISK; + } + + public Builder setVolumeName(String volume) { + this.volumeName = volume; + return this; + } + + public Builder setBucketName(String bucket) { + this.bucketName = bucket; + return this; + } + + public Builder setAcls(List<OzoneAcl> listOfAcls) { + this.acls = listOfAcls; + return this; + } + + public Builder setIsVersionEnabled(Boolean versionFlag) { + this.isVersionEnabled = versionFlag; + return this; + } + + public Builder setStorageType(StorageType storage) { + this.storageType = storage; + return this; + } + + /** + * Constructs the KsmBucketInfo. + * @return instance of KsmBucketInfo. + */ + public KsmBucketInfo build() { + Preconditions.checkNotNull(volumeName); + Preconditions.checkNotNull(bucketName); + Preconditions.checkNotNull(acls); + Preconditions.checkNotNull(isVersionEnabled); + Preconditions.checkNotNull(storageType); + return new KsmBucketInfo(volumeName, bucketName, acls, + isVersionEnabled, storageType); + } + } + + /** + * Creates BucketInfo protobuf from KsmBucketInfo. + */ + public BucketInfo getProtobuf() { + return BucketInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .addAllAcls(acls.stream().map( + KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())) + .setIsVersionEnabled(isVersionEnabled) + .setStorageType(PBHelperClient.convertStorageType( + storageType)) + .build(); + } + + /** + * Parses BucketInfo protobuf and creates KsmBucketInfo. + * @param bucketInfo + * @return instance of KsmBucketInfo + */ + public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) { + return new KsmBucketInfo( + bucketInfo.getVolumeName(), + bucketInfo.getBucketName(), + bucketInfo.getAclsList().stream().map( + KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()), + bucketInfo.getIsVersionEnabled(), + PBHelperClient.convertStorageType( + bucketInfo.getStorageType())); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java new file mode 100644 index 0000000..7375ec5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java @@ -0,0 +1,88 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +/** + * Args for key. Client use this to specify key's attributes on key creation + * (putKey()). + */ +public final class KsmKeyArgs { + private final String volumeName; + private final String bucketName; + private final String keyName; + + private final long dataSize; + + private KsmKeyArgs(String volumeName, String bucketName, String keyName, + long dataSize) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.keyName = keyName; + this.dataSize = dataSize; + } + + public String getVolumeName() { + return volumeName; + } + + public String getBucketName() { + return bucketName; + } + + public String getKeyName() { + return keyName; + } + + public long getDataSize() { + return dataSize; + } + + /** + * Builder class of KsmKeyArgs. + */ + public static class Builder { + private String volumeName; + private String bucketName; + private String keyName; + private long dataSize; + + public Builder setVolumeName(String volume) { + this.volumeName = volume; + return this; + } + + public Builder setBucketName(String bucket) { + this.bucketName = bucket; + return this; + } + + public Builder setKeyName(String key) { + this.keyName = key; + return this; + } + + public Builder setDataSize(long size) { + this.dataSize = size; + return this; + } + + public KsmKeyArgs build() { + return new KsmKeyArgs(volumeName, bucketName, keyName, dataSize); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java new file mode 100644 index 0000000..6dea835 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java @@ -0,0 +1,161 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * Args for key block. The block instance for the key requested in putKey. + * This is returned from KSM to client, and client use class to talk to + * datanode. Also, this is the metadata written to ksm.db on server side. + */ +public final class KsmKeyInfo { + private final String volumeName; + private final String bucketName; + // name of key client specified + private final String keyName; + private final long dataSize; + private List<KsmKeyLocationInfo> keyLocationList; + private final long creationTime; + private final long modificationTime; + + private KsmKeyInfo(String volumeName, String bucketName, String keyName, + List<KsmKeyLocationInfo> locationInfos, long dataSize, long creationTime, + long modificationTime) { + this.volumeName = volumeName; + this.bucketName = bucketName; + this.keyName = keyName; + this.dataSize = dataSize; + this.keyLocationList = locationInfos; + this.creationTime = creationTime; + this.modificationTime = modificationTime; + } + + public String getVolumeName() { + return volumeName; + } + + public String getBucketName() { + return bucketName; + } + + public String getKeyName() { + return keyName; + } + + public long getDataSize() { + return dataSize; + } + + public List<KsmKeyLocationInfo> getKeyLocationList() { + return keyLocationList; + } + + public long getCreationTime() { + return creationTime; + } + + public long getModificationTime() { + return modificationTime; + } + + /** + * Builder of KsmKeyInfo. + */ + public static class Builder { + private String volumeName; + private String bucketName; + private String keyName; + private long dataSize; + private List<KsmKeyLocationInfo> ksmKeyLocationInfos; + private long creationTime; + private long modificationTime; + + public Builder setVolumeName(String volume) { + this.volumeName = volume; + return this; + } + + public Builder setBucketName(String bucket) { + this.bucketName = bucket; + return this; + } + + public Builder setKeyName(String key) { + this.keyName = key; + return this; + } + + public Builder setKsmKeyLocationInfos( + List<KsmKeyLocationInfo> ksmKeyLocationInfoList) { + this.ksmKeyLocationInfos = ksmKeyLocationInfoList; + return this; + } + + public Builder setDataSize(long size) { + this.dataSize = size; + return this; + } + + public Builder setCreationTime(long creationTime) { + this.creationTime = creationTime; + return this; + } + + public Builder setModificationTime(long modificationTime) { + this.modificationTime = modificationTime; + return this; + } + + public KsmKeyInfo build() { + return new KsmKeyInfo( + volumeName, bucketName, keyName, ksmKeyLocationInfos, + dataSize, creationTime, modificationTime); + } + } + + public KeyInfo getProtobuf() { + return KeyInfo.newBuilder() + .setVolumeName(volumeName) + .setBucketName(bucketName) + .setKeyName(keyName) + .setDataSize(dataSize) + .addAllKeyLocationList(keyLocationList.stream() + .map(KsmKeyLocationInfo::getProtobuf).collect(Collectors.toList())) + .setCreationTime(creationTime) + .setModificationTime(modificationTime) + .build(); + } + + public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) { + return new KsmKeyInfo( + keyInfo.getVolumeName(), + keyInfo.getBucketName(), + keyInfo.getKeyName(), + keyInfo.getKeyLocationListList().stream() + .map(KsmKeyLocationInfo::getFromProtobuf) + .collect(Collectors.toList()), + keyInfo.getDataSize(), + keyInfo.getCreationTime(), + keyInfo.getModificationTime()); + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java new file mode 100644 index 0000000..039f8e0 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocation; + +/** + * One key can be too huge to fit in one container. In which case it gets split + * into a number of subkeys. This class represents one such subkey instance. + */ +public final class KsmKeyLocationInfo { + private final String containerName; + // name of the block id SCM assigned for the key + private final String blockID; + private final boolean shouldCreateContainer; + // the id of this subkey in all the subkeys. + private final int index; + private final long length; + private final long offset; + + private KsmKeyLocationInfo(String containerName, + String blockID, boolean shouldCreateContainer, int index, + long length, long offset) { + this.containerName = containerName; + this.blockID = blockID; + this.shouldCreateContainer = shouldCreateContainer; + this.index = index; + this.length = length; + this.offset = offset; + } + + public String getContainerName() { + return containerName; + } + + public String getBlockID() { + return blockID; + } + + public boolean getShouldCreateContainer() { + return shouldCreateContainer; + } + + public int getIndex() { + return index; + } + + public long getLength() { + return length; + } + + public long getOffset() { + return offset; + } + + /** + * Builder of KsmKeyLocationInfo. + */ + public static class Builder { + private String containerName; + private String blockID; + private boolean shouldCreateContainer; + // the id of this subkey in all the subkeys. + private int index; + private long length; + private long offset; + public Builder setContainerName(String container) { + this.containerName = container; + return this; + } + + public Builder setBlockID(String block) { + this.blockID = block; + return this; + } + + public Builder setShouldCreateContainer(boolean create) { + this.shouldCreateContainer = create; + return this; + } + + public Builder setIndex(int id) { + this.index = id; + return this; + } + + public Builder setLength(long len) { + this.length = len; + return this; + } + + public Builder setOffset(long off) { + this.offset = off; + return this; + } + + public KsmKeyLocationInfo build() { + return new KsmKeyLocationInfo(containerName, blockID, + shouldCreateContainer, index, length, offset); + } + } + + public KeyLocation getProtobuf() { + return KeyLocation.newBuilder() + .setContainerName(containerName) + .setBlockID(blockID) + .setShouldCreateContainer(shouldCreateContainer) + .setIndex(index) + .setLength(length) + .setOffset(offset) + .build(); + } + + public static KsmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) { + return new KsmKeyLocationInfo( + keyLocation.getContainerName(), + keyLocation.getBlockID(), + keyLocation.getShouldCreateContainer(), + keyLocation.getIndex(), + keyLocation.getLength(), + keyLocation.getOffset()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java new file mode 100644 index 0000000..7d9efad --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.ksm.helpers; + +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.OzoneAclInfo; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType; + +import java.util.List; +import java.util.LinkedList; +import java.util.Map; +import java.util.ArrayList; +import java.util.HashMap; + +/** + * This helper class keeps a map of all user and their permissions. + */ +public class KsmOzoneAclMap { + // per Acl Type user:rights map + private ArrayList<Map<String, OzoneAclRights>> aclMaps; + + KsmOzoneAclMap() { + aclMaps = new ArrayList<>(); + for (OzoneAclType aclType : OzoneAclType.values()) { + aclMaps.add(aclType.ordinal(), new HashMap<>()); + } + } + + private Map<String, OzoneAclRights> getMap(OzoneAclType type) { + return aclMaps.get(type.ordinal()); + } + + // For a given acl type and user, get the stored acl + private OzoneAclRights getAcl(OzoneAclType type, String user) { + return getMap(type).get(user); + } + + // Add a new acl to the map + public void addAcl(OzoneAclInfo acl) { + getMap(acl.getType()).put(acl.getName(), acl.getRights()); + } + + // for a given acl, check if the user has access rights + public boolean hasAccess(OzoneAclInfo acl) { + OzoneAclRights storedRights = getAcl(acl.getType(), acl.getName()); + if (storedRights != null) { + switch (acl.getRights()) { + case READ: + return (storedRights == OzoneAclRights.READ) + || (storedRights == OzoneAclRights.READ_WRITE); + case WRITE: + return (storedRights == OzoneAclRights.WRITE) + || (storedRights == OzoneAclRights.READ_WRITE); + case READ_WRITE: + return (storedRights == OzoneAclRights.READ_WRITE); + default: + return false; + } + } else { + return false; + } + } + + // Convert this map to OzoneAclInfo Protobuf List + public List<OzoneAclInfo> ozoneAclGetProtobuf() { + List<OzoneAclInfo> aclList = new LinkedList<>(); + for (OzoneAclType type: OzoneAclType.values()) { + for (Map.Entry<String, OzoneAclRights> entry : + aclMaps.get(type.ordinal()).entrySet()) { + OzoneAclInfo aclInfo = OzoneAclInfo.newBuilder() + .setName(entry.getKey()) + .setType(type) + .setRights(entry.getValue()) + .build(); + aclList.add(aclInfo); + } + } + + return aclList; + } + + // Create map from list of OzoneAclInfos + public static KsmOzoneAclMap ozoneAclGetFromProtobuf( + List<OzoneAclInfo> aclList) { + KsmOzoneAclMap aclMap = new KsmOzoneAclMap(); + for (OzoneAclInfo acl : aclList) { + aclMap.addAcl(acl); + } + return aclMap; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java new file mode 100644 index 0000000..4d60827 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java @@ -0,0 +1,223 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +import com.google.common.base.Preconditions; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.OzoneAclInfo; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.VolumeInfo; +import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; + +import java.io.IOException; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + + +/** + * A class that encapsulates the KsmVolumeArgs Args. + */ +public final class KsmVolumeArgs { + private final String adminName; + private final String ownerName; + private final String volume; + private final long creationTime; + private final long quotaInBytes; + private final Map<String, String> keyValueMap; + private final KsmOzoneAclMap aclMap; + + /** + * Private constructor, constructed via builder. + * @param adminName - Administrator's name. + * @param ownerName - Volume owner's name + * @param volume - volume name + * @param quotaInBytes - Volume Quota in bytes. + * @param keyValueMap - keyValue map. + * @param aclMap - User to access rights map. + * @param creationTime - Volume creation time. + */ + private KsmVolumeArgs(String adminName, String ownerName, String volume, + long quotaInBytes, Map<String, String> keyValueMap, + KsmOzoneAclMap aclMap, long creationTime) { + this.adminName = adminName; + this.ownerName = ownerName; + this.volume = volume; + this.quotaInBytes = quotaInBytes; + this.keyValueMap = keyValueMap; + this.aclMap = aclMap; + this.creationTime = creationTime; + } + + /** + * Returns the Admin Name. + * @return String. + */ + public String getAdminName() { + return adminName; + } + + /** + * Returns the owner Name. + * @return String + */ + public String getOwnerName() { + return ownerName; + } + + /** + * Returns the volume Name. + * @return String + */ + public String getVolume() { + return volume; + } + + /** + * Returns creation time. + * @return long + */ + public long getCreationTime() { + return creationTime; + } + + /** + * Returns Quota in Bytes. + * @return long, Quota in bytes. + */ + public long getQuotaInBytes() { + return quotaInBytes; + } + + public Map<String, String> getKeyValueMap() { + return keyValueMap; + } + + public KsmOzoneAclMap getAclMap() { + return aclMap; + } + /** + * Returns new builder class that builds a KsmVolumeArgs. + * + * @return Builder + */ + public static Builder newBuilder() { + return new Builder(); + } + + /** + * Builder for KsmVolumeArgs. + */ + public static class Builder { + private String adminName; + private String ownerName; + private String volume; + private long creationTime; + private long quotaInBytes; + private Map<String, String> keyValueMap; + private KsmOzoneAclMap aclMap; + + /** + * Constructs a builder. + */ + Builder() { + keyValueMap = new HashMap<>(); + aclMap = new KsmOzoneAclMap(); + } + + public Builder setAdminName(String adminName) { + this.adminName = adminName; + return this; + } + + public Builder setOwnerName(String ownerName) { + this.ownerName = ownerName; + return this; + } + + public Builder setVolume(String volume) { + this.volume = volume; + return this; + } + + public Builder setCreationTime(long createdOn) { + this.creationTime = createdOn; + return this; + } + + public Builder setQuotaInBytes(long quotaInBytes) { + this.quotaInBytes = quotaInBytes; + return this; + } + + public Builder addMetadata(String key, String value) { + keyValueMap.put(key, value); // overwrite if present. + return this; + } + + public Builder addOzoneAcls(OzoneAclInfo acl) throws IOException { + aclMap.addAcl(acl); + return this; + } + + /** + * Constructs a CreateVolumeArgument. + * @return CreateVolumeArgs. + */ + public KsmVolumeArgs build() { + Preconditions.checkNotNull(adminName); + Preconditions.checkNotNull(ownerName); + Preconditions.checkNotNull(volume); + return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes, + keyValueMap, aclMap, creationTime); + } + } + + public VolumeInfo getProtobuf() { + List<KeyValue> metadataList = new LinkedList<>(); + for (Map.Entry<String, String> entry : keyValueMap.entrySet()) { + metadataList.add(KeyValue.newBuilder().setKey(entry.getKey()). + setValue(entry.getValue()).build()); + } + List<OzoneAclInfo> aclList = aclMap.ozoneAclGetProtobuf(); + + return VolumeInfo.newBuilder() + .setAdminName(adminName) + .setOwnerName(ownerName) + .setVolume(volume) + .setQuotaInBytes(quotaInBytes) + .addAllMetadata(metadataList) + .addAllVolumeAcls(aclList) + .setCreationTime(creationTime) + .build(); + } + + public static KsmVolumeArgs getFromProtobuf(VolumeInfo volInfo) { + Map<String, String> kvMap = volInfo.getMetadataList().stream() + .collect(Collectors.toMap(KeyValue::getKey, + KeyValue::getValue)); + KsmOzoneAclMap aclMap = + KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList()); + + return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(), + volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap, + volInfo.getCreationTime()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java new file mode 100644 index 0000000..1a3d486 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java @@ -0,0 +1,140 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; + +import com.google.common.base.Preconditions; + +import java.util.HashMap; +import java.util.Map; + +/** + * A class that encapsulates the createVolume Args. + */ +public final class VolumeArgs { + private final String adminName; + private final String ownerName; + private final String volume; + private final long quotaInBytes; + private final Map<String, String> extendedAttributes; + + /** + * Private constructor, constructed via builder. + * + * @param adminName - Administrator name. + * @param ownerName - Volume owner's name + * @param volume - volume name + * @param quotaInBytes - Volume Quota in bytes. + * @param keyValueMap - keyValue map. + */ + private VolumeArgs(String adminName, String ownerName, String volume, + long quotaInBytes, Map<String, String> keyValueMap) { + this.adminName = adminName; + this.ownerName = ownerName; + this.volume = volume; + this.quotaInBytes = quotaInBytes; + this.extendedAttributes = keyValueMap; + } + + /** + * Returns the Admin Name. + * + * @return String. + */ + public String getAdminName() { + return adminName; + } + + /** + * Returns the owner Name. + * + * @return String + */ + public String getOwnerName() { + return ownerName; + } + + /** + * Returns the volume Name. + * + * @return String + */ + public String getVolume() { + return volume; + } + + /** + * Returns Quota in Bytes. + * + * @return long, Quota in bytes. + */ + public long getQuotaInBytes() { + return quotaInBytes; + } + + public Map<String, String> getExtendedAttributes() { + return extendedAttributes; + } + + static class Builder { + private String adminName; + private String ownerName; + private String volume; + private long quotaInBytes; + private Map<String, String> extendedAttributes; + + /** + * Constructs a builder. + */ + Builder() { + extendedAttributes = new HashMap<>(); + } + + public void setAdminName(String adminName) { + this.adminName = adminName; + } + + public void setOwnerName(String ownerName) { + this.ownerName = ownerName; + } + + public void setVolume(String volume) { + this.volume = volume; + } + + public void setQuotaInBytes(long quotaInBytes) { + this.quotaInBytes = quotaInBytes; + } + + public void addMetadata(String key, String value) { + extendedAttributes.put(key, value); // overwrite if present. + } + + /** + * Constructs a CreateVolumeArgument. + * + * @return CreateVolumeArgs. + */ + public VolumeArgs build() { + Preconditions.checkNotNull(adminName); + Preconditions.checkNotNull(ownerName); + Preconditions.checkNotNull(volume); + return new VolumeArgs(adminName, ownerName, volume, quotaInBytes, + extendedAttributes); + } + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java new file mode 100644 index 0000000..ce627a5 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java @@ -0,0 +1,18 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.helpers; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java new file mode 100644 index 0000000..7698ee1 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java @@ -0,0 +1,21 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm; +/** + This package contains client side protocol library to communicate with KSM. + */ \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java new file mode 100644 index 0000000..7d84114 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java @@ -0,0 +1,212 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.protocol; + +import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.OzoneAclInfo; +import java.io.IOException; +import java.util.List; + +/** + * Protocol to talk to KSM. + */ +public interface KeySpaceManagerProtocol { + + /** + * Creates a volume. + * @param args - Arguments to create Volume. + * @throws IOException + */ + void createVolume(KsmVolumeArgs args) throws IOException; + + /** + * Changes the owner of a volume. + * @param volume - Name of the volume. + * @param owner - Name of the owner. + * @throws IOException + */ + void setOwner(String volume, String owner) throws IOException; + + /** + * Changes the Quota on a volume. + * @param volume - Name of the volume. + * @param quota - Quota in bytes. + * @throws IOException + */ + void setQuota(String volume, long quota) throws IOException; + + /** + * Checks if the specified user can access this volume. + * @param volume - volume + * @param userAcl - user acls which needs to be checked for access + * @return true if the user has required access for the volume, + * false otherwise + * @throws IOException + */ + boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl) + throws IOException; + + /** + * Gets the volume information. + * @param volume - Volume name. + * @return VolumeArgs or exception is thrown. + * @throws IOException + */ + KsmVolumeArgs getVolumeInfo(String volume) throws IOException; + + /** + * Deletes an existing empty volume. + * @param volume - Name of the volume. + * @throws IOException + */ + void deleteVolume(String volume) throws IOException; + + /** + * Lists volume owned by a specific user. + * @param userName - user name + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix, String + prevKey, int maxKeys) throws IOException; + + /** + * Lists volume all volumes in the cluster. + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + List<KsmVolumeArgs> listAllVolumes(String prefix, String + prevKey, int maxKeys) throws IOException; + + /** + * Creates a bucket. + * @param bucketInfo - BucketInfo to create Bucket. + * @throws IOException + */ + void createBucket(KsmBucketInfo bucketInfo) throws IOException; + + /** + * Gets the bucket information. + * @param volumeName - Volume name. + * @param bucketName - Bucket name. + * @return KsmBucketInfo or exception is thrown. + * @throws IOException + */ + KsmBucketInfo getBucketInfo(String volumeName, String bucketName) + throws IOException; + + /** + * Sets bucket property from args. + * @param args - BucketArgs. + * @throws IOException + */ + void setBucketProperty(KsmBucketArgs args) throws IOException; + + /** + * Allocate a block to a container, the block is returned to the client. + * + * @param args the args of the key. + * @return KsmKeyInfo isntacne that client uses to talk to container. + * @throws IOException + */ + KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException; + + /** + * Look up for the container of an existing key. + * + * @param args the args of the key. + * @return KsmKeyInfo isntacne that client uses to talk to container. + * @throws IOException + */ + KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException; + + /** + * Deletes an existing key. + * + * @param args the args of the key. + * @throws IOException + */ + void deleteKey(KsmKeyArgs args) throws IOException; + + /** + * Deletes an existing empty bucket from volume. + * @param volume - Name of the volume. + * @param bucket - Name of the bucket. + * @throws IOException + */ + void deleteBucket(String volume, String bucket) throws IOException; + + /** + * Returns a list of buckets represented by {@link KsmBucketInfo} + * in the given volume. Argument volumeName is required, others + * are optional. + * + * @param volumeName + * the name of the volume. + * @param startBucketName + * the start bucket name, only the buckets whose name is + * after this value will be included in the result. + * @param bucketPrefix + * bucket name prefix, only the buckets whose name has + * this prefix will be included in the result. + * @param maxNumOfBuckets + * the maximum number of buckets to return. It ensures + * the size of the result will not exceed this limit. + * @return a list of buckets. + * @throws IOException + */ + List<KsmBucketInfo> listBuckets(String volumeName, + String startBucketName, String bucketPrefix, int maxNumOfBuckets) + throws IOException; + + /** + * Returns a list of keys represented by {@link KsmKeyInfo} + * in the given bucket. Argument volumeName, bucketName is required, + * others are optional. + * + * @param volumeName + * the name of the volume. + * @param bucketName + * the name of the bucket. + * @param startKeyName + * the start key name, only the keys whose name is + * after this value will be included in the result. + * @param keyPrefix + * key name prefix, only the keys whose name has + * this prefix will be included in the result. + * @param maxKeys + * the maximum number of keys to return. It ensures + * the size of the result will not exceed this limit. + * @return a list of keys. + * @throws IOException + */ + List<KsmKeyInfo> listKeys(String volumeName, + String bucketName, String startKeyName, String keyPrefix, int maxKeys) + throws IOException; +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java new file mode 100644 index 0000000..f77e5fd --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.ksm.protocol; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java new file mode 100644 index 0000000..a13c415 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java @@ -0,0 +1,651 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.protocolPB; + +import com.google.common.base.Strings; +import com.google.common.collect.Lists; +import com.google.protobuf.RpcController; +import com.google.protobuf.ServiceException; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.ipc.ProtobufHelper; +import org.apache.hadoop.ipc.ProtocolTranslator; +import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs; +import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.BucketArgs; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.BucketInfo; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.CreateBucketRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.CreateBucketResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.InfoBucketRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.InfoBucketResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.SetBucketPropertyRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.SetBucketPropertyResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.DeleteBucketRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.DeleteBucketResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.CreateVolumeRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.CreateVolumeResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.LocateKeyRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.LocateKeyResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.KeyArgs; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.SetVolumePropertyResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.DeleteVolumeRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.DeleteVolumeResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.InfoVolumeRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.InfoVolumeResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.CheckVolumeAccessRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.ListBucketsRequest; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.ListBucketsResponse; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysRequest; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.ListKeysResponse; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.VolumeInfo; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.Status; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.OzoneAclInfo; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos + .ListVolumeRequest; +import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos + .ListVolumeResponse; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; +import java.util.ArrayList; +import java.util.stream.Collectors; + +/** + * The client side implementation of KeySpaceManagerProtocol. + */ + +@InterfaceAudience.Private +public final class KeySpaceManagerProtocolClientSideTranslatorPB + implements KeySpaceManagerProtocol, ProtocolTranslator, Closeable { + + /** + * RpcController is not used and hence is set to null. + */ + private static final RpcController NULL_RPC_CONTROLLER = null; + + private final KeySpaceManagerProtocolPB rpcProxy; + + /** + * Constructor for KeySpaceManger Client. + * @param rpcProxy + */ + public KeySpaceManagerProtocolClientSideTranslatorPB( + KeySpaceManagerProtocolPB rpcProxy) { + this.rpcProxy = rpcProxy; + } + + /** + * Closes this stream and releases any system resources associated + * with it. If the stream is already closed then invoking this + * method has no effect. + * <p> + * <p> As noted in {@link AutoCloseable#close()}, cases where the + * close may fail require careful attention. It is strongly advised + * to relinquish the underlying resources and to internally + * <em>mark</em> the {@code Closeable} as closed, prior to throwing + * the {@code IOException}. + * + * @throws IOException if an I/O error occurs + */ + @Override + public void close() throws IOException { + + } + + /** + * Creates a volume. + * + * @param args - Arguments to create Volume. + * @throws IOException + */ + @Override + public void createVolume(KsmVolumeArgs args) throws IOException { + CreateVolumeRequest.Builder req = + CreateVolumeRequest.newBuilder(); + VolumeInfo volumeInfo = args.getProtobuf(); + req.setVolumeInfo(volumeInfo); + + final CreateVolumeResponse resp; + try { + resp = rpcProxy.createVolume(NULL_RPC_CONTROLLER, + req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if (resp.getStatus() != Status.OK) { + throw new + IOException("Volume creation failed, error:" + resp.getStatus()); + } + } + + /** + * Changes the owner of a volume. + * + * @param volume - Name of the volume. + * @param owner - Name of the owner. + * @throws IOException + */ + @Override + public void setOwner(String volume, String owner) throws IOException { + SetVolumePropertyRequest.Builder req = + SetVolumePropertyRequest.newBuilder(); + req.setVolumeName(volume).setOwnerName(owner); + final SetVolumePropertyResponse resp; + try { + resp = rpcProxy.setVolumeProperty(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new + IOException("Volume owner change failed, error:" + resp.getStatus()); + } + } + + /** + * Changes the Quota on a volume. + * + * @param volume - Name of the volume. + * @param quota - Quota in bytes. + * @throws IOException + */ + @Override + public void setQuota(String volume, long quota) throws IOException { + SetVolumePropertyRequest.Builder req = + SetVolumePropertyRequest.newBuilder(); + req.setVolumeName(volume).setQuotaInBytes(quota); + final SetVolumePropertyResponse resp; + try { + resp = rpcProxy.setVolumeProperty(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new + IOException("Volume quota change failed, error:" + resp.getStatus()); + } + } + + /** + * Checks if the specified user can access this volume. + * + * @param volume - volume + * @param userAcl - user acls which needs to be checked for access + * @return true if the user has required access for the volume, + * false otherwise + * @throws IOException + */ + @Override + public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl) throws + IOException { + CheckVolumeAccessRequest.Builder req = + CheckVolumeAccessRequest.newBuilder(); + req.setVolumeName(volume).setUserAcl(userAcl); + final CheckVolumeAccessResponse resp; + try { + resp = rpcProxy.checkVolumeAccess(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if (resp.getStatus() == Status.ACCESS_DENIED) { + return false; + } else if (resp.getStatus() == Status.OK) { + return true; + } else { + throw new + IOException("Check Volume Access failed, error:" + resp.getStatus()); + } + } + + /** + * Gets the volume information. + * + * @param volume - Volume name. + * @return KsmVolumeArgs or exception is thrown. + * @throws IOException + */ + @Override + public KsmVolumeArgs getVolumeInfo(String volume) throws IOException { + InfoVolumeRequest.Builder req = InfoVolumeRequest.newBuilder(); + req.setVolumeName(volume); + final InfoVolumeResponse resp; + try { + resp = rpcProxy.infoVolume(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new + IOException("Info Volume failed, error:" + resp.getStatus()); + } + return KsmVolumeArgs.getFromProtobuf(resp.getVolumeInfo()); + } + + /** + * Deletes an existing empty volume. + * + * @param volume - Name of the volume. + * @throws IOException + */ + @Override + public void deleteVolume(String volume) throws IOException { + DeleteVolumeRequest.Builder req = DeleteVolumeRequest.newBuilder(); + req.setVolumeName(volume); + final DeleteVolumeResponse resp; + try { + resp = rpcProxy.deleteVolume(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new + IOException("Delete Volume failed, error:" + resp.getStatus()); + } + } + + /** + * Lists volume owned by a specific user. + * + * @param userName - user name + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the + * prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + @Override + public List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix, + String prevKey, int maxKeys) + throws IOException { + ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder(); + if (!Strings.isNullOrEmpty(prefix)) { + builder.setPrefix(prefix); + } + if (!Strings.isNullOrEmpty(prevKey)) { + builder.setPrevKey(prevKey); + } + builder.setMaxKeys(maxKeys); + builder.setUserName(userName); + builder.setScope(ListVolumeRequest.Scope.VOLUMES_BY_USER); + return listVolume(builder.build()); + } + + /** + * Lists volume all volumes in the cluster. + * + * @param prefix - Filter prefix -- Return only entries that match this. + * @param prevKey - Previous key -- List starts from the next from the + * prevkey + * @param maxKeys - Max number of keys to return. + * @return List of Volumes. + * @throws IOException + */ + @Override + public List<KsmVolumeArgs> listAllVolumes(String prefix, String prevKey, + int maxKeys) throws IOException { + ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder(); + if (!Strings.isNullOrEmpty(prefix)) { + builder.setPrefix(prefix); + } + if (!Strings.isNullOrEmpty(prevKey)) { + builder.setPrevKey(prevKey); + } + builder.setMaxKeys(maxKeys); + builder.setScope(ListVolumeRequest.Scope.VOLUMES_BY_CLUSTER); + return listVolume(builder.build()); + } + + private List<KsmVolumeArgs> listVolume(ListVolumeRequest request) + throws IOException { + final ListVolumeResponse resp; + try { + resp = rpcProxy.listVolumes(NULL_RPC_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if (resp.getStatus() != Status.OK) { + throw new IOException("List volume failed, error: " + + resp.getStatus()); + } + + List<KsmVolumeArgs> result = Lists.newArrayList(); + for (VolumeInfo volInfo : resp.getVolumeInfoList()) { + KsmVolumeArgs volArgs = KsmVolumeArgs.getFromProtobuf(volInfo); + result.add(volArgs); + } + + return resp.getVolumeInfoList().stream() + .map(item -> KsmVolumeArgs.getFromProtobuf(item)) + .collect(Collectors.toList()); + } + + /** + * Creates a bucket. + * + * @param bucketInfo - BucketInfo to create bucket. + * @throws IOException + */ + @Override + public void createBucket(KsmBucketInfo bucketInfo) throws IOException { + CreateBucketRequest.Builder req = + CreateBucketRequest.newBuilder(); + BucketInfo bucketInfoProtobuf = bucketInfo.getProtobuf(); + req.setBucketInfo(bucketInfoProtobuf); + + final CreateBucketResponse resp; + try { + resp = rpcProxy.createBucket(NULL_RPC_CONTROLLER, + req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new IOException("Bucket creation failed, error: " + + resp.getStatus()); + } + } + + /** + * Gets the bucket information. + * + * @param volume - Volume name. + * @param bucket - Bucket name. + * @return KsmBucketInfo or exception is thrown. + * @throws IOException + */ + @Override + public KsmBucketInfo getBucketInfo(String volume, String bucket) + throws IOException { + InfoBucketRequest.Builder req = + InfoBucketRequest.newBuilder(); + req.setVolumeName(volume); + req.setBucketName(bucket); + + final InfoBucketResponse resp; + try { + resp = rpcProxy.infoBucket(NULL_RPC_CONTROLLER, + req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() == Status.OK) { + return KsmBucketInfo.getFromProtobuf(resp.getBucketInfo()); + } else { + throw new IOException("Info Bucket failed, error: " + + resp.getStatus()); + } + } + + /** + * Sets bucket property from args. + * @param args - BucketArgs. + * @throws IOException + */ + @Override + public void setBucketProperty(KsmBucketArgs args) + throws IOException { + SetBucketPropertyRequest.Builder req = + SetBucketPropertyRequest.newBuilder(); + BucketArgs bucketArgs = args.getProtobuf(); + req.setBucketArgs(bucketArgs); + final SetBucketPropertyResponse resp; + try { + resp = rpcProxy.setBucketProperty(NULL_RPC_CONTROLLER, + req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new IOException("Setting bucket property failed, error: " + + resp.getStatus()); + } + } + + /** + * List buckets in a volume. + * + * @param volumeName + * @param startKey + * @param prefix + * @param count + * @return + * @throws IOException + */ + @Override + public List<KsmBucketInfo> listBuckets(String volumeName, + String startKey, String prefix, int count) throws IOException { + List<KsmBucketInfo> buckets = new ArrayList<>(); + ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.newBuilder(); + reqBuilder.setVolumeName(volumeName); + reqBuilder.setCount(count); + if (startKey != null) { + reqBuilder.setStartKey(startKey); + } + if (prefix != null) { + reqBuilder.setPrefix(prefix); + } + ListBucketsRequest request = reqBuilder.build(); + final ListBucketsResponse resp; + try { + resp = rpcProxy.listBuckets(NULL_RPC_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if (resp.getStatus() == Status.OK) { + buckets.addAll( + resp.getBucketInfoList().stream() + .map(KsmBucketInfo::getFromProtobuf) + .collect(Collectors.toList())); + return buckets; + } else { + throw new IOException("List Buckets failed, error: " + + resp.getStatus()); + } + } + + /** + * Allocate a block for a key, then use the returned meta info to talk to data + * node to actually write the key. + * @param args the args for the key to be allocated + * @return a handler to the key, returned client + * @throws IOException + */ + @Override + public KsmKeyInfo allocateKey(KsmKeyArgs args) throws IOException { + LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder(); + KeyArgs keyArgs = KeyArgs.newBuilder() + .setVolumeName(args.getVolumeName()) + .setBucketName(args.getBucketName()) + .setKeyName(args.getKeyName()) + .setDataSize(args.getDataSize()).build(); + req.setKeyArgs(keyArgs); + + final LocateKeyResponse resp; + try { + resp = rpcProxy.createKey(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new IOException("Create key failed, error:" + + resp.getStatus()); + } + return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo()); + } + + @Override + public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException { + LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder(); + KeyArgs keyArgs = KeyArgs.newBuilder() + .setVolumeName(args.getVolumeName()) + .setBucketName(args.getBucketName()) + .setKeyName(args.getKeyName()) + .setDataSize(args.getDataSize()).build(); + req.setKeyArgs(keyArgs); + + final LocateKeyResponse resp; + try { + resp = rpcProxy.lookupKey(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new IOException("Lookup key failed, error:" + + resp.getStatus()); + } + return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo()); + } + + /** + * Deletes an existing key. + * + * @param args the args of the key. + * @throws IOException + */ + @Override + public void deleteKey(KsmKeyArgs args) throws IOException { + LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder(); + KeyArgs keyArgs = KeyArgs.newBuilder() + .setVolumeName(args.getVolumeName()) + .setBucketName(args.getBucketName()) + .setKeyName(args.getKeyName()).build(); + req.setKeyArgs(keyArgs); + + final LocateKeyResponse resp; + try { + resp = rpcProxy.deleteKey(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new IOException("Delete key failed, error:" + + resp.getStatus()); + } + } + + /** + * Deletes an existing empty bucket from volume. + * @param volume - Name of the volume. + * @param bucket - Name of the bucket. + * @throws IOException + */ + public void deleteBucket(String volume, String bucket) throws IOException { + DeleteBucketRequest.Builder req = DeleteBucketRequest.newBuilder(); + req.setVolumeName(volume); + req.setBucketName(bucket); + final DeleteBucketResponse resp; + try { + resp = rpcProxy.deleteBucket(NULL_RPC_CONTROLLER, req.build()); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + if (resp.getStatus() != Status.OK) { + throw new + IOException("Delete Bucket failed, error:" + resp.getStatus()); + } + } + + /** + * List keys in a bucket. + */ + @Override + public List<KsmKeyInfo> listKeys(String volumeName, String bucketName, + String startKey, String prefix, int maxKeys) throws IOException { + List<KsmKeyInfo> keys = new ArrayList<>(); + ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder(); + reqBuilder.setVolumeName(volumeName); + reqBuilder.setBucketName(bucketName); + reqBuilder.setCount(maxKeys); + + if (startKey != null) { + reqBuilder.setStartKey(startKey); + } + + if (prefix != null) { + reqBuilder.setPrefix(prefix); + } + + ListKeysRequest request = reqBuilder.build(); + final ListKeysResponse resp; + try { + resp = rpcProxy.listKeys(NULL_RPC_CONTROLLER, request); + } catch (ServiceException e) { + throw ProtobufHelper.getRemoteException(e); + } + + if (resp.getStatus() == Status.OK) { + keys.addAll( + resp.getKeyInfoList().stream() + .map(KsmKeyInfo::getFromProtobuf) + .collect(Collectors.toList())); + return keys; + } else { + throw new IOException("List Keys failed, error: " + + resp.getStatus()); + } + } + + /** + * Return the proxy object underlying this protocol translator. + * + * @return the proxy object underlying this protocol translator. + */ + @Override + public Object getUnderlyingProxyObject() { + return null; + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java new file mode 100644 index 0000000..8acca8a --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.ksm.protocolPB; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.ipc.ProtocolInfo; +import org.apache.hadoop.ozone.protocol.proto + .KeySpaceManagerProtocolProtos.KeySpaceManagerService; + +/** + * Protocol used to communicate with KSM. + */ +@ProtocolInfo(protocolName = + "org.apache.hadoop.ozone.protocol.KeySpaceManagerProtocol", + protocolVersion = 1) +@InterfaceAudience.Private +public interface KeySpaceManagerProtocolPB + extends KeySpaceManagerService.BlockingInterface { +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java new file mode 100644 index 0000000..67f9f7b --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java @@ -0,0 +1,19 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.ksm.protocolPB; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/TestArchive.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/TestArchive.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/TestArchive.java deleted file mode 100644 index 6fff47e..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/TestArchive.java +++ /dev/null @@ -1,106 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.RandomStringUtils; -import org.apache.hadoop.fs.FileUtil; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Iterator; -import java.util.Random; -import java.util.zip.Adler32; -import java.util.zip.Checksum; - -/** - * Test archive creation and unpacking. - */ -public class TestArchive { - private static final int DIR_COUNT = 10; - private static final int SUB_DIR_COUNT = 3; - private static final int FILE_COUNT = 10; - @Rule - public TemporaryFolder folder = new TemporaryFolder(); - - @Rule - public TemporaryFolder outputFolder = new TemporaryFolder(); - - Checksum crc = new Adler32(); - - @Before - public void setUp() throws Exception { - Random r = new Random(); - final int megaByte = 1024 * 1024; - - for (int x = 0; x < DIR_COUNT; x++) { - File subdir = folder.newFolder(String.format("dir%d", x)); - for (int y = 0; y < SUB_DIR_COUNT; y++) { - File targetDir = new File(subdir.getPath().concat(File.separator) - .concat(String.format("subdir%d%d", x, y))); - if(!targetDir.mkdirs()) { - throw new IOException("Failed to create subdirectory. " + - targetDir.toString()); - } - for (int z = 0; z < FILE_COUNT; z++) { - Path temp = Paths.get(targetDir.getPath().concat(File.separator) - .concat(String.format("File%d.txt", z))); - byte[] buf = RandomStringUtils.randomAlphanumeric(r.nextInt(megaByte)) - .getBytes("UTF-8"); - Files.write(temp, buf); - crc.update(buf, 0, buf.length); - } - } - } - } - - @Test - public void testArchive() throws Exception { - Checksum readCrc = new Adler32(); - File archiveFile = new File(outputFolder.getRoot() + File.separator - + "test.container.zip"); - long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile); - Assert.assertTrue(zipCheckSum > 0); - File decomp = new File(outputFolder.getRoot() + File.separator + - "decompress"); - if (!decomp.exists() && !decomp.mkdirs()) { - throw new IOException("Unable to create the destination directory. " + - decomp.getPath()); - } - - FileUtil.unZip(archiveFile, decomp); - String[] patterns = {"txt"}; - Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true); - int count = 0; - while (iter.hasNext()) { - count++; - byte[] buf = Files.readAllBytes(iter.next().toPath()); - readCrc.update(buf, 0, buf.length); - } - Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count); - Assert.assertEquals(crc.getValue(), readCrc.getValue()); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/package-info.java deleted file mode 100644 index 9c480d6..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/src/test/java/org/apache/hadoop/scm/package-info.java +++ /dev/null @@ -1,21 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.scm; -/** - Test cases for SCM client classes. - */ \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
