http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/ChunkGroupOutputStream.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/ChunkGroupOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/ChunkGroupOutputStream.java deleted file mode 100644 index 59c2639..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/ChunkGroupOutputStream.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.ozone.web.storage; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result; -import org.apache.hadoop.ksm.helpers.KsmKeyInfo; -import org.apache.hadoop.ksm.helpers.KsmKeyLocationInfo; -import org.apache.hadoop.scm.XceiverClientManager; -import org.apache.hadoop.scm.XceiverClientSpi; -import org.apache.hadoop.scm.container.common.helpers.Pipeline; -import org.apache.hadoop.scm.container.common.helpers.StorageContainerException; -import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB; -import org.apache.hadoop.scm.storage.ChunkOutputStream; -import org.apache.hadoop.scm.storage.ContainerProtocolCalls; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; - -/** - * Maintaining a list of ChunkInputStream. Write based on offset. - * - * Note that this may write to multiple containers in one write call. In case - * that first container succeeded but later ones failed, the succeeded writes - * are not rolled back. - * - * TODO : currently not support multi-thread access. - */ -public class ChunkGroupOutputStream extends OutputStream { - - private static final Logger LOG = - LoggerFactory.getLogger(ChunkGroupOutputStream.class); - - // array list's get(index) is O(1) - private final ArrayList<ChunkOutputStreamEntry> streamEntries; - private int currentStreamIndex; - private long totalSize; - private long byteOffset; - - //This has to be removed once HDFS-11888 is resolved. - //local cache which will have list of created container names. - private static Set<String> containersCreated = new HashSet<>(); - - public ChunkGroupOutputStream() { - this.streamEntries = new ArrayList<>(); - this.currentStreamIndex = 0; - this.totalSize = 0; - this.byteOffset = 0; - } - - @VisibleForTesting - public long getByteOffset() { - return byteOffset; - } - - /** - * Append another stream to the end of the list. Note that the streams are not - * actually created to this point, only enough meta data about the stream is - * stored. When something is to be actually written to the stream, the stream - * will be created (if not already). - * - * @param containerKey the key to store in the container - * @param key the ozone key - * @param xceiverClientManager xceiver manager instance - * @param xceiverClient xceiver manager instance - * @param requestID the request id - * @param chunkSize the chunk size for this key chunks - * @param length the total length of this key - */ - public synchronized void addStream(String containerKey, String key, - XceiverClientManager xceiverClientManager, XceiverClientSpi xceiverClient, - String requestID, int chunkSize, long length) { - streamEntries.add(new ChunkOutputStreamEntry(containerKey, key, - xceiverClientManager, xceiverClient, requestID, chunkSize, length)); - totalSize += length; - } - - @VisibleForTesting - public synchronized void addStream(OutputStream outputStream, long length) { - streamEntries.add(new ChunkOutputStreamEntry(outputStream, length)); - totalSize += length; - } - - @Override - public synchronized void write(int b) throws IOException { - if (streamEntries.size() <= currentStreamIndex) { - throw new IndexOutOfBoundsException(); - } - ChunkOutputStreamEntry entry = streamEntries.get(currentStreamIndex); - entry.write(b); - if (entry.getRemaining() <= 0) { - currentStreamIndex += 1; - } - byteOffset += 1; - } - - /** - * Try to write the bytes sequence b[off:off+len) to streams. - * - * NOTE: Throws exception if the data could not fit into the remaining space. - * In which case nothing will be written. - * TODO:May need to revisit this behaviour. - * - * @param b byte data - * @param off starting offset - * @param len length to write - * @throws IOException - */ - @Override - public synchronized void write(byte[] b, int off, int len) - throws IOException { - if (b == null) { - throw new NullPointerException(); - } - if ((off < 0) || (off > b.length) || (len < 0) || - ((off + len) > b.length) || ((off + len) < 0)) { - throw new IndexOutOfBoundsException(); - } - if (len == 0) { - return; - } - if (streamEntries.size() <= currentStreamIndex) { - throw new IOException("Write out of stream range! stream index:" + - currentStreamIndex); - } - if (totalSize - byteOffset < len) { - throw new IOException("Can not write " + len + " bytes with only " + - (totalSize - byteOffset) + " byte space"); - } - while (len > 0) { - // in theory, this condition should never violate due the check above - // still do a sanity check. - Preconditions.checkArgument(currentStreamIndex < streamEntries.size()); - ChunkOutputStreamEntry current = streamEntries.get(currentStreamIndex); - int writeLen = Math.min(len, (int)current.getRemaining()); - current.write(b, off, writeLen); - if (current.getRemaining() <= 0) { - currentStreamIndex += 1; - } - len -= writeLen; - off += writeLen; - byteOffset += writeLen; - } - } - - @Override - public synchronized void flush() throws IOException { - for (int i = 0; i <= currentStreamIndex; i++) { - streamEntries.get(i).flush(); - } - } - - @Override - public synchronized void close() throws IOException { - for (ChunkOutputStreamEntry entry : streamEntries) { - entry.close(); - } - } - - private static class ChunkOutputStreamEntry extends OutputStream { - private OutputStream outputStream; - private final String containerKey; - private final String key; - private final XceiverClientManager xceiverClientManager; - private final XceiverClientSpi xceiverClient; - private final String requestId; - private final int chunkSize; - // total number of bytes that should be written to this stream - private final long length; - // the current position of this stream 0 <= currentPosition < length - private long currentPosition; - - ChunkOutputStreamEntry(String containerKey, String key, - XceiverClientManager xceiverClientManager, - XceiverClientSpi xceiverClient, String requestId, int chunkSize, - long length) { - this.outputStream = null; - this.containerKey = containerKey; - this.key = key; - this.xceiverClientManager = xceiverClientManager; - this.xceiverClient = xceiverClient; - this.requestId = requestId; - this.chunkSize = chunkSize; - - this.length = length; - this.currentPosition = 0; - } - - /** - * For testing purpose, taking a some random created stream instance. - * @param outputStream a existing writable output stream - * @param length the length of data to write to the stream - */ - ChunkOutputStreamEntry(OutputStream outputStream, long length) { - this.outputStream = outputStream; - this.containerKey = null; - this.key = null; - this.xceiverClientManager = null; - this.xceiverClient = null; - this.requestId = null; - this.chunkSize = -1; - - this.length = length; - this.currentPosition = 0; - } - - long getLength() { - return length; - } - - long getRemaining() { - return length - currentPosition; - } - - private synchronized void checkStream() { - if (this.outputStream == null) { - this.outputStream = new ChunkOutputStream(containerKey, - key, xceiverClientManager, xceiverClient, - requestId, chunkSize); - } - } - - @Override - public void write(int b) throws IOException { - checkStream(); - outputStream.write(b); - this.currentPosition += 1; - } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - checkStream(); - outputStream.write(b, off, len); - this.currentPosition += len; - } - - @Override - public void flush() throws IOException { - if (this.outputStream != null) { - this.outputStream.flush(); - } - } - - @Override - public void close() throws IOException { - if (this.outputStream != null) { - this.outputStream.close(); - } - } - } - - public static ChunkGroupOutputStream getFromKsmKeyInfo( - KsmKeyInfo keyInfo, XceiverClientManager xceiverClientManager, - StorageContainerLocationProtocolClientSideTranslatorPB - storageContainerLocationClient, - int chunkSize, String requestId) throws IOException { - // TODO: the following createContainer and key writes may fail, in which - // case we should revert the above allocateKey to KSM. - // check index as sanity check - int index = 0; - String containerKey; - ChunkGroupOutputStream groupOutputStream = new ChunkGroupOutputStream(); - for (KsmKeyLocationInfo subKeyInfo : keyInfo.getKeyLocationList()) { - containerKey = subKeyInfo.getBlockID(); - - Preconditions.checkArgument(index++ == subKeyInfo.getIndex()); - String containerName = subKeyInfo.getContainerName(); - Pipeline pipeline = - storageContainerLocationClient.getContainer(containerName); - XceiverClientSpi xceiverClient = - xceiverClientManager.acquireClient(pipeline); - // create container if needed - // TODO : should be subKeyInfo.getShouldCreateContainer(), but for now - //The following change has to reverted once HDFS-11888 is fixed. - if(!containersCreated.contains(containerName)) { - synchronized (containerName.intern()) { - //checking again, there is a chance that some other thread has - // created it. - if (!containersCreated.contains(containerName)) { - LOG.debug("Need to create container {}.", containerName); - try { - ContainerProtocolCalls.createContainer(xceiverClient, requestId); - } catch (StorageContainerException ex) { - if (ex.getResult().equals(Result.CONTAINER_EXISTS)) { - //container already exist. - LOG.debug("Container {} already exists.", containerName); - } else { - LOG.error("Container creation failed for {}.", - containerName, ex); - throw ex; - } - } - containersCreated.add(containerName); - } - } - } - - groupOutputStream.addStream(containerKey, keyInfo.getKeyName(), - xceiverClientManager, xceiverClient, requestId, chunkSize, - subKeyInfo.getLength()); - } - return groupOutputStream; - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java index 713a085..76c9be0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java @@ -19,20 +19,21 @@ package org.apache.hadoop.ozone.web.storage; import com.google.common.base.Strings; -import org.apache.hadoop.hdfs.server.datanode.fsdataset - .LengthInputStream; import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.ksm.helpers.KsmBucketArgs; -import org.apache.hadoop.ksm.helpers.KsmBucketInfo; -import org.apache.hadoop.ksm.helpers.KsmKeyArgs; -import org.apache.hadoop.ksm.helpers.KsmKeyInfo; -import org.apache.hadoop.ksm.helpers.KsmVolumeArgs; -import org.apache.hadoop.ksm.protocolPB +import org.apache.hadoop.ozone.client.io.LengthInputStream; +import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs; +import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo; +import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs; +import org.apache.hadoop.ozone.ksm.protocolPB .KeySpaceManagerProtocolClientSideTranslatorPB; import org.apache.hadoop.ozone.OzoneConfiguration; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.OzoneConsts.Versioning; -import org.apache.hadoop.ozone.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream; +import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos; import org.apache.hadoop.ozone.protocolPB.KSMPBHelper; import org.apache.hadoop.ozone.ksm.KSMConfigKeys; http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java deleted file mode 100644 index 18ade6e..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/OzoneContainerTranslation.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone.web.storage; - -import java.util.List; - -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData; -import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue; -import org.apache.hadoop.ozone.OzoneConsts.Versioning; -import org.apache.hadoop.ozone.web.request.OzoneQuota; -import org.apache.hadoop.ozone.web.response.BucketInfo; -import org.apache.hadoop.ozone.web.response.KeyInfo; -import org.apache.hadoop.ozone.web.response.VolumeInfo; -import org.apache.hadoop.ozone.web.response.VolumeOwner; -import org.apache.hadoop.util.StringUtils; - -/** - * This class contains methods that define the translation between the Ozone - * domain model and the storage container domain model. - */ -final class OzoneContainerTranslation { - - private static final String ACLS = "ACLS"; - private static final String BUCKET = "BUCKET"; - private static final String BUCKET_NAME = "BUCKET_NAME"; - private static final String CREATED_BY = "CREATED_BY"; - private static final String CREATED_ON = "CREATED_ON"; - private static final String KEY = "KEY"; - private static final String OWNER = "OWNER"; - private static final String QUOTA = "QUOTA"; - private static final String STORAGE_TYPE = "STORAGE_TYPE"; - private static final String TYPE = "TYPE"; - private static final String VERSIONING = "VERSIONING"; - private static final String VOLUME = "VOLUME"; - private static final String VOLUME_NAME = "VOLUME_NAME"; - - /** - * Creates key data intended for reading a container key. - * - * @param containerName container name - * @param containerKey container key - * @return KeyData intended for reading the container key - */ - public static KeyData containerKeyDataForRead(String containerName, - String containerKey) { - return KeyData - .newBuilder() - .setContainerName(containerName) - .setName(containerKey) - .build(); - } - - /** - * Translates a bucket to its container representation. - * - * @param containerName container name - * @param containerKey container key - * @param bucket the bucket to translate - * @return KeyData representation of bucket - */ - public static KeyData fromBucketToContainerKeyData( - String containerName, String containerKey, BucketInfo bucket) { - KeyData.Builder containerKeyData = KeyData - .newBuilder() - .setContainerName(containerName) - .setName(containerKey) - .addMetadata(newKeyValue(TYPE, BUCKET)) - .addMetadata(newKeyValue(VOLUME_NAME, bucket.getVolumeName())) - .addMetadata(newKeyValue(BUCKET_NAME, bucket.getBucketName())); - - if (bucket.getAcls() != null) { - containerKeyData.addMetadata(newKeyValue(ACLS, - StringUtils.join(',', bucket.getAcls()))); - } - - if (bucket.getVersioning() != null && - bucket.getVersioning() != Versioning.NOT_DEFINED) { - containerKeyData.addMetadata(newKeyValue(VERSIONING, - bucket.getVersioning().name())); - } - - if (bucket.getStorageType() != StorageType.RAM_DISK) { - containerKeyData.addMetadata(newKeyValue(STORAGE_TYPE, - bucket.getStorageType().name())); - } - - return containerKeyData.build(); - } - - /** - * Translates a bucket from its container representation. - * - * @param metadata container metadata representing the bucket - * @return bucket translated from container representation - */ - public static BucketInfo fromContainerKeyValueListToBucket( - List<KeyValue> metadata) { - BucketInfo bucket = new BucketInfo(); - for (KeyValue keyValue : metadata) { - switch (keyValue.getKey()) { - case VOLUME_NAME: - bucket.setVolumeName(keyValue.getValue()); - break; - case BUCKET_NAME: - bucket.setBucketName(keyValue.getValue()); - break; - case VERSIONING: - bucket.setVersioning( - Enum.valueOf(Versioning.class, keyValue.getValue())); - break; - case STORAGE_TYPE: - bucket.setStorageType( - Enum.valueOf(StorageType.class, keyValue.getValue())); - break; - default: - break; - } - } - return bucket; - } - - /** - * Translates a volume from its container representation. - * - * @param metadata container metadata representing the volume - * @return volume translated from container representation - */ - public static VolumeInfo fromContainerKeyValueListToVolume( - List<KeyValue> metadata) { - VolumeInfo volume = new VolumeInfo(); - for (KeyValue keyValue : metadata) { - switch (keyValue.getKey()) { - case VOLUME_NAME: - volume.setVolumeName(keyValue.getValue()); - break; - case CREATED_BY: - volume.setCreatedBy(keyValue.getValue()); - break; - case CREATED_ON: - volume.setCreatedOn(keyValue.getValue()); - break; - case OWNER: - volume.setOwner(new VolumeOwner(keyValue.getValue())); - break; - case QUOTA: - volume.setQuota(OzoneQuota.parseQuota(keyValue.getValue())); - break; - default: - break; - } - } - return volume; - } - - /** - * Translates a key to its container representation. - * - * @param containerName container name - * @param containerKey container key - * @param keyInfo key information received from call - * @return KeyData intended for reading the container key - */ - public static KeyData fromKeyToContainerKeyData(String containerName, - String containerKey, KeyInfo key) { - return KeyData - .newBuilder() - .setContainerName(containerName) - .setName(containerKey) - .addMetadata(newKeyValue(TYPE, KEY)) - .build(); - } - - /** - * Translates a key to its container representation. The return value is a - * builder that can be manipulated further before building the result. - * - * @param containerName container name - * @param containerKey container key - * @param keyInfo key information received from call - * @return KeyData builder - */ - public static KeyData.Builder fromKeyToContainerKeyDataBuilder( - String containerName, String containerKey, KeyInfo key) { - return KeyData - .newBuilder() - .setContainerName(containerName) - .setName(containerKey) - .addMetadata(newKeyValue(TYPE, KEY)); - } - - /** - * Translates a volume to its container representation. - * - * @param containerName container name - * @param containerKey container key - * @param volume the volume to translate - * @return KeyData representation of volume - */ - public static KeyData fromVolumeToContainerKeyData( - String containerName, String containerKey, VolumeInfo volume) { - KeyData.Builder containerKeyData = KeyData - .newBuilder() - .setContainerName(containerName) - .setName(containerKey) - .addMetadata(newKeyValue(TYPE, VOLUME)) - .addMetadata(newKeyValue(VOLUME_NAME, volume.getVolumeName())) - .addMetadata(newKeyValue(CREATED_ON, volume.getCreatedOn())); - - if (volume.getQuota() != null && volume.getQuota().sizeInBytes() != -1L) { - containerKeyData.addMetadata(newKeyValue(QUOTA, - OzoneQuota.formatQuota(volume.getQuota()))); - } - - if (volume.getOwner() != null && volume.getOwner().getName() != null && - !volume.getOwner().getName().isEmpty()) { - containerKeyData.addMetadata(newKeyValue(OWNER, - volume.getOwner().getName())); - } - - if (volume.getCreatedBy() != null && !volume.getCreatedBy().isEmpty()) { - containerKeyData.addMetadata( - newKeyValue(CREATED_BY, volume.getCreatedBy())); - } - - return containerKeyData.build(); - } - - /** - * Translates a key-value pair to its container representation. - * - * @param key the key - * @param value the value - * @return container representation of key-value pair - */ - private static KeyValue newKeyValue(String key, Object value) { - return KeyValue.newBuilder().setKey(key).setValue(value.toString()).build(); - } - - /** - * There is no need to instantiate this class. - */ - private OzoneContainerTranslation() { - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java index 609a47b..6cce47e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java @@ -22,7 +22,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.ozone.web.exceptions.ErrorTable; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.handlers.UserArgs; -import org.apache.hadoop.ozone.web.headers.Header; +import org.apache.hadoop.ozone.client.rest.headers.Header; import org.apache.hadoop.ozone.web.interfaces.UserAuth; import org.apache.hadoop.ozone.OzoneConsts; http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java index c417601..2fe64d5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java @@ -23,11 +23,11 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ozone.OzoneConfigKeys; import org.apache.hadoop.ozone.OzoneConsts; -import org.apache.hadoop.hdfs.server.datanode.fsdataset.LengthInputStream; +import org.apache.hadoop.ozone.client.io.LengthInputStream; import org.apache.hadoop.ozone.web.exceptions.ErrorTable; import org.apache.hadoop.ozone.web.exceptions.OzoneException; import org.apache.hadoop.ozone.web.handlers.UserArgs; -import org.apache.hadoop.ozone.web.headers.Header; +import org.apache.hadoop.ozone.client.rest.headers.Header; import org.apache.hadoop.util.Time; import javax.ws.rs.core.HttpHeaders; http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/Corona.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/Corona.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/Corona.java index 581ccc6..4d69ba9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/Corona.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/Corona.java @@ -26,7 +26,9 @@ import org.apache.commons.cli.Options; import org.apache.commons.lang.RandomStringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; -import org.apache.hadoop.ozone.io.OzoneOutputStream; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.hadoop.util.Tool; import org.apache.hadoop.util.ToolRunner; @@ -132,7 +134,7 @@ public final class Corona extends Configured implements Tool { numberOfBucketsCreated = new AtomicInteger(); numberOfKeysAdded = new AtomicLong(); OzoneClientFactory.setConfiguration(conf); - ozoneClient = OzoneClientFactory.getRpcClient(); + ozoneClient = OzoneClientFactory.getClient(); } @Override http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java deleted file mode 100644 index b861f7d..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientImpl.java +++ /dev/null @@ -1,357 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone; - -import org.apache.hadoop.fs.StorageType; -import org.apache.hadoop.ozone.io.OzoneInputStream; -import org.apache.hadoop.ozone.io.OzoneOutputStream; -import org.apache.hadoop.ozone.web.exceptions.OzoneException; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -/** - * This class is to test all the public facing APIs of Ozone Client. - */ -public class TestOzoneClientImpl { - - @Rule - public ExpectedException thrown = ExpectedException.none(); - - private static MiniOzoneCluster cluster = null; - private static OzoneClientImpl ozClient = null; - - /** - * Create a MiniDFSCluster for testing. - * <p> - * Ozone is made active by setting OZONE_ENABLED = true and - * OZONE_HANDLER_TYPE_KEY = "distributed" - * - * @throws IOException - */ - @BeforeClass - public static void init() throws Exception { - OzoneConfiguration conf = new OzoneConfiguration(); - conf.set(OzoneConfigKeys.OZONE_HANDLER_TYPE_KEY, - OzoneConsts.OZONE_HANDLER_DISTRIBUTED); - cluster = new MiniOzoneCluster.Builder(conf) - .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build(); - ozClient = new OzoneClientImpl(conf); - } - - @Test - public void testCreateVolume() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - OzoneVolume volume = ozClient.getVolumeDetails(volumeName); - Assert.assertEquals(volumeName, volume.getVolumeName()); - } - - @Test - public void testCreateVolumeWithOwner() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName, "test"); - OzoneVolume volume = ozClient.getVolumeDetails(volumeName); - Assert.assertEquals(volumeName, volume.getVolumeName()); - Assert.assertEquals("test", volume.getOwnerName()); - } - - @Test - public void testCreateVolumeWithQuota() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName, "test", - 10000000000L); - OzoneVolume volume = ozClient.getVolumeDetails(volumeName); - Assert.assertEquals(volumeName, volume.getVolumeName()); - Assert.assertEquals("test", volume.getOwnerName()); - Assert.assertEquals(10000000000L, volume.getQuota()); - } - - @Test - public void testVolumeAlreadyExist() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - try { - ozClient.createVolume(volumeName); - } catch (IOException ex) { - Assert.assertEquals( - "Volume creation failed, error:VOLUME_ALREADY_EXISTS", - ex.getMessage()); - } - } - - @Test - public void testSetVolumeOwner() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.setVolumeOwner(volumeName, "test"); - OzoneVolume volume = ozClient.getVolumeDetails(volumeName); - Assert.assertEquals("test", volume.getOwnerName()); - } - - @Test - public void testSetVolumeQuota() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.setVolumeQuota(volumeName, 10000000000L); - OzoneVolume volume = ozClient.getVolumeDetails(volumeName); - Assert.assertEquals(10000000000L, volume.getQuota()); - } - - @Test - public void testDeleteVolume() - throws IOException, OzoneException { - thrown.expectMessage("Info Volume failed, error"); - String volumeName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - OzoneVolume volume = ozClient.getVolumeDetails(volumeName); - Assert.assertNotNull(volume); - ozClient.deleteVolume(volumeName); - ozClient.getVolumeDetails(volumeName); - } - - @Test - public void testCreateBucket() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - } - - @Test - public void testCreateBucketWithVersioning() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName, - OzoneConsts.Versioning.ENABLED); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertEquals(OzoneConsts.Versioning.ENABLED, - bucket.getVersioning()); - } - - @Test - public void testCreateBucketWithStorageType() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName, StorageType.SSD); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertEquals(StorageType.SSD, bucket.getStorageType()); - } - - @Test - public void testCreateBucketWithAcls() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test", - OzoneAcl.OzoneACLRights.READ_WRITE); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName, userAcl); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertTrue(bucket.getAcls().contains(userAcl)); - } - - @Test - public void testCreateBucketWithAllArgument() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test", - OzoneAcl.OzoneACLRights.READ_WRITE); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName, - OzoneConsts.Versioning.ENABLED, - StorageType.SSD, userAcl); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertEquals(OzoneConsts.Versioning.ENABLED, - bucket.getVersioning()); - Assert.assertEquals(StorageType.SSD, bucket.getStorageType()); - Assert.assertTrue(bucket.getAcls().contains(userAcl)); - } - - @Test - public void testCreateBucketInInvalidVolume() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - try { - ozClient.createBucket(volumeName, bucketName); - } catch (IOException ex) { - Assert.assertEquals( - "Bucket creation failed, error: VOLUME_NOT_FOUND", - ex.getMessage()); - } - } - - @Test - public void testAddBucketAcl() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - List<OzoneAcl> acls = new ArrayList<>(); - acls.add(new OzoneAcl( - OzoneAcl.OzoneACLType.USER, "test", - OzoneAcl.OzoneACLRights.READ_WRITE)); - ozClient.addBucketAcls(volumeName, bucketName, acls); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertTrue(bucket.getAcls().contains(acls.get(0))); - } - - @Test - public void testRemoveBucketAcl() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - OzoneAcl userAcl = new OzoneAcl(OzoneAcl.OzoneACLType.USER, "test", - OzoneAcl.OzoneACLRights.READ_WRITE); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName, userAcl); - List<OzoneAcl> acls = new ArrayList<>(); - acls.add(userAcl); - ozClient.removeBucketAcls(volumeName, bucketName, acls); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertTrue(!bucket.getAcls().contains(acls.get(0))); - } - - @Test - public void testSetBucketVersioning() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - ozClient.setBucketVersioning(volumeName, bucketName, - OzoneConsts.Versioning.ENABLED); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertEquals(OzoneConsts.Versioning.ENABLED, - bucket.getVersioning()); - } - - @Test - public void testSetBucketStorageType() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - ozClient.setBucketStorageType(volumeName, bucketName, - StorageType.SSD); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertEquals(bucketName, bucket.getBucketName()); - Assert.assertEquals(StorageType.SSD, bucket.getStorageType()); - } - - - @Test - public void testDeleteBucket() - throws IOException, OzoneException { - thrown.expectMessage("Info Bucket failed, error"); - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - OzoneBucket bucket = ozClient.getBucketDetails(volumeName, bucketName); - Assert.assertNotNull(bucket); - ozClient.deleteBucket(volumeName, bucketName); - ozClient.getBucketDetails(volumeName, bucketName); - } - - - @Test - public void testPutKey() - throws IOException, OzoneException { - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); - String value = "sample value"; - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - OzoneOutputStream out = ozClient.createKey(volumeName, bucketName, - keyName, value.getBytes().length); - out.write(value.getBytes()); - out.close(); - OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName); - Assert.assertEquals(keyName, key.getKeyName()); - OzoneInputStream is = ozClient.getKey(volumeName, bucketName, keyName); - byte[] fileContent = new byte[value.getBytes().length]; - is.read(fileContent); - Assert.assertEquals(value, new String(fileContent)); - } - - @Test - public void testDeleteKey() - throws IOException, OzoneException { - thrown.expectMessage("Lookup key failed, error"); - String volumeName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); - String keyName = UUID.randomUUID().toString(); - String value = "sample value"; - ozClient.createVolume(volumeName); - ozClient.createBucket(volumeName, bucketName); - OzoneOutputStream out = ozClient.createKey(volumeName, bucketName, - keyName, value.getBytes().length); - out.write(value.getBytes()); - out.close(); - OzoneKey key = ozClient.getKeyDetails(volumeName, bucketName, keyName); - Assert.assertEquals(keyName, key.getKeyName()); - ozClient.deleteKey(volumeName, bucketName, keyName); - ozClient.getKeyDetails(volumeName, bucketName, keyName); - } - - /** - * Shutdown MiniDFSCluster. - */ - @AfterClass - public static void shutdown() { - if (cluster != null) { - cluster.shutdown(); - } - } - -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java deleted file mode 100644 index a5dbea7..0000000 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/TestOzoneClientUtils.java +++ /dev/null @@ -1,350 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.ozone; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.ozone.ksm.KSMConfigKeys; -import org.apache.hadoop.scm.ScmConfigKeys; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -import org.junit.Rule; -import org.junit.rules.Timeout; - -import java.net.InetSocketAddress; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; - -import static org.hamcrest.core.Is.is; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -/** - * This test class verifies the parsing of SCM endpoint config settings. - * The parsing logic is in {@link OzoneClientUtils}. - */ -public class TestOzoneClientUtils { - @Rule - public Timeout timeout = new Timeout(300000); - - @Rule - public ExpectedException thrown= ExpectedException.none(); - - /** - * Verify client endpoint lookup failure if it is not configured. - */ - @Test - public void testMissingScmClientAddress() { - final Configuration conf = new OzoneConfiguration(); - thrown.expect(IllegalArgumentException.class); - OzoneClientUtils.getScmAddressForClients(conf); - } - - /** - * Verify that the client endpoint can be correctly parsed from - * configuration. - */ - @Test - public void testGetScmClientAddress() { - final Configuration conf = new OzoneConfiguration(); - - // First try a client address with just a host name. Verify it falls - // back to the default port. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = OzoneClientUtils.getScmAddressForClients(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); - - // Next try a client address with a host name and port. Verify both - // are used correctly. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - addr = OzoneClientUtils.getScmAddressForClients(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(100)); - } - - /** - * Verify DataNode endpoint lookup failure if neither the client nor - * datanode endpoint are configured. - */ - @Test - public void testMissingScmDataNodeAddress() { - final Configuration conf = new OzoneConfiguration(); - thrown.expect(IllegalArgumentException.class); - OzoneClientUtils.getScmAddressForDataNodes(conf); - } - - /** - * Verify that the datanode endpoint is parsed correctly. - * This tests the logic used by the DataNodes to determine which address - * to connect to. - */ - @Test - public void testGetScmDataNodeAddress() { - final Configuration conf = new OzoneConfiguration(); - - // First try a client address with just a host name. Verify it falls - // back to the default port. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = OzoneClientUtils.getScmAddressForDataNodes(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - - // Next try a client address with just a host name and port. Verify the port - // is ignored and the default DataNode port is used. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - addr = OzoneClientUtils.getScmAddressForDataNodes(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - - // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and OZONE_SCM_DATANODE_ADDRESS_KEY. - // Verify that the latter overrides and the port number is still the default. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8"); - addr = OzoneClientUtils.getScmAddressForDataNodes(conf); - assertThat(addr.getHostString(), is("5.6.7.8")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - - // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and OZONE_SCM_DATANODE_ADDRESS_KEY. - // Verify that the latter overrides and the port number from the latter is - // used. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8:200"); - addr = OzoneClientUtils.getScmAddressForDataNodes(conf); - assertThat(addr.getHostString(), is("5.6.7.8")); - assertThat(addr.getPort(), is(200)); - } - - /** - * Verify that the client endpoint bind address is computed correctly. - * This tests the logic used by the SCM to determine its own bind address. - */ - @Test - public void testScmClientBindHostDefault() { - final Configuration conf = new OzoneConfiguration(); - - // The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY - // is set differently. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = OzoneClientUtils.getScmClientBindAddress(conf); - assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); - - // The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY - // is set differently. The port number from OZONE_SCM_CLIENT_ADDRESS_KEY - // should be respected. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); - addr = OzoneClientUtils.getScmClientBindAddress(conf); - assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(100)); - - // OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected. - // Port number should be default if none is specified via - // OZONE_SCM_DATANODE_ADDRESS_KEY. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4"); - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8"); - addr = OzoneClientUtils.getScmClientBindAddress(conf); - assertThat(addr.getHostString(), is("5.6.7.8")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); - - // OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected. - // Port number from OZONE_SCM_CLIENT_ADDRESS_KEY should be - // respected. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8"); - addr = OzoneClientUtils.getScmClientBindAddress(conf); - assertThat(addr.getHostString(), is("5.6.7.8")); - assertThat(addr.getPort(), is(100)); - } - - /** - * Verify that the DataNode endpoint bind address is computed correctly. - * This tests the logic used by the SCM to determine its own bind address. - */ - @Test - public void testScmDataNodeBindHostDefault() { - final Configuration conf = new OzoneConfiguration(); - - // The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY - // is set differently. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); - assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - - // The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY - // is set differently. The port number from OZONE_SCM_DATANODE_ADDRESS_KEY - // should be respected. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); - addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); - assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(200)); - - // OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected. - // Port number should be default if none is specified via - // OZONE_SCM_DATANODE_ADDRESS_KEY. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8"); - addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); - assertThat(addr.getHostString(), is("5.6.7.8")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); - - // OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected. - // Port number from OZONE_SCM_DATANODE_ADDRESS_KEY should be - // respected. - conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); - conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8"); - addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); - assertThat(addr.getHostString(), is("5.6.7.8")); - assertThat(addr.getPort(), is(200)); - } - - @Test - public void testGetSCMAddresses() { - final Configuration conf = new OzoneConfiguration(); - Collection<InetSocketAddress> addresses = null; - InetSocketAddress addr = null; - Iterator<InetSocketAddress> it = null; - - // Verify valid IP address setup - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "1.2.3.4"); - addresses = OzoneClientUtils.getSCMAddresses(conf); - assertThat(addresses.size(), is(1)); - addr = addresses.iterator().next(); - assertThat(addr.getHostName(), is("1.2.3.4")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT)); - - // Verify valid hostname setup - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1"); - addresses = OzoneClientUtils.getSCMAddresses(conf); - assertThat(addresses.size(), is(1)); - addr = addresses.iterator().next(); - assertThat(addr.getHostName(), is("scm1")); - assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT)); - - // Verify valid hostname and port - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234"); - addresses = OzoneClientUtils.getSCMAddresses(conf); - assertThat(addresses.size(), is(1)); - addr = addresses.iterator().next(); - assertThat(addr.getHostName(), is("scm1")); - assertThat(addr.getPort(), is(1234)); - - final HashMap<String, Integer> hostsAndPorts = - new HashMap<String, Integer>(); - hostsAndPorts.put("scm1", 1234); - hostsAndPorts.put("scm2", 2345); - hostsAndPorts.put("scm3", 3456); - - // Verify multiple hosts and port - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456"); - addresses = OzoneClientUtils.getSCMAddresses(conf); - assertThat(addresses.size(), is(3)); - it = addresses.iterator(); - HashMap<String, Integer> expected1 = new HashMap<>(hostsAndPorts); - while(it.hasNext()) { - InetSocketAddress current = it.next(); - assertTrue(expected1.remove(current.getHostName(), - current.getPort())); - } - assertTrue(expected1.isEmpty()); - - // Verify names with spaces - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 "); - addresses = OzoneClientUtils.getSCMAddresses(conf); - assertThat(addresses.size(), is(3)); - it = addresses.iterator(); - HashMap<String, Integer> expected2 = new HashMap<>(hostsAndPorts); - while(it.hasNext()) { - InetSocketAddress current = it.next(); - assertTrue(expected2.remove(current.getHostName(), - current.getPort())); - } - assertTrue(expected2.isEmpty()); - - // Verify empty value - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, ""); - try { - addresses = OzoneClientUtils.getSCMAddresses(conf); - fail("Empty value should cause an IllegalArgumentException"); - } catch (Exception e) { - assertTrue(e instanceof IllegalArgumentException); - } - - // Verify invalid hostname - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "s..x..:1234"); - try { - addresses = OzoneClientUtils.getSCMAddresses(conf); - fail("An invalid hostname should cause an IllegalArgumentException"); - } catch (Exception e) { - assertTrue(e instanceof IllegalArgumentException); - } - - // Verify invalid port - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz"); - try { - addresses = OzoneClientUtils.getSCMAddresses(conf); - fail("An invalid port should cause an IllegalArgumentException"); - } catch (Exception e) { - assertTrue(e instanceof IllegalArgumentException); - } - - // Verify a mixed case (valid and invalid value both appears) - conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234, scm:xyz"); - try { - addresses = OzoneClientUtils.getSCMAddresses(conf); - fail("An invalid value should cause an IllegalArgumentException"); - } catch (Exception e) { - assertTrue(e instanceof IllegalArgumentException); - } - } - - @Test - public void testGetKSMAddress() { - final Configuration conf = new OzoneConfiguration(); - - // First try a client address with just a host name. Verify it falls - // back to the default port. - conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4"); - InetSocketAddress addr = OzoneClientUtils.getKsmAddress(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); - - // Next try a client address with just a host name and port. Verify the port - // is ignored and the default KSM port is used. - conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100"); - addr = OzoneClientUtils.getKsmAddress(conf); - assertThat(addr.getHostString(), is("1.2.3.4")); - assertThat(addr.getPort(), is(100)); - - // Assert the we are able to use default configs if no value is specified. - conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, ""); - addr = OzoneClientUtils.getKsmAddress(conf); - assertThat(addr.getHostString(), is("0.0.0.0")); - assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientUtils.java new file mode 100644 index 0000000..4ca5d83 --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/TestOzoneClientUtils.java @@ -0,0 +1,362 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.client; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.ozone.OzoneConfiguration; +import org.apache.hadoop.ozone.ksm.KSMConfigKeys; +import org.apache.hadoop.scm.ScmConfigKeys; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import org.junit.Rule; +import org.junit.rules.Timeout; + +import java.net.InetSocketAddress; +import java.util.Collection; +import java.util.HashMap; +import java.util.Iterator; + +import static org.hamcrest.core.Is.is; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * This test class verifies the parsing of SCM endpoint config settings. + * The parsing logic is in {@link OzoneClientUtils}. + */ +public class TestOzoneClientUtils { + @Rule + public Timeout timeout = new Timeout(300000); + + @Rule + public ExpectedException thrown= ExpectedException.none(); + + /** + * Verify client endpoint lookup failure if it is not configured. + */ + @Test + public void testMissingScmClientAddress() { + final Configuration conf = new OzoneConfiguration(); + thrown.expect(IllegalArgumentException.class); + OzoneClientUtils.getScmAddressForClients(conf); + } + + /** + * Verify that the client endpoint can be correctly parsed from + * configuration. + */ + @Test + public void testGetScmClientAddress() { + final Configuration conf = new OzoneConfiguration(); + + // First try a client address with just a host name. Verify it falls + // back to the default port. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = OzoneClientUtils.getScmAddressForClients(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); + + // Next try a client address with a host name and port. Verify both + // are used correctly. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + addr = OzoneClientUtils.getScmAddressForClients(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is(100)); + } + + /** + * Verify DataNode endpoint lookup failure if neither the client nor + * datanode endpoint are configured. + */ + @Test + public void testMissingScmDataNodeAddress() { + final Configuration conf = new OzoneConfiguration(); + thrown.expect(IllegalArgumentException.class); + OzoneClientUtils.getScmAddressForDataNodes(conf); + } + + /** + * Verify that the datanode endpoint is parsed correctly. + * This tests the logic used by the DataNodes to determine which address + * to connect to. + */ + @Test + public void testGetScmDataNodeAddress() { + final Configuration conf = new OzoneConfiguration(); + + // First try a client address with just a host name. Verify it falls + // back to the default port. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = OzoneClientUtils.getScmAddressForDataNodes(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is( + ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + + // Next try a client address with just a host name and port. + // Verify the port is ignored and the default DataNode port is used. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + addr = OzoneClientUtils.getScmAddressForDataNodes(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is( + ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + + // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and + // OZONE_SCM_DATANODE_ADDRESS_KEY. + // Verify that the latter overrides and the port number is still the + // default. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8"); + addr = OzoneClientUtils.getScmAddressForDataNodes(conf); + assertThat(addr.getHostString(), is("5.6.7.8")); + assertThat(addr.getPort(), is( + ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + + // Set both OZONE_SCM_CLIENT_ADDRESS_KEY and + // OZONE_SCM_DATANODE_ADDRESS_KEY. + // Verify that the latter overrides and the port number from the latter is + // used. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "5.6.7.8:200"); + addr = OzoneClientUtils.getScmAddressForDataNodes(conf); + assertThat(addr.getHostString(), is("5.6.7.8")); + assertThat(addr.getPort(), is(200)); + } + + /** + * Verify that the client endpoint bind address is computed correctly. + * This tests the logic used by the SCM to determine its own bind address. + */ + @Test + public void testScmClientBindHostDefault() { + final Configuration conf = new OzoneConfiguration(); + + // The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY + // is set differently. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = OzoneClientUtils.getScmClientBindAddress(conf); + assertThat(addr.getHostString(), is("0.0.0.0")); + assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); + + // The bind host should be 0.0.0.0 unless OZONE_SCM_CLIENT_BIND_HOST_KEY + // is set differently. The port number from OZONE_SCM_CLIENT_ADDRESS_KEY + // should be respected. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); + addr = OzoneClientUtils.getScmClientBindAddress(conf); + assertThat(addr.getHostString(), is("0.0.0.0")); + assertThat(addr.getPort(), is(100)); + + // OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected. + // Port number should be default if none is specified via + // OZONE_SCM_DATANODE_ADDRESS_KEY. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4"); + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8"); + addr = OzoneClientUtils.getScmClientBindAddress(conf); + assertThat(addr.getHostString(), is("5.6.7.8")); + assertThat(addr.getPort(), is( + ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT)); + + // OZONE_SCM_CLIENT_BIND_HOST_KEY should be respected. + // Port number from OZONE_SCM_CLIENT_ADDRESS_KEY should be + // respected. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY, "5.6.7.8"); + addr = OzoneClientUtils.getScmClientBindAddress(conf); + assertThat(addr.getHostString(), is("5.6.7.8")); + assertThat(addr.getPort(), is(100)); + } + + /** + * Verify that the DataNode endpoint bind address is computed correctly. + * This tests the logic used by the SCM to determine its own bind address. + */ + @Test + public void testScmDataNodeBindHostDefault() { + final Configuration conf = new OzoneConfiguration(); + + // The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY + // is set differently. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); + assertThat(addr.getHostString(), is("0.0.0.0")); + assertThat(addr.getPort(), is( + ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + + // The bind host should be 0.0.0.0 unless OZONE_SCM_DATANODE_BIND_HOST_KEY + // is set differently. The port number from OZONE_SCM_DATANODE_ADDRESS_KEY + // should be respected. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); + addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); + assertThat(addr.getHostString(), is("0.0.0.0")); + assertThat(addr.getPort(), is(200)); + + // OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected. + // Port number should be default if none is specified via + // OZONE_SCM_DATANODE_ADDRESS_KEY. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8"); + addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); + assertThat(addr.getHostString(), is("5.6.7.8")); + assertThat(addr.getPort(), is( + ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT)); + + // OZONE_SCM_DATANODE_BIND_HOST_KEY should be respected. + // Port number from OZONE_SCM_DATANODE_ADDRESS_KEY should be + // respected. + conf.set(ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY, "1.2.3.4:100"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY, "1.2.3.4:200"); + conf.set(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY, "5.6.7.8"); + addr = OzoneClientUtils.getScmDataNodeBindAddress(conf); + assertThat(addr.getHostString(), is("5.6.7.8")); + assertThat(addr.getPort(), is(200)); + } + + @Test + public void testGetSCMAddresses() { + final Configuration conf = new OzoneConfiguration(); + Collection<InetSocketAddress> addresses = null; + InetSocketAddress addr = null; + Iterator<InetSocketAddress> it = null; + + // Verify valid IP address setup + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "1.2.3.4"); + addresses = OzoneClientUtils.getSCMAddresses(conf); + assertThat(addresses.size(), is(1)); + addr = addresses.iterator().next(); + assertThat(addr.getHostName(), is("1.2.3.4")); + assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT)); + + // Verify valid hostname setup + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1"); + addresses = OzoneClientUtils.getSCMAddresses(conf); + assertThat(addresses.size(), is(1)); + addr = addresses.iterator().next(); + assertThat(addr.getHostName(), is("scm1")); + assertThat(addr.getPort(), is(ScmConfigKeys.OZONE_SCM_DEFAULT_PORT)); + + // Verify valid hostname and port + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234"); + addresses = OzoneClientUtils.getSCMAddresses(conf); + assertThat(addresses.size(), is(1)); + addr = addresses.iterator().next(); + assertThat(addr.getHostName(), is("scm1")); + assertThat(addr.getPort(), is(1234)); + + final HashMap<String, Integer> hostsAndPorts = + new HashMap<String, Integer>(); + hostsAndPorts.put("scm1", 1234); + hostsAndPorts.put("scm2", 2345); + hostsAndPorts.put("scm3", 3456); + + // Verify multiple hosts and port + conf.setStrings( + ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234,scm2:2345,scm3:3456"); + addresses = OzoneClientUtils.getSCMAddresses(conf); + assertThat(addresses.size(), is(3)); + it = addresses.iterator(); + HashMap<String, Integer> expected1 = new HashMap<>(hostsAndPorts); + while(it.hasNext()) { + InetSocketAddress current = it.next(); + assertTrue(expected1.remove(current.getHostName(), + current.getPort())); + } + assertTrue(expected1.isEmpty()); + + // Verify names with spaces + conf.setStrings( + ScmConfigKeys.OZONE_SCM_NAMES, " scm1:1234, scm2:2345 , scm3:3456 "); + addresses = OzoneClientUtils.getSCMAddresses(conf); + assertThat(addresses.size(), is(3)); + it = addresses.iterator(); + HashMap<String, Integer> expected2 = new HashMap<>(hostsAndPorts); + while(it.hasNext()) { + InetSocketAddress current = it.next(); + assertTrue(expected2.remove(current.getHostName(), + current.getPort())); + } + assertTrue(expected2.isEmpty()); + + // Verify empty value + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, ""); + try { + addresses = OzoneClientUtils.getSCMAddresses(conf); + fail("Empty value should cause an IllegalArgumentException"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + + // Verify invalid hostname + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "s..x..:1234"); + try { + addresses = OzoneClientUtils.getSCMAddresses(conf); + fail("An invalid hostname should cause an IllegalArgumentException"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + + // Verify invalid port + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm:xyz"); + try { + addresses = OzoneClientUtils.getSCMAddresses(conf); + fail("An invalid port should cause an IllegalArgumentException"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + + // Verify a mixed case (valid and invalid value both appears) + conf.setStrings(ScmConfigKeys.OZONE_SCM_NAMES, "scm1:1234, scm:xyz"); + try { + addresses = OzoneClientUtils.getSCMAddresses(conf); + fail("An invalid value should cause an IllegalArgumentException"); + } catch (Exception e) { + assertTrue(e instanceof IllegalArgumentException); + } + } + + @Test + public void testGetKSMAddress() { + final Configuration conf = new OzoneConfiguration(); + + // First try a client address with just a host name. Verify it falls + // back to the default port. + conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4"); + InetSocketAddress addr = OzoneClientUtils.getKsmAddress(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); + + // Next try a client address with just a host name and port. Verify the port + // is ignored and the default KSM port is used. + conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, "1.2.3.4:100"); + addr = OzoneClientUtils.getKsmAddress(conf); + assertThat(addr.getHostString(), is("1.2.3.4")); + assertThat(addr.getPort(), is(100)); + + // Assert the we are able to use default configs if no value is specified. + conf.set(KSMConfigKeys.OZONE_KSM_ADDRESS_KEY, ""); + addr = OzoneClientUtils.getKsmAddress(conf); + assertThat(addr.getHostString(), is("0.0.0.0")); + assertThat(addr.getPort(), is(KSMConfigKeys.OZONE_KSM_PORT_DEFAULT)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/43d38114/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/package-info.java new file mode 100644 index 0000000..be63eab --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/ozone/client/package-info.java @@ -0,0 +1,23 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.client; + +/** + * This package contains test classes for Ozone Client. + */ \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org