HDDS-516. Implement CopyObject REST endpoint. Contributed by Bharat Viswanadham.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/021caaa5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/021caaa5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/021caaa5 Branch: refs/heads/HDFS-13532 Commit: 021caaa55e3f4315f927adb130fe95abcfe66744 Parents: c16c49b Author: Bharat Viswanadham <bha...@apache.org> Authored: Wed Oct 24 15:53:31 2018 -0700 Committer: Bharat Viswanadham <bha...@apache.org> Committed: Wed Oct 24 15:53:31 2018 -0700 ---------------------------------------------------------------------- .../dist/src/main/smoketest/s3/objectcopy.robot | 66 +++++++++++ .../ozone/s3/endpoint/CopyObjectResponse.java | 63 ++++++++++ .../ozone/s3/endpoint/ObjectEndpoint.java | 112 ++++++++++++++++-- .../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 + .../hadoop/ozone/s3/endpoint/TestPutObject.java | 114 ++++++++++++++++++- 5 files changed, 348 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/021caaa5/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot new file mode 100644 index 0000000..2daa861 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/objectcopy.robot @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation S3 gateway test with aws cli +Library OperatingSystem +Library String +Resource ../commonlib.robot +Resource commonawslib.robot +Test Setup Setup s3 tests + +*** Variables *** +${ENDPOINT_URL} http://s3g:9878 +${BUCKET} generated +${DESTBUCKET} generated1 + + +*** Keywords *** +Create Dest Bucket + + ${postfix} = Generate Random String 5 [NUMBERS] + Set Suite Variable ${DESTBUCKET} destbucket-${postfix} + Execute AWSS3APICli create-bucket --bucket ${DESTBUCKET} + + +*** Test Cases *** +Copy Object Happy Scenario + Run Keyword if '${DESTBUCKET}' == 'generated1' Create Dest Bucket + Execute date > /tmp/copyfile + ${result} = Execute AWSS3ApiCli put-object --bucket ${BUCKET} --key copyobject/f1 --body /tmp/copyfile + ${result} = Execute AWSS3ApiCli list-objects --bucket ${BUCKET} --prefix copyobject/ + Should contain ${result} f1 + + ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 + ${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/ + Should contain ${result} f1 + #copying again will not throw error + ${result} = Execute AWSS3ApiCli copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 + ${result} = Execute AWSS3ApiCli list-objects --bucket ${DESTBUCKET} --prefix copyobject/ + Should contain ${result} f1 + +Copy Object Where Bucket is not available + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket dfdfdfdfdfnonexistent --key copyobject/f1 --copy-source ${BUCKET}/copyobject/f1 255 + Should contain ${result} NoSuchBucket + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source dfdfdfdfdfnonexistent/copyobject/f1 255 + Should contain ${result} NoSuchBucket + +Copy Object Where both source and dest are same + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${DESTBUCKET}/copyobject/f1 255 + Should contain ${result} InvalidRequest + +Copy Object Where Key not available + ${result} = Execute AWSS3APICli and checkrc copy-object --bucket ${DESTBUCKET} --key copyobject/f1 --copy-source ${BUCKET}/nonnonexistentkey 255 + Should contain ${result} NoSuchKey http://git-wip-us.apache.org/repos/asf/hadoop/blob/021caaa5/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyObjectResponse.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyObjectResponse.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyObjectResponse.java new file mode 100644 index 0000000..f090791 --- /dev/null +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/CopyObjectResponse.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.s3.endpoint; + +import org.apache.hadoop.ozone.s3.commontypes.IsoDateAdapter; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter; +import java.time.Instant; + +/** + * Copy object Response. + */ +@XmlAccessorType(XmlAccessType.FIELD) +@XmlRootElement(name = "ListAllMyBucketsResult", + namespace = "http://s3.amazonaws.com/doc/2006-03-01/";) +public class CopyObjectResponse { + + @XmlJavaTypeAdapter(IsoDateAdapter.class) + @XmlElement(name = "LastModified") + private Instant lastModified; + + @XmlElement(name = "ETag") + private String eTag; + + + public Instant getLastModified() { + return lastModified; + } + + public void setLastModified(Instant lastModified) { + this.lastModified = lastModified; + } + + public String getETag() { + return eTag; + } + + public void setETag(String tag) { + this.eTag = tag; + } + + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/021caaa5/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index c622938..9782d75 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -41,11 +41,8 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hdds.client.ReplicationFactor; import org.apache.hadoop.hdds.client.ReplicationType; -import org.apache.hadoop.hdds.conf.OzoneConfiguration; -import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneKeyDetails; import org.apache.hadoop.ozone.client.io.OzoneInputStream; @@ -56,6 +53,7 @@ import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.io.IOUtils; +import org.apache.hadoop.ozone.web.utils.OzoneUtils; import org.apache.http.HttpStatus; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -97,18 +95,27 @@ public class ObjectEndpoint extends EndpointBase { ReplicationType replicationType, @DefaultValue("ONE") @QueryParam("replicationFactor") ReplicationFactor replicationFactor, - @DefaultValue("32 * 1024 * 1024") @QueryParam("chunkSize") - String chunkSize, @HeaderParam("Content-Length") long length, InputStream body) throws IOException, OS3Exception { + OzoneOutputStream output = null; try { - Configuration config = new OzoneConfiguration(); - config.set(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY, chunkSize); + String copyHeader = headers.getHeaderString("x-amz-copy-source"); + if (copyHeader != null) { + //Copy object, as copy source available. + CopyObjectResponse copyObjectResponse = copyObject( + copyHeader, bucketName, keyPath, replicationType, + replicationFactor); + return Response.status(Status.OK).entity(copyObjectResponse).header( + "Connection", "close").build(); + } + + // Normal put object OzoneBucket bucket = getBucket(bucketName); - OzoneOutputStream output = bucket - .createKey(keyPath, length, replicationType, replicationFactor); + + output = bucket.createKey(keyPath, length, replicationType, + replicationFactor); if ("STREAMING-AWS4-HMAC-SHA256-PAYLOAD" .equals(headers.getHeaderString("x-amz-content-sha256"))) { @@ -116,13 +123,16 @@ public class ObjectEndpoint extends EndpointBase { } IOUtils.copy(body, output); - output.close(); return Response.ok().status(HttpStatus.SC_OK) .build(); } catch (IOException ex) { LOG.error("Exception occurred in PutObject", ex); throw ex; + } finally { + if (output != null) { + output.close(); + } } } @@ -239,4 +249,86 @@ public class ObjectEndpoint extends EndpointBase { public void setHeaders(HttpHeaders headers) { this.headers = headers; } + + private CopyObjectResponse copyObject(String copyHeader, + String destBucket, + String destkey, + ReplicationType replicationType, + ReplicationFactor replicationFactor) + throws OS3Exception, IOException { + + if (copyHeader.startsWith("/")) { + copyHeader = copyHeader.substring(1); + } + int pos = copyHeader.indexOf("/"); + if (pos == -1) { + OS3Exception ex = S3ErrorTable.newError(S3ErrorTable + .INVALID_ARGUMENT, copyHeader); + ex.setErrorMessage("Copy Source must mention the source bucket and " + + "key: sourcebucket/sourcekey"); + throw ex; + } + String sourceBucket = copyHeader.substring(0, pos); + String sourceKey = copyHeader.substring(pos + 1); + + OzoneInputStream sourceInputStream = null; + OzoneOutputStream destOutputStream = null; + boolean closed = false; + try { + // Checking whether we trying to copying to it self. + if (sourceBucket.equals(destBucket)) { + if (sourceKey.equals(destkey)) { + OS3Exception ex = S3ErrorTable.newError(S3ErrorTable + .INVALID_REQUEST, copyHeader); + ex.setErrorMessage("This copy request is illegal because it is " + + "trying to copy an object to it self itself without changing " + + "the object's metadata, storage class, website redirect " + + "location or encryption attributes."); + throw ex; + } + } + + OzoneBucket sourceOzoneBucket = getBucket(sourceBucket); + OzoneBucket destOzoneBucket = getBucket(destBucket); + + OzoneKeyDetails sourceKeyDetails = sourceOzoneBucket.getKey(sourceKey); + long sourceKeyLen = sourceKeyDetails.getDataSize(); + + sourceInputStream = sourceOzoneBucket.readKey(sourceKey); + + destOutputStream = destOzoneBucket.createKey(destkey, sourceKeyLen, + replicationType, replicationFactor); + + IOUtils.copy(sourceInputStream, destOutputStream); + + // Closing here, as if we don't call close this key will not commit in + // OM, and getKey fails. + sourceInputStream.close(); + destOutputStream.close(); + closed = true; + + OzoneKeyDetails destKeyDetails = destOzoneBucket.getKey(destkey); + + CopyObjectResponse copyObjectResponse = new CopyObjectResponse(); + copyObjectResponse.setETag(OzoneUtils.getRequestID()); + copyObjectResponse.setLastModified(Instant.ofEpochMilli(destKeyDetails + .getModificationTime())); + return copyObjectResponse; + } catch (IOException ex) { + if (ex.getMessage().contains("KEY_NOT_FOUND")) { + throw S3ErrorTable.newError(S3ErrorTable.NO_SUCH_KEY, sourceKey); + } + LOG.error("Exception occurred in PutObject", ex); + throw ex; + } finally { + if (!closed) { + if (sourceInputStream != null) { + sourceInputStream.close(); + } + if (destOutputStream != null) { + destOutputStream.close(); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/021caaa5/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java index b5c6c72..78ec9db 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/exception/S3ErrorTable.java @@ -52,6 +52,12 @@ public final class S3ErrorTable { public static final OS3Exception NO_SUCH_KEY = new OS3Exception( "NoSuchKey", "The specified key does not exist", HTTP_NOT_FOUND); + public static final OS3Exception INVALID_ARGUMENT = new OS3Exception( + "InvalidArgument", "Invalid Argument", HTTP_BAD_REQUEST); + + public static final OS3Exception INVALID_REQUEST = new OS3Exception( + "InvalidRequest", "Invalid Request", HTTP_BAD_REQUEST); + /** * Create a new instance of Error. * @param e Error Template http://git-wip-us.apache.org/repos/asf/hadoop/blob/021caaa5/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java ---------------------------------------------------------------------- diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java index 03b9a0f..4f94e56 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPutObject.java @@ -38,6 +38,8 @@ import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; + +import static org.junit.Assert.fail; import static org.mockito.Mockito.when; /** @@ -48,6 +50,9 @@ public class TestPutObject { private String userName = "ozone"; private String bucketName = "b1"; private String keyName = "key1"; + private String destBucket = "b2"; + private String destkey = "key2"; + private String nonexist = "nonexist"; private OzoneClientStub clientStub; private ObjectStore objectStoreStub; private ObjectEndpoint objectEndpoint; @@ -60,6 +65,7 @@ public class TestPutObject { // Create bucket objectStoreStub.createS3Bucket(userName, bucketName); + objectStoreStub.createS3Bucket("ozone1", destBucket); // Create PutObject and setClient to OzoneClientStub objectEndpoint = new ObjectEndpoint(); @@ -75,8 +81,8 @@ public class TestPutObject { //WHEN Response response = objectEndpoint.put(bucketName, keyName, - ReplicationType.STAND_ALONE, ReplicationFactor.ONE, "32 * 1024 * 1024", - CONTENT.length(), body); + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); //THEN String volumeName = clientStub.getObjectStore() @@ -109,7 +115,6 @@ public class TestPutObject { Response response = objectEndpoint.put(bucketName, keyName, ReplicationType.STAND_ALONE, ReplicationFactor.ONE, - "32 * 1024 * 1024", chunkedContent.length(), new ByteArrayInputStream(chunkedContent.getBytes())); @@ -125,4 +130,107 @@ public class TestPutObject { Assert.assertEquals(200, response.getStatus()); Assert.assertEquals("1234567890abcde", keyContent); } + + @Test + public void testCopyObject() throws IOException, OS3Exception { + // Put object in to source bucket + HttpHeaders headers = Mockito.mock(HttpHeaders.class); + ByteArrayInputStream body = new ByteArrayInputStream(CONTENT.getBytes()); + objectEndpoint.setHeaders(headers); + keyName = "sourceKey"; + + Response response = objectEndpoint.put(bucketName, keyName, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); + + String volumeName = clientStub.getObjectStore().getOzoneVolumeName( + bucketName); + + OzoneInputStream ozoneInputStream = clientStub.getObjectStore().getVolume( + volumeName).getBucket(bucketName).readKey(keyName); + + String keyContent = IOUtils.toString(ozoneInputStream, Charset.forName( + "UTF-8")); + + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals(CONTENT, keyContent); + + + // Add copy header, and then call put + when(headers.getHeaderString("x-amz-copy-source")).thenReturn( + bucketName + "/" + keyName); + + response = objectEndpoint.put(destBucket, destkey, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); + + // Check destination key and response + volumeName = clientStub.getObjectStore().getOzoneVolumeName(destBucket); + ozoneInputStream = clientStub.getObjectStore().getVolume(volumeName) + .getBucket(destBucket).readKey(destkey); + + keyContent = IOUtils.toString(ozoneInputStream, Charset.forName("UTF-8")); + + Assert.assertEquals(200, response.getStatus()); + Assert.assertEquals(CONTENT, keyContent); + + // source and dest same + try { + objectEndpoint.put(bucketName, keyName, ReplicationType.STAND_ALONE, + ReplicationFactor.ONE, CONTENT.length(), body); + fail("test copy object failed"); + } catch (OS3Exception ex) { + Assert.assertTrue(ex.getErrorMessage().contains("This copy request is " + + "illegal")); + } + + // source bucket not found + try { + when(headers.getHeaderString("x-amz-copy-source")).thenReturn( + nonexist + "/" + keyName); + response = objectEndpoint.put(destBucket, destkey, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); + fail("test copy object failed"); + } catch (OS3Exception ex) { + Assert.assertTrue(ex.getCode().contains("NoSuchBucket")); + } + + // dest bucket not found + try { + when(headers.getHeaderString("x-amz-copy-source")).thenReturn( + bucketName + "/" + keyName); + response = objectEndpoint.put(nonexist, destkey, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); + fail("test copy object failed"); + } catch (OS3Exception ex) { + Assert.assertTrue(ex.getCode().contains("NoSuchBucket")); + } + + //Both source and dest bucket not found + try { + when(headers.getHeaderString("x-amz-copy-source")).thenReturn( + nonexist + "/" + keyName); + response = objectEndpoint.put(nonexist, destkey, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); + fail("test copy object failed"); + } catch (OS3Exception ex) { + Assert.assertTrue(ex.getCode().contains("NoSuchBucket")); + } + + // source key not found + try { + when(headers.getHeaderString("x-amz-copy-source")).thenReturn( + bucketName + "/" + nonexist); + response = objectEndpoint.put("nonexistent", keyName, + ReplicationType.STAND_ALONE, ReplicationFactor.ONE, CONTENT.length(), + body); + fail("test copy object failed"); + } catch (OS3Exception ex) { + Assert.assertTrue(ex.getCode().contains("NoSuchBucket")); + } + + } } \ No newline at end of file --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
