This is an automated email from the ASF dual-hosted git repository. joewitt pushed a commit to branch NIFI-6169-RC2 in repository https://gitbox.apache.org/repos/asf/nifi.git
commit 45ed3660538b30d470c7df6b066aba213050e7d8 Author: Peter Turcsanyi <[email protected]> AuthorDate: Thu Mar 7 23:18:20 2019 +0100 NIFI-6122: PutS3Object supports national characters in filenames. This closes #3373. Signed-off-by: Koji Kawamura <[email protected]> --- .../apache/nifi/processors/aws/s3/PutS3Object.java | 3 +- .../nifi/processors/aws/s3/ITPutS3Object.java | 17 +++++ .../nifi/processors/aws/s3/TestPutS3Object.java | 76 +++++++++++++--------- 3 files changed, 65 insertions(+), 31 deletions(-) diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java index dfbcbda..d331d15 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/main/java/org/apache/nifi/processors/aws/s3/PutS3Object.java @@ -23,6 +23,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.Serializable; +import java.net.URLEncoder; import java.nio.file.Files; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -445,7 +446,7 @@ public class PutS3Object extends AbstractS3Processor { public void process(final InputStream rawIn) throws IOException { try (final InputStream in = new BufferedInputStream(rawIn)) { final ObjectMetadata objectMetadata = new ObjectMetadata(); - objectMetadata.setContentDisposition(ff.getAttribute(CoreAttributes.FILENAME.key())); + objectMetadata.setContentDisposition(URLEncoder.encode(ff.getAttribute(CoreAttributes.FILENAME.key()), "UTF-8")); objectMetadata.setContentLength(ff.getSize()); final String contentType = context.getProperty(CONTENT_TYPE) diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITPutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITPutS3Object.java index 2db139a..5cb1a97 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITPutS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/ITPutS3Object.java @@ -119,6 +119,23 @@ public class ITPutS3Object extends AbstractS3IT { } } + @Test + public void testSimplePutFilenameWithNationalCharacters() throws IOException { + final TestRunner runner = TestRunners.newTestRunner(new PutS3Object()); + + runner.setProperty(PutS3Object.CREDENTIALS_FILE, CREDENTIALS_FILE); + runner.setProperty(PutS3Object.REGION, REGION); + runner.setProperty(PutS3Object.BUCKET, BUCKET_NAME); + + final Map<String, String> attrs = new HashMap<>(); + attrs.put("filename", "Iñtërnâtiônàližætiøn.txt"); + runner.enqueue(getResourcePath(SAMPLE_FILE_RESOURCE_NAME), attrs); + + runner.run(1); + + runner.assertAllFlowFilesTransferred(PutS3Object.REL_SUCCESS, 1); + } + private void testPutThenFetch(String sseAlgorithm) throws IOException { // Put diff --git a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java index 2c3db81..67ebae7 100644 --- a/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java +++ b/nifi-nar-bundles/nifi-aws-bundle/nifi-aws-processors/src/test/java/org/apache/nifi/processors/aws/s3/TestPutS3Object.java @@ -16,6 +16,8 @@ */ package org.apache.nifi.processors.aws.s3; +import java.io.UnsupportedEncodingException; +import java.net.URLDecoder; import java.util.Date; import java.util.HashMap; import java.util.List; @@ -53,27 +55,28 @@ import static org.junit.Assert.assertTrue; public class TestPutS3Object { - private TestRunner runner = null; - private PutS3Object mockPutS3Object = null; - private AmazonS3Client actualS3Client = null; - private AmazonS3Client mockS3Client = null; + + private TestRunner runner; + private PutS3Object putS3Object; + private AmazonS3Client mockS3Client; @Before public void setUp() { mockS3Client = Mockito.mock(AmazonS3Client.class); - mockPutS3Object = new PutS3Object() { + putS3Object = new PutS3Object() { protected AmazonS3Client getClient() { - actualS3Client = client; return mockS3Client; } }; - runner = TestRunners.newTestRunner(mockPutS3Object); + runner = TestRunners.newTestRunner(putS3Object); } @Test public void testPutSinglePart() { runner.setProperty("x-custom-prop", "hello"); - testBase(); + prepareTest(); + + runner.run(1); ArgumentCaptor<PutObjectRequest> captureRequest = ArgumentCaptor.forClass(PutObjectRequest.class); Mockito.verify(mockS3Client, Mockito.times(1)).putObject(captureRequest.capture()); @@ -92,17 +95,10 @@ public class TestPutS3Object { @Test public void testPutSinglePartException() { - runner.setProperty(PutS3Object.REGION, "ap-northeast-1"); - runner.setProperty(PutS3Object.BUCKET, "test-bucket"); - final Map<String, String> ffAttributes = new HashMap<>(); - ffAttributes.put("filename", "testfile.txt"); - runner.enqueue("Test Content", ffAttributes); + prepareTest(); - MultipartUploadListing uploadListing = new MultipartUploadListing(); - Mockito.when(mockS3Client.listMultipartUploads(Mockito.any(ListMultipartUploadsRequest.class))).thenReturn(uploadListing); Mockito.when(mockS3Client.putObject(Mockito.any(PutObjectRequest.class))).thenThrow(new AmazonS3Exception("TestFail")); - runner.assertValid(); runner.run(1); runner.assertAllFlowFilesTransferred(PutS3Object.REL_FAILURE, 1); @@ -124,7 +120,7 @@ public class TestPutS3Object { runner.setProperty(PutS3Object.SIGNER_OVERRIDE, signerType); ProcessContext context = runner.getProcessContext(); try { - AmazonS3Client s3Client = processor.createClient(context, credentialsProvider, config); + processor.createClient(context, credentialsProvider, config); } catch (IllegalArgumentException argEx) { Assert.fail(argEx.getMessage()); } @@ -136,7 +132,9 @@ public class TestPutS3Object { public void testObjectTags() { runner.setProperty(PutS3Object.OBJECT_TAGS_PREFIX, "tagS3"); runner.setProperty(PutS3Object.REMOVE_TAG_PREFIX, "false"); - testBase(); + prepareTest(); + + runner.run(1); ArgumentCaptor<PutObjectRequest> captureRequest = ArgumentCaptor.forClass(PutObjectRequest.class); Mockito.verify(mockS3Client, Mockito.times(1)).putObject(captureRequest.capture()); @@ -147,37 +145,50 @@ public class TestPutS3Object { assertEquals(1, tagSet.size()); assertEquals("tagS3PII", tagSet.get(0).getKey()); assertEquals("true", tagSet.get(0).getValue()); + } + + @Test + public void testFilenameWithNationalCharacters() throws UnsupportedEncodingException { + prepareTest("Iñtërnâtiônàližætiøn.txt"); + runner.run(1); + + ArgumentCaptor<PutObjectRequest> captureRequest = ArgumentCaptor.forClass(PutObjectRequest.class); + Mockito.verify(mockS3Client, Mockito.times(1)).putObject(captureRequest.capture()); + PutObjectRequest request = captureRequest.getValue(); + + ObjectMetadata objectMetadata = request.getMetadata(); + assertEquals("Iñtërnâtiônàližætiøn.txt", URLDecoder.decode(objectMetadata.getContentDisposition(), "UTF-8")); } - private void testBase() { + private void prepareTest() { + prepareTest("testfile.txt"); + } + + private void prepareTest(String filename) { runner.setProperty(PutS3Object.REGION, "ap-northeast-1"); runner.setProperty(PutS3Object.BUCKET, "test-bucket"); + runner.assertValid(); - final Map<String, String> ffAttributes = new HashMap<>(); - ffAttributes.put("filename", "testfile.txt"); + Map<String, String> ffAttributes = new HashMap<>(); + ffAttributes.put("filename", filename); ffAttributes.put("tagS3PII", "true"); runner.enqueue("Test Content", ffAttributes); - PutObjectResult putObjectResult = Mockito.spy(PutObjectResult.class); - Date expiration = new Date(); - - putObjectResult.setExpirationTime(expiration); + PutObjectResult putObjectResult = new PutObjectResult(); + putObjectResult.setExpirationTime(new Date()); putObjectResult.setMetadata(new ObjectMetadata()); putObjectResult.setVersionId("test-version"); + putObjectResult.setETag("test-etag"); - Mockito.when(putObjectResult.getETag()).thenReturn("test-etag"); Mockito.when(mockS3Client.putObject(Mockito.any(PutObjectRequest.class))).thenReturn(putObjectResult); MultipartUploadListing uploadListing = new MultipartUploadListing(); Mockito.when(mockS3Client.listMultipartUploads(Mockito.any(ListMultipartUploadsRequest.class))).thenReturn(uploadListing); - - runner.assertValid(); - runner.run(1); } @Test - public void testGetPropertyDescriptors() throws Exception { + public void testGetPropertyDescriptors() { PutS3Object processor = new PutS3Object(); List<PropertyDescriptor> pd = processor.getSupportedPropertyDescriptors(); assertEquals("size should be eq", 33, pd.size()); @@ -209,6 +220,11 @@ public class TestPutS3Object { assertTrue(pd.contains(PutS3Object.PROXY_PASSWORD)); assertTrue(pd.contains(PutS3Object.OBJECT_TAGS_PREFIX)); assertTrue(pd.contains(PutS3Object.REMOVE_TAG_PREFIX)); + assertTrue(pd.contains(PutS3Object.CONTENT_TYPE)); + assertTrue(pd.contains(PutS3Object.MULTIPART_THRESHOLD)); + assertTrue(pd.contains(PutS3Object.MULTIPART_PART_SIZE)); + assertTrue(pd.contains(PutS3Object.MULTIPART_S3_AGEOFF_INTERVAL)); + assertTrue(pd.contains(PutS3Object.MULTIPART_S3_MAX_AGE)); } }
