This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch s3-zero in repository https://gitbox.apache.org/repos/asf/camel.git
commit 5868a8726e6b2bc23a90eb02f2739531d9553647 Author: Claus Ibsen <[email protected]> AuthorDate: Mon Oct 23 11:19:08 2023 +0200 CAMEL-19707: camel-aws-s3 - Upload small / zero length files in single operation instead of multi-part --- .../apache/camel/catalog/components/aws2-s3.json | 8 +++---- .../apache/camel/component/aws2/s3/aws2-s3.json | 8 +++---- .../component/aws2/s3/AWS2S3Configuration.java | 11 ++++++--- .../camel/component/aws2/s3/AWS2S3Producer.java | 18 ++++++++++----- .../integration/S3MultipartUploadOperationIT.java | 20 +++++++++++++++++ .../dsl/Aws2S3ComponentBuilderFactory.java | 13 +++++++---- .../endpoint/dsl/AWS2S3EndpointBuilderFactory.java | 26 +++++++++++++++------- 7 files changed, 76 insertions(+), 28 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json index 39ff469be5a..05398c8d8d7 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/aws2-s3.json @@ -53,10 +53,10 @@ "deleteAfterWrite": { "index": 28, "kind": "property", "displayName": "Delete After Write", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Delete file object after the S3 file has been uploaded" }, "keyName": { "index": 29, "kind": "property", "displayName": "Key Name", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setting the key name for an element in the bucket through endpoint parameter" }, "lazyStartProducer": { "index": 30, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fai [...] - "multiPartUpload": { "index": 31, "kind": "property", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi part format, the part size [...] + "multiPartUpload": { "index": 31, "kind": "property", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi-part format, the part size [...] "namingStrategy": { "index": 32, "kind": "property", "displayName": "Naming Strategy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3NamingStrategyEnum", "enum": [ "progressive", "random" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "progressive", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuratio [...] "operation": { "index": 33, "kind": "property", "displayName": "Operation", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.AWS2S3Operations", "enum": [ "copyObject", "listObjects", "deleteObject", "deleteBucket", "listBuckets", "getObject", "getObjectRange", "createDownloadLink" ], "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Con [...] - "partSize": { "index": 34, "kind": "property", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi part upload, the default size is 25M." }, + "partSize": { "index": 34, "kind": "property", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi-part upload, the default size is 25M. Camel will onl [...] "restartingPolicy": { "index": 35, "kind": "property", "displayName": "Restarting Policy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3RestartingPolicyEnum", "enum": [ "override", "lastPart" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "override", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configurat [...] "storageClass": { "index": 36, "kind": "property", "displayName": "Storage Class", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "The storage class to set in the com.amazonaws.services.s3.model.PutObjectRequest request." }, "streamingUploadMode": { "index": 37, "kind": "property", "displayName": "Streaming Upload Mode", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "When stream mode is true the upload to bucket will be done in streaming" }, @@ -149,10 +149,10 @@ "bufferSize": { "index": 33, "kind": "parameter", "displayName": "Buffer Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 1000000, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "The buffer size (in bytes) in streaming upload mode" }, "deleteAfterWrite": { "index": 34, "kind": "parameter", "displayName": "Delete After Write", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Delete file object after the S3 file has been uploaded" }, "keyName": { "index": 35, "kind": "parameter", "displayName": "Key Name", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setting the key name for an element in the bucket through endpoint parameter" }, - "multiPartUpload": { "index": 36, "kind": "parameter", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi part format, the part siz [...] + "multiPartUpload": { "index": 36, "kind": "parameter", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi-part format, the part siz [...] "namingStrategy": { "index": 37, "kind": "parameter", "displayName": "Naming Strategy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3NamingStrategyEnum", "enum": [ "progressive", "random" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "progressive", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configurati [...] "operation": { "index": 38, "kind": "parameter", "displayName": "Operation", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.AWS2S3Operations", "enum": [ "copyObject", "listObjects", "deleteObject", "deleteBucket", "listBuckets", "getObject", "getObjectRange", "createDownloadLink" ], "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Co [...] - "partSize": { "index": 39, "kind": "parameter", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi part upload, the default size is 25M." }, + "partSize": { "index": 39, "kind": "parameter", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi-part upload, the default size is 25M. Camel will on [...] "restartingPolicy": { "index": 40, "kind": "parameter", "displayName": "Restarting Policy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3RestartingPolicyEnum", "enum": [ "override", "lastPart" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "override", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configura [...] "storageClass": { "index": 41, "kind": "parameter", "displayName": "Storage Class", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "The storage class to set in the com.amazonaws.services.s3.model.PutObjectRequest request." }, "streamingUploadMode": { "index": 42, "kind": "parameter", "displayName": "Streaming Upload Mode", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "When stream mode is true the upload to bucket will be done in streaming" }, diff --git a/components/camel-aws/camel-aws2-s3/src/generated/resources/org/apache/camel/component/aws2/s3/aws2-s3.json b/components/camel-aws/camel-aws2-s3/src/generated/resources/org/apache/camel/component/aws2/s3/aws2-s3.json index 39ff469be5a..05398c8d8d7 100644 --- a/components/camel-aws/camel-aws2-s3/src/generated/resources/org/apache/camel/component/aws2/s3/aws2-s3.json +++ b/components/camel-aws/camel-aws2-s3/src/generated/resources/org/apache/camel/component/aws2/s3/aws2-s3.json @@ -53,10 +53,10 @@ "deleteAfterWrite": { "index": 28, "kind": "property", "displayName": "Delete After Write", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Delete file object after the S3 file has been uploaded" }, "keyName": { "index": 29, "kind": "property", "displayName": "Key Name", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setting the key name for an element in the bucket through endpoint parameter" }, "lazyStartProducer": { "index": 30, "kind": "property", "displayName": "Lazy Start Producer", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "description": "Whether the producer should be started lazy (on the first message). By starting lazy you can use this to allow CamelContext and routes to startup in situations where a producer may otherwise fai [...] - "multiPartUpload": { "index": 31, "kind": "property", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi part format, the part size [...] + "multiPartUpload": { "index": 31, "kind": "property", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi-part format, the part size [...] "namingStrategy": { "index": 32, "kind": "property", "displayName": "Naming Strategy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3NamingStrategyEnum", "enum": [ "progressive", "random" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "progressive", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuratio [...] "operation": { "index": 33, "kind": "property", "displayName": "Operation", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.AWS2S3Operations", "enum": [ "copyObject", "listObjects", "deleteObject", "deleteBucket", "listBuckets", "getObject", "getObjectRange", "createDownloadLink" ], "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Con [...] - "partSize": { "index": 34, "kind": "property", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi part upload, the default size is 25M." }, + "partSize": { "index": 34, "kind": "property", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi-part upload, the default size is 25M. Camel will onl [...] "restartingPolicy": { "index": 35, "kind": "property", "displayName": "Restarting Policy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3RestartingPolicyEnum", "enum": [ "override", "lastPart" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "override", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configurat [...] "storageClass": { "index": 36, "kind": "property", "displayName": "Storage Class", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "The storage class to set in the com.amazonaws.services.s3.model.PutObjectRequest request." }, "streamingUploadMode": { "index": 37, "kind": "property", "displayName": "Streaming Upload Mode", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "When stream mode is true the upload to bucket will be done in streaming" }, @@ -149,10 +149,10 @@ "bufferSize": { "index": 33, "kind": "parameter", "displayName": "Buffer Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "int", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 1000000, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "The buffer size (in bytes) in streaming upload mode" }, "deleteAfterWrite": { "index": 34, "kind": "parameter", "displayName": "Delete After Write", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Delete file object after the S3 file has been uploaded" }, "keyName": { "index": 35, "kind": "parameter", "displayName": "Key Name", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setting the key name for an element in the bucket through endpoint parameter" }, - "multiPartUpload": { "index": 36, "kind": "parameter", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi part format, the part siz [...] + "multiPartUpload": { "index": 36, "kind": "parameter", "displayName": "Multi Part Upload", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "If it is true, camel will upload the file with multi-part format, the part siz [...] "namingStrategy": { "index": 37, "kind": "parameter", "displayName": "Naming Strategy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3NamingStrategyEnum", "enum": [ "progressive", "random" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "progressive", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configurati [...] "operation": { "index": 38, "kind": "parameter", "displayName": "Operation", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.AWS2S3Operations", "enum": [ "copyObject", "listObjects", "deleteObject", "deleteBucket", "listBuckets", "getObject", "getObjectRange", "createDownloadLink" ], "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Co [...] - "partSize": { "index": 39, "kind": "parameter", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi part upload, the default size is 25M." }, + "partSize": { "index": 39, "kind": "parameter", "displayName": "Part Size", "group": "producer", "label": "producer", "required": false, "type": "integer", "javaType": "long", "deprecated": false, "autowired": false, "secret": false, "defaultValue": 26214400, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "Setup the partSize which is used in multi-part upload, the default size is 25M. Camel will on [...] "restartingPolicy": { "index": 40, "kind": "parameter", "displayName": "Restarting Policy", "group": "producer", "label": "producer", "required": false, "type": "object", "javaType": "org.apache.camel.component.aws2.s3.stream.AWSS3RestartingPolicyEnum", "enum": [ "override", "lastPart" ], "deprecated": false, "autowired": false, "secret": false, "defaultValue": "override", "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configura [...] "storageClass": { "index": 41, "kind": "parameter", "displayName": "Storage Class", "group": "producer", "label": "producer", "required": false, "type": "string", "javaType": "java.lang.String", "deprecated": false, "autowired": false, "secret": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "The storage class to set in the com.amazonaws.services.s3.model.PutObjectRequest request." }, "streamingUploadMode": { "index": 42, "kind": "parameter", "displayName": "Streaming Upload Mode", "group": "producer", "label": "producer", "required": false, "type": "boolean", "javaType": "boolean", "deprecated": false, "autowired": false, "secret": false, "defaultValue": false, "configurationClass": "org.apache.camel.component.aws2.s3.AWS2S3Configuration", "configurationField": "configuration", "description": "When stream mode is true the upload to bucket will be done in streaming" }, diff --git a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Configuration.java b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Configuration.java index 9e8bdb848b7..1a9a18bcc6f 100644 --- a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Configuration.java +++ b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Configuration.java @@ -142,7 +142,10 @@ public class AWS2S3Configuration implements Cloneable { } /** - * Setup the partSize which is used in multi part upload, the default size is 25M. + * Setup the partSize which is used in multi-part upload, the default size is 25M. + * + * Camel will only do multi-part uploads for files that are larger than the part-size thresholds. + * Files that are smaller will be uploaded in a single operation. */ public void setPartSize(long partSize) { this.partSize = partSize; @@ -153,8 +156,10 @@ public class AWS2S3Configuration implements Cloneable { } /** - * If it is true, camel will upload the file with multi part format, the part size is decided by the option of - * `partSize` + * If it is true, camel will upload the file with multi-part format, the part size is decided by the partSize option. + * + * Camel will only do multi-part uploads for files that are larger than the part-size thresholds. + * Files that are smaller will be uploaded in a single operation. */ public void setMultiPartUpload(boolean multiPartUpload) { this.multiPartUpload = multiPartUpload; diff --git a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java index b80fea74376..899c4cc81cb 100644 --- a/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java +++ b/components/camel-aws/camel-aws2-s3/src/main/java/org/apache/camel/component/aws2/s3/AWS2S3Producer.java @@ -129,7 +129,7 @@ public class AWS2S3Producer extends DefaultProducer { } public void processMultiPart(final Exchange exchange) throws Exception { - File filePayload = null; + File filePayload; Object obj = exchange.getIn().getMandatoryBody(); // Need to check if the message body is WrappedFile if (obj instanceof WrappedFile) { @@ -147,6 +147,17 @@ public class AWS2S3Producer extends DefaultProducer { if (contentLength == null || contentLength == 0) { contentLength = filePayload.length(); } + + long partSize = getConfiguration().getPartSize(); + if (contentLength == 0 && contentLength < partSize) { + // optimize to do a single op if content length is known and < part size + LOG.debug("File size < partSize. Uploading file in single operation: {}", filePayload); + processSingleOp(exchange); + return; + } + + LOG.debug("File size >= partSize. Uploading file using multi-part operation: {}", filePayload); + objectMetadata.put("Content-Length", contentLength.toString()); final String keyName = AWS2S3Utils.determineKey(exchange, getConfiguration()); @@ -198,13 +209,10 @@ public class AWS2S3Producer extends DefaultProducer { CreateMultipartUploadResponse initResponse = getEndpoint().getS3Client().createMultipartUpload(createMultipartUploadRequest.build()); - //final long contentLength = Long.parseLong(objectMetadata.get("Content-Length")); List<CompletedPart> completedParts = new ArrayList<CompletedPart>(); - long partSize = getConfiguration().getPartSize(); - CompleteMultipartUploadResponse uploadResult = null; + CompleteMultipartUploadResponse uploadResult; long filePosition = 0; - try { for (int part = 1; filePosition < contentLength; part++) { partSize = Math.min(partSize, contentLength - filePosition); diff --git a/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIT.java b/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIT.java index bdc64dc2a86..be6df3ead16 100644 --- a/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIT.java +++ b/components/camel-aws/camel-aws2-s3/src/test/java/org/apache/camel/component/aws2/s3/integration/S3MultipartUploadOperationIT.java @@ -26,6 +26,7 @@ import org.apache.camel.builder.RouteBuilder; import org.apache.camel.component.aws2.s3.AWS2S3Constants; import org.apache.camel.component.mock.MockEndpoint; import org.apache.camel.test.infra.aws2.clients.AWSSDKClientUtils; +import org.apache.camel.util.IOHelper; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.services.s3.S3Client; @@ -78,6 +79,25 @@ public class S3MultipartUploadOperationIT extends Aws2S3Base { assertEquals("application/text", response.response().contentType()); } + @Test + public void sendZeroLength() throws Exception { + result.expectedMessageCount(1); + + File zero = new File("target/zero.txt"); + IOHelper.writeText("", zero); + + template.send("direct:putObject", new Processor() { + + @Override + public void process(Exchange exchange) { + exchange.getIn().setHeader(AWS2S3Constants.KEY, "zero.txt"); + exchange.getIn().setBody(zero); + } + }); + + MockEndpoint.assertIsSatisfied(context); + } + @Override protected RouteBuilder createRouteBuilder() { return new RouteBuilder() { diff --git a/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Aws2S3ComponentBuilderFactory.java b/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Aws2S3ComponentBuilderFactory.java index 206b2bb2384..32dfd6b2d89 100644 --- a/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Aws2S3ComponentBuilderFactory.java +++ b/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Aws2S3ComponentBuilderFactory.java @@ -578,8 +578,11 @@ public interface Aws2S3ComponentBuilderFactory { return this; } /** - * If it is true, camel will upload the file with multi part format, the - * part size is decided by the option of partSize. + * If it is true, camel will upload the file with multi-part format, the + * part size is decided by the partSize option. Camel will only do + * multi-part uploads for files that are larger than the part-size + * thresholds. Files that are smaller will be uploaded in a single + * operation. * * The option is a: <code>boolean</code> type. * @@ -627,8 +630,10 @@ public interface Aws2S3ComponentBuilderFactory { return this; } /** - * Setup the partSize which is used in multi part upload, the default - * size is 25M. + * Setup the partSize which is used in multi-part upload, the default + * size is 25M. Camel will only do multi-part uploads for files that are + * larger than the part-size thresholds. Files that are smaller will be + * uploaded in a single operation. * * The option is a: <code>long</code> type. * diff --git a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/AWS2S3EndpointBuilderFactory.java b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/AWS2S3EndpointBuilderFactory.java index a046827c6c4..4e1561914c6 100644 --- a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/AWS2S3EndpointBuilderFactory.java +++ b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/AWS2S3EndpointBuilderFactory.java @@ -2052,8 +2052,11 @@ public interface AWS2S3EndpointBuilderFactory { return this; } /** - * If it is true, camel will upload the file with multi part format, the - * part size is decided by the option of partSize. + * If it is true, camel will upload the file with multi-part format, the + * part size is decided by the partSize option. Camel will only do + * multi-part uploads for files that are larger than the part-size + * thresholds. Files that are smaller will be uploaded in a single + * operation. * * The option is a: <code>boolean</code> type. * @@ -2069,8 +2072,11 @@ public interface AWS2S3EndpointBuilderFactory { return this; } /** - * If it is true, camel will upload the file with multi part format, the - * part size is decided by the option of partSize. + * If it is true, camel will upload the file with multi-part format, the + * part size is decided by the partSize option. Camel will only do + * multi-part uploads for files that are larger than the part-size + * thresholds. Files that are smaller will be uploaded in a single + * operation. * * The option will be converted to a <code>boolean</code> * type. @@ -2152,8 +2158,10 @@ public interface AWS2S3EndpointBuilderFactory { return this; } /** - * Setup the partSize which is used in multi part upload, the default - * size is 25M. + * Setup the partSize which is used in multi-part upload, the default + * size is 25M. Camel will only do multi-part uploads for files that are + * larger than the part-size thresholds. Files that are smaller will be + * uploaded in a single operation. * * The option is a: <code>long</code> type. * @@ -2168,8 +2176,10 @@ public interface AWS2S3EndpointBuilderFactory { return this; } /** - * Setup the partSize which is used in multi part upload, the default - * size is 25M. + * Setup the partSize which is used in multi-part upload, the default + * size is 25M. Camel will only do multi-part uploads for files that are + * larger than the part-size thresholds. Files that are smaller will be + * uploaded in a single operation. * * The option will be converted to a <code>long</code> type. *
