This is an automated email from the ASF dual-hosted git repository. acosentino pushed a commit to branch ibm-cos in repository https://gitbox.apache.org/repos/asf/camel-kamelets.git
commit d98404c5838f764de5ea835f3c8adccf0caa4c84 Author: Andrea Cosentino <[email protected]> AuthorDate: Thu Oct 30 15:12:11 2025 +0100 Camel IBM COS: Create Kamelets Signed-off-by: Andrea Cosentino <[email protected]> --- docs/modules/ROOT/nav.adoc | 2 + .../ROOT/partials/ibm-cos-sink-description.adoc | 128 +++++++++++++++ .../ROOT/partials/ibm-cos-source-description.adoc | 124 +++++++++++++++ kamelets/ibm-cos-sink.kamelet.yaml | 134 ++++++++++++++++ kamelets/ibm-cos-source.kamelet.yaml | 177 +++++++++++++++++++++ .../resources/kamelets/ibm-cos-sink.kamelet.yaml | 134 ++++++++++++++++ .../resources/kamelets/ibm-cos-source.kamelet.yaml | 177 +++++++++++++++++++++ 7 files changed, 876 insertions(+) diff --git a/docs/modules/ROOT/nav.adoc b/docs/modules/ROOT/nav.adoc index 56b7467af..0ad644c1e 100644 --- a/docs/modules/ROOT/nav.adoc +++ b/docs/modules/ROOT/nav.adoc @@ -107,6 +107,8 @@ * xref:http-secured-source.adoc[] * xref:http-sink.adoc[] * xref:http-source.adoc[] +* xref:ibm-cos-sink.adoc[] +* xref:ibm-cos-source.adoc[] * xref:infinispan-sink.adoc[] * xref:infinispan-source.adoc[] * xref:insert-field-action.adoc[] diff --git a/docs/modules/ROOT/partials/ibm-cos-sink-description.adoc b/docs/modules/ROOT/partials/ibm-cos-sink-description.adoc new file mode 100644 index 000000000..080987d6e --- /dev/null +++ b/docs/modules/ROOT/partials/ibm-cos-sink-description.adoc @@ -0,0 +1,128 @@ +== IBM Cloud Object Storage Sink Kamelet Description + +=== Authentication methods + +This Kamelet requires authentication with IBM Cloud using an API Key and Service Instance ID (CRN - Cloud Resource Name). + +To obtain these credentials: + + - API Key - Create an IBM Cloud API Key through the IBM Cloud console (Manage > Access (IAM) > API keys). + - Service Instance ID - This is the Cloud Resource Name (CRN) of your IBM COS instance, found in your IBM COS service instance details. + - Endpoint URL - The IBM COS endpoint URL for your region (e.g., https://s3.us-south.cloud-object-storage.appdomain.cloud). + +For more information, see the https://cloud.ibm.com/docs/cloud-object-storage[IBM Cloud Object Storage documentation] + +=== Optional Headers + +In the header, you can optionally set the `file` / `ce-file` property to specify the name of the object to upload. + +If you do not set the property in the header, the Kamelet uses the exchange ID for the object key name. + +=== Usage examples + +Upload data to an IBM COS bucket: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:timer-source" + steps: + - to: + uri: "kamelet:ibm-cos-sink" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" +---- + +==== Auto-creating buckets + +The Kamelet can automatically create the bucket if it doesn't exist: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:timer-source" + steps: + - to: + uri: "kamelet:ibm-cos-sink" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + autoCreateBucket: true +---- + +==== Specifying a custom key name + +You can specify a fixed key name for all uploaded objects: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:timer-source" + steps: + - to: + uri: "kamelet:ibm-cos-sink" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + keyName: "data/myfile.txt" +---- + +==== Using multi-part upload for large files + +For large files, you can enable multi-part upload: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:timer-source" + steps: + - to: + uri: "kamelet:ibm-cos-sink" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + multiPartUpload: true + partSize: 52428800 +---- + +The `partSize` is specified in bytes (default is 25MB = 26214400 bytes). + +==== Specifying storage class + +You can specify different storage classes for cost optimization: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:timer-source" + steps: + - to: + uri: "kamelet:ibm-cos-sink" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + storageClass: "STANDARD" +---- + +Available storage classes include STANDARD, VAULT, COLD, and FLEX. diff --git a/docs/modules/ROOT/partials/ibm-cos-source-description.adoc b/docs/modules/ROOT/partials/ibm-cos-source-description.adoc new file mode 100644 index 000000000..a99507231 --- /dev/null +++ b/docs/modules/ROOT/partials/ibm-cos-source-description.adoc @@ -0,0 +1,124 @@ +== IBM Cloud Object Storage Source Kamelet Description + +=== Authentication methods + +This Kamelet requires authentication with IBM Cloud using an API Key and Service Instance ID (CRN - Cloud Resource Name). + +To obtain these credentials: + + - API Key - Create an IBM Cloud API Key through the IBM Cloud console (Manage > Access (IAM) > API keys). + - Service Instance ID - This is the Cloud Resource Name (CRN) of your IBM COS instance, found in your IBM COS service instance details. + - Endpoint URL - The IBM COS endpoint URL for your region (e.g., https://s3.us-south.cloud-object-storage.appdomain.cloud). + +For more information, see the https://cloud.ibm.com/docs/cloud-object-storage[IBM Cloud Object Storage documentation] + +=== Usage examples + +You can consume objects from an IBM COS bucket and delete them after processing: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:ibm-cos-source" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + steps: + - to: + uri: "kamelet:log-sink" +---- + +The `deleteAfterRead` property is true by default, ensuring each object is consumed only once. + +==== Consuming objects without deletion + +To consume objects without deleting them, set `deleteAfterRead` to false: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:ibm-cos-source" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + deleteAfterRead: false + steps: + - to: + uri: "kamelet:log-sink" +---- + +==== Moving objects after reading + +Instead of deleting objects, you can move them to another bucket: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:ibm-cos-source" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + deleteAfterRead: false + moveAfterRead: true + destinationBucket: "archive-bucket" + destinationBucketPrefix: "processed/" + steps: + - to: + uri: "kamelet:log-sink" +---- + +==== Consuming with a prefix filter + +You can filter objects using a prefix: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:ibm-cos-source" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + prefix: "inbox/" + steps: + - to: + uri: "kamelet:log-sink" +---- + +This consumes only objects with keys starting with `inbox/`. + +==== Consuming only metadata + +To consume only object metadata without the body, set `includeBody` to false: + +[source,yaml,subs='+attributes,macros'] +---- +- route: + from: + uri: "kamelet:ibm-cos-source" + parameters: + bucketName: "my-bucket" + apiKey: "{{ibm.api.key}}" + serviceInstanceId: "{{ibm.service.instance.id}}" + endpointUrl: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: "us-south" + includeBody: false + steps: + - to: + uri: "kamelet:log-sink" +---- diff --git a/kamelets/ibm-cos-sink.kamelet.yaml b/kamelets/ibm-cos-sink.kamelet.yaml new file mode 100644 index 000000000..a5a30f23a --- /dev/null +++ b/kamelets/ibm-cos-sink.kamelet.yaml @@ -0,0 +1,134 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: ibm-cos-sink + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.16.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjUwMCIgaGVpZ2h0PSIyNTAwIiB2aWV3Qm94PSIwIDAgMjU2IDI1NiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBwcmVzZXJ2ZUFzcGVjdFJhdGlvPSJ4TWlkWU1pZCI+PHBhdGggZD0iTTAgMGgyNTZ2MjU2SDB6IiBmaWxsPSIjMDUzMzYyIi8+PHBhdGggZD0iTTE4Ny43MzIgMTI1LjkxOGMtLjAzNi0xLjM1Ny0uMDktMi43MTQtLjE4LTQuMDcyLS4wMzYtLjUyMi0uMDkxLTEuMDQ1LS4xMjgtMS41NjctLjA1NC0uNjEzLS4xMDgtMS4yMjYtLjE4LTEuODM5LS4wNzItLjc3LS4xOC0xLjU0LS4yODgtMi4zMWEzOS43NjYgMzkuNz [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "IBM Cloud Object Storage" + camel.apache.org/kamelet.namespace: "IBM" + labels: + camel.apache.org/kamelet.type: "sink" +spec: + definition: + title: "IBM Cloud Object Storage Sink" + description: Upload data to an IBM Cloud Object Storage Bucket. + required: + - bucketName + - apiKey + - serviceInstanceId + - endpointUrl + type: object + properties: + bucketName: + title: Bucket Name + description: The IBM COS Bucket name. + type: string + apiKey: + title: API Key + description: IBM Cloud API Key for authentication. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + serviceInstanceId: + title: Service Instance ID + description: IBM COS Service Instance ID (CRN). + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + endpointUrl: + title: Endpoint URL + description: IBM COS Endpoint URL (e.g., https://s3.us-south.cloud-object-storage.appdomain.cloud). + type: string + example: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: + title: Location + description: IBM COS Location/Region (e.g., us-south, eu-gb). + type: string + example: "us-south" + autoCreateBucket: + title: Autocreate Bucket + description: Specifies to automatically create the IBM COS bucket. + type: boolean + default: false + keyName: + title: Key Name + description: The key name for saving an element in the bucket. + type: string + multiPartUpload: + title: Multi-Part Upload + description: Use multi-part upload for large files. + type: boolean + default: false + partSize: + title: Part Size + description: Part size for multi-part uploads (default 25MB). + type: integer + default: 26214400 + storageClass: + title: Storage Class + description: The storage class to use when storing objects (e.g., STANDARD, VAULT, COLD, FLEX). + type: string + dependencies: + - "camel:core" + - "camel:ibm-cos" + - "camel:kamelet" + template: + from: + uri: "kamelet:source" + steps: + - choice: + precondition: true + when: + - simple: '${propertiesExist:!keyName}' + steps: + - choice: + when: + - simple: "${header[file]}" + steps: + - setHeader: + name: CamelIbmCosKey + simple: "${header[file]}" + - simple: "${header[ce-file]}" + steps: + - setHeader: + name: CamelIbmCosKey + simple: "${header[ce-file]}" + otherwise: + steps: + - setHeader: + name: CamelIbmCosKey + simple: "${exchangeId}" + - to: + uri: "ibm-cos:{{bucketName}}" + parameters: + apiKey: "{{apiKey}}" + serviceInstanceId: "{{serviceInstanceId}}" + endpointUrl: "{{endpointUrl}}" + location: "{{?location}}" + autoCreateBucket: "{{autoCreateBucket}}" + keyName: "{{?keyName}}" + multiPartUpload: "{{multiPartUpload}}" + partSize: "{{partSize}}" + storageClass: "{{?storageClass}}" diff --git a/kamelets/ibm-cos-source.kamelet.yaml b/kamelets/ibm-cos-source.kamelet.yaml new file mode 100644 index 000000000..24e24c627 --- /dev/null +++ b/kamelets/ibm-cos-source.kamelet.yaml @@ -0,0 +1,177 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: ibm-cos-source + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.16.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjUwMCIgaGVpZ2h0PSIyNTAwIiB2aWV3Qm94PSIwIDAgMjU2IDI1NiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBwcmVzZXJ2ZUFzcGVjdFJhdGlvPSJ4TWlkWU1pZCI+PHBhdGggZD0iTTAgMGgyNTZ2MjU2SDB6IiBmaWxsPSIjMDUzMzYyIi8+PHBhdGggZD0iTTE4Ny43MzIgMTI1LjkxOGMtLjAzNi0xLjM1Ny0uMDktMi43MTQtLjE4LTQuMDcyLS4wMzYtLjUyMi0uMDkxLTEuMDQ1LS4xMjgtMS41NjctLjA1NC0uNjEzLS4xMDgtMS4yMjYtLjE4LTEuODM5LS4wNzItLjc3LS4xOC0xLjU0LS4yODgtMi4zMWEzOS43NjYgMzkuNz [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "IBM Cloud Object Storage" + camel.apache.org/kamelet.namespace: "IBM" + labels: + camel.apache.org/kamelet.type: "source" +spec: + definition: + title: "IBM Cloud Object Storage Source" + description: Receive data from an IBM Cloud Object Storage Bucket. + required: + - bucketName + - apiKey + - serviceInstanceId + - endpointUrl + type: object + properties: + bucketName: + title: Bucket Name + description: The IBM COS Bucket name. + type: string + apiKey: + title: API Key + description: IBM Cloud API Key for authentication. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + serviceInstanceId: + title: Service Instance ID + description: IBM COS Service Instance ID (CRN). + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + endpointUrl: + title: Endpoint URL + description: IBM COS Endpoint URL (e.g., https://s3.us-south.cloud-object-storage.appdomain.cloud). + type: string + example: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: + title: Location + description: IBM COS Location/Region (e.g., us-south, eu-gb). + type: string + example: "us-south" + deleteAfterRead: + title: Auto-delete Objects + description: Specifies to delete objects after consuming them. + type: boolean + default: true + moveAfterRead: + title: Move Objects After Read + description: Move objects from IBM COS bucket to a different bucket after they have been retrieved. + type: boolean + default: false + destinationBucket: + title: Destination Bucket + description: Define the destination bucket where an object must be moved when moveAfterRead is set to true. + type: string + destinationBucketPrefix: + title: Destination Bucket Prefix + description: Define the destination bucket prefix to use when an object must be moved, and moveAfterRead is set to true. + type: string + destinationBucketSuffix: + title: Destination Bucket Suffix + description: Define the destination bucket suffix to use when an object must be moved, and moveAfterRead is set to true. + type: string + autoCreateBucket: + title: Autocreate Bucket + description: Specifies to automatically create the IBM COS bucket. + type: boolean + default: false + prefix: + title: Prefix + description: The IBM COS bucket prefix to consider while searching. + type: string + example: 'folder/' + delimiter: + title: Delimiter + description: The delimiter to use for listing objects. + type: string + includeBody: + title: Include Body + description: If true, the object body is included in the exchange. If false, only headers are set. + type: boolean + default: true + includeFolders: + title: Include Folders + description: Include folders/directories when listing objects. + type: boolean + default: true + delay: + title: Delay + description: The number of milliseconds before the next poll of the selected bucket. + type: integer + default: 500 + maxMessagesPerPoll: + title: Max Messages Per Poll + description: Gets the maximum number of messages as a limit to poll at each polling. The default value is 10. Use 0 or a negative number to set it as unlimited. + type: integer + default: 10 + dataTypes: + out: + default: binary + headers: + CamelIbmCosBucketName: + title: IBM COS Bucket Name + description: The bucket name which has been used to retrieve objects + type: string + CamelIbmCosKey: + title: IBM COS Key + description: The key under which the retrieved object is stored. + type: string + CamelIbmCosContentType: + title: Content Type + description: The content type of the retrieved object. + default: application/octet-stream + type: string + CamelIbmCosETag: + title: ETag Value + description: The hex encoded 128-bit MD5 digest of the associated object according to RFC 1864. + type: string + types: + binary: + format: "application-octet-stream" + description: Default binary representation of the IBM COS object retrieved from the bucket. + mediaType: application/octet-stream + dependencies: + - "camel:core" + - "camel:ibm-cos" + - "camel:kamelet" + template: + from: + uri: "ibm-cos:{{bucketName}}" + parameters: + apiKey: "{{apiKey}}" + serviceInstanceId: "{{serviceInstanceId}}" + endpointUrl: "{{endpointUrl}}" + location: "{{?location}}" + deleteAfterRead: "{{deleteAfterRead}}" + moveAfterRead: "{{moveAfterRead}}" + destinationBucket: "{{?destinationBucket}}" + destinationBucketPrefix: "{{?destinationBucketPrefix}}" + destinationBucketSuffix: "{{?destinationBucketSuffix}}" + autoCreateBucket: "{{autoCreateBucket}}" + prefix: "{{?prefix}}" + delimiter: "{{?delimiter}}" + includeBody: "{{includeBody}}" + includeFolders: "{{includeFolders}}" + delay: "{{delay}}" + maxMessagesPerPoll: "{{maxMessagesPerPoll}}" + steps: + - to: "kamelet:sink" diff --git a/library/camel-kamelets/src/main/resources/kamelets/ibm-cos-sink.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/ibm-cos-sink.kamelet.yaml new file mode 100644 index 000000000..a5a30f23a --- /dev/null +++ b/library/camel-kamelets/src/main/resources/kamelets/ibm-cos-sink.kamelet.yaml @@ -0,0 +1,134 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: ibm-cos-sink + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.16.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjUwMCIgaGVpZ2h0PSIyNTAwIiB2aWV3Qm94PSIwIDAgMjU2IDI1NiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBwcmVzZXJ2ZUFzcGVjdFJhdGlvPSJ4TWlkWU1pZCI+PHBhdGggZD0iTTAgMGgyNTZ2MjU2SDB6IiBmaWxsPSIjMDUzMzYyIi8+PHBhdGggZD0iTTE4Ny43MzIgMTI1LjkxOGMtLjAzNi0xLjM1Ny0uMDktMi43MTQtLjE4LTQuMDcyLS4wMzYtLjUyMi0uMDkxLTEuMDQ1LS4xMjgtMS41NjctLjA1NC0uNjEzLS4xMDgtMS4yMjYtLjE4LTEuODM5LS4wNzItLjc3LS4xOC0xLjU0LS4yODgtMi4zMWEzOS43NjYgMzkuNz [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "IBM Cloud Object Storage" + camel.apache.org/kamelet.namespace: "IBM" + labels: + camel.apache.org/kamelet.type: "sink" +spec: + definition: + title: "IBM Cloud Object Storage Sink" + description: Upload data to an IBM Cloud Object Storage Bucket. + required: + - bucketName + - apiKey + - serviceInstanceId + - endpointUrl + type: object + properties: + bucketName: + title: Bucket Name + description: The IBM COS Bucket name. + type: string + apiKey: + title: API Key + description: IBM Cloud API Key for authentication. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + serviceInstanceId: + title: Service Instance ID + description: IBM COS Service Instance ID (CRN). + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + endpointUrl: + title: Endpoint URL + description: IBM COS Endpoint URL (e.g., https://s3.us-south.cloud-object-storage.appdomain.cloud). + type: string + example: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: + title: Location + description: IBM COS Location/Region (e.g., us-south, eu-gb). + type: string + example: "us-south" + autoCreateBucket: + title: Autocreate Bucket + description: Specifies to automatically create the IBM COS bucket. + type: boolean + default: false + keyName: + title: Key Name + description: The key name for saving an element in the bucket. + type: string + multiPartUpload: + title: Multi-Part Upload + description: Use multi-part upload for large files. + type: boolean + default: false + partSize: + title: Part Size + description: Part size for multi-part uploads (default 25MB). + type: integer + default: 26214400 + storageClass: + title: Storage Class + description: The storage class to use when storing objects (e.g., STANDARD, VAULT, COLD, FLEX). + type: string + dependencies: + - "camel:core" + - "camel:ibm-cos" + - "camel:kamelet" + template: + from: + uri: "kamelet:source" + steps: + - choice: + precondition: true + when: + - simple: '${propertiesExist:!keyName}' + steps: + - choice: + when: + - simple: "${header[file]}" + steps: + - setHeader: + name: CamelIbmCosKey + simple: "${header[file]}" + - simple: "${header[ce-file]}" + steps: + - setHeader: + name: CamelIbmCosKey + simple: "${header[ce-file]}" + otherwise: + steps: + - setHeader: + name: CamelIbmCosKey + simple: "${exchangeId}" + - to: + uri: "ibm-cos:{{bucketName}}" + parameters: + apiKey: "{{apiKey}}" + serviceInstanceId: "{{serviceInstanceId}}" + endpointUrl: "{{endpointUrl}}" + location: "{{?location}}" + autoCreateBucket: "{{autoCreateBucket}}" + keyName: "{{?keyName}}" + multiPartUpload: "{{multiPartUpload}}" + partSize: "{{partSize}}" + storageClass: "{{?storageClass}}" diff --git a/library/camel-kamelets/src/main/resources/kamelets/ibm-cos-source.kamelet.yaml b/library/camel-kamelets/src/main/resources/kamelets/ibm-cos-source.kamelet.yaml new file mode 100644 index 000000000..24e24c627 --- /dev/null +++ b/library/camel-kamelets/src/main/resources/kamelets/ibm-cos-source.kamelet.yaml @@ -0,0 +1,177 @@ +# --------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# --------------------------------------------------------------------------- + +apiVersion: camel.apache.org/v1 +kind: Kamelet +metadata: + name: ibm-cos-source + annotations: + camel.apache.org/kamelet.support.level: "Preview" + camel.apache.org/catalog.version: "4.16.0-SNAPSHOT" + camel.apache.org/kamelet.icon: "data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iMjUwMCIgaGVpZ2h0PSIyNTAwIiB2aWV3Qm94PSIwIDAgMjU2IDI1NiIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiBwcmVzZXJ2ZUFzcGVjdFJhdGlvPSJ4TWlkWU1pZCI+PHBhdGggZD0iTTAgMGgyNTZ2MjU2SDB6IiBmaWxsPSIjMDUzMzYyIi8+PHBhdGggZD0iTTE4Ny43MzIgMTI1LjkxOGMtLjAzNi0xLjM1Ny0uMDktMi43MTQtLjE4LTQuMDcyLS4wMzYtLjUyMi0uMDkxLTEuMDQ1LS4xMjgtMS41NjctLjA1NC0uNjEzLS4xMDgtMS4yMjYtLjE4LTEuODM5LS4wNzItLjc3LS4xOC0xLjU0LS4yODgtMi4zMWEzOS43NjYgMzkuNz [...] + camel.apache.org/provider: "Apache Software Foundation" + camel.apache.org/kamelet.group: "IBM Cloud Object Storage" + camel.apache.org/kamelet.namespace: "IBM" + labels: + camel.apache.org/kamelet.type: "source" +spec: + definition: + title: "IBM Cloud Object Storage Source" + description: Receive data from an IBM Cloud Object Storage Bucket. + required: + - bucketName + - apiKey + - serviceInstanceId + - endpointUrl + type: object + properties: + bucketName: + title: Bucket Name + description: The IBM COS Bucket name. + type: string + apiKey: + title: API Key + description: IBM Cloud API Key for authentication. + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + serviceInstanceId: + title: Service Instance ID + description: IBM COS Service Instance ID (CRN). + type: string + format: password + x-descriptors: + - urn:camel:group:credentials + endpointUrl: + title: Endpoint URL + description: IBM COS Endpoint URL (e.g., https://s3.us-south.cloud-object-storage.appdomain.cloud). + type: string + example: "https://s3.us-south.cloud-object-storage.appdomain.cloud"; + location: + title: Location + description: IBM COS Location/Region (e.g., us-south, eu-gb). + type: string + example: "us-south" + deleteAfterRead: + title: Auto-delete Objects + description: Specifies to delete objects after consuming them. + type: boolean + default: true + moveAfterRead: + title: Move Objects After Read + description: Move objects from IBM COS bucket to a different bucket after they have been retrieved. + type: boolean + default: false + destinationBucket: + title: Destination Bucket + description: Define the destination bucket where an object must be moved when moveAfterRead is set to true. + type: string + destinationBucketPrefix: + title: Destination Bucket Prefix + description: Define the destination bucket prefix to use when an object must be moved, and moveAfterRead is set to true. + type: string + destinationBucketSuffix: + title: Destination Bucket Suffix + description: Define the destination bucket suffix to use when an object must be moved, and moveAfterRead is set to true. + type: string + autoCreateBucket: + title: Autocreate Bucket + description: Specifies to automatically create the IBM COS bucket. + type: boolean + default: false + prefix: + title: Prefix + description: The IBM COS bucket prefix to consider while searching. + type: string + example: 'folder/' + delimiter: + title: Delimiter + description: The delimiter to use for listing objects. + type: string + includeBody: + title: Include Body + description: If true, the object body is included in the exchange. If false, only headers are set. + type: boolean + default: true + includeFolders: + title: Include Folders + description: Include folders/directories when listing objects. + type: boolean + default: true + delay: + title: Delay + description: The number of milliseconds before the next poll of the selected bucket. + type: integer + default: 500 + maxMessagesPerPoll: + title: Max Messages Per Poll + description: Gets the maximum number of messages as a limit to poll at each polling. The default value is 10. Use 0 or a negative number to set it as unlimited. + type: integer + default: 10 + dataTypes: + out: + default: binary + headers: + CamelIbmCosBucketName: + title: IBM COS Bucket Name + description: The bucket name which has been used to retrieve objects + type: string + CamelIbmCosKey: + title: IBM COS Key + description: The key under which the retrieved object is stored. + type: string + CamelIbmCosContentType: + title: Content Type + description: The content type of the retrieved object. + default: application/octet-stream + type: string + CamelIbmCosETag: + title: ETag Value + description: The hex encoded 128-bit MD5 digest of the associated object according to RFC 1864. + type: string + types: + binary: + format: "application-octet-stream" + description: Default binary representation of the IBM COS object retrieved from the bucket. + mediaType: application/octet-stream + dependencies: + - "camel:core" + - "camel:ibm-cos" + - "camel:kamelet" + template: + from: + uri: "ibm-cos:{{bucketName}}" + parameters: + apiKey: "{{apiKey}}" + serviceInstanceId: "{{serviceInstanceId}}" + endpointUrl: "{{endpointUrl}}" + location: "{{?location}}" + deleteAfterRead: "{{deleteAfterRead}}" + moveAfterRead: "{{moveAfterRead}}" + destinationBucket: "{{?destinationBucket}}" + destinationBucketPrefix: "{{?destinationBucketPrefix}}" + destinationBucketSuffix: "{{?destinationBucketSuffix}}" + autoCreateBucket: "{{autoCreateBucket}}" + prefix: "{{?prefix}}" + delimiter: "{{?delimiter}}" + includeBody: "{{includeBody}}" + includeFolders: "{{includeFolders}}" + delay: "{{delay}}" + maxMessagesPerPoll: "{{maxMessagesPerPoll}}" + steps: + - to: "kamelet:sink"
