This is an automated email from the ASF dual-hosted git repository. acosentino pushed a commit to branch CAMEL-20586 in repository https://gitbox.apache.org/repos/asf/camel.git
commit ad1c284c0a2da09a453f892eb1b8ae7f1c051e68 Author: Andrea Cosentino <[email protected]> AuthorDate: Thu Mar 21 11:49:58 2024 +0100 CAMEL-20594 - Camel-Milvus: Add a datatype for transforming langchain embeddings in Milvus objects Signed-off-by: Andrea Cosentino <[email protected]> --- ...LangchainEmbeddingsComponentMilvusTargetIT.java | 10 +++- .../embeddings/LangchainEmbeddingsTestSupport.java | 50 ------------------ components/camel-milvus/pom.xml | 6 +++ .../org/apache/camel/component/milvus/milvus.json | 5 +- .../org/apache/camel/transformer.properties | 7 +++ .../org/apache/camel/transformer/milvus-embeddings | 2 + .../camel/transformer/milvus-embeddings.json | 14 +++++ .../org/apache/camel/transformer/qdrant-embeddings | 2 + .../camel/transformer/qdrant-embeddings.json | 14 +++++ .../org/apache/camel/component/milvus/Milvus.java | 9 ++++ .../MilvusEmbeddingsDataTypeTransformer.java | 60 ++++++++++++++++++++++ 11 files changed, 126 insertions(+), 53 deletions(-) diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java index 3ad16173bbe..516003c0aa6 100644 --- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java +++ b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsComponentMilvusTargetIT.java @@ -90,6 +90,12 @@ public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport .withDimension(384) .build(); + FieldType fieldType3 = FieldType.newBuilder() + .withName("text") + .withDataType(DataType.VarChar) + .withMaxLength(65535) + .build(); + CreateCollectionParam createCollectionReq = CreateCollectionParam.newBuilder() .withCollectionName("embeddings") .withDescription("customer info") @@ -97,6 +103,7 @@ public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport .withEnableDynamicField(false) .addFieldType(fieldType1) .addFieldType(fieldType2) + .addFieldType(fieldType3) .build(); Exchange result = fluentTemplate.to(MILVUS_URI) @@ -170,9 +177,8 @@ public class LangchainEmbeddingsComponentMilvusTargetIT extends CamelTestSupport public void configure() { from("direct:in") .to("langchain-embeddings:test") - .bean(LangchainEmbeddingsTestSupport.AsInsertParam.class) .setHeader(Milvus.Headers.ACTION).constant(MilvusAction.INSERT) - .bean(LangchainEmbeddingsTestSupport.AsInsertParam.class) + .transform(new org.apache.camel.spi.DataType("milvus:embeddings")) .to(MILVUS_URI); } }; diff --git a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java b/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java deleted file mode 100644 index f7781284dae..00000000000 --- a/components/camel-ai/camel-langchain-embeddings/src/test/java/org/apache/camel/component/langchain/embeddings/LangchainEmbeddingsTestSupport.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.camel.component.langchain.embeddings; - -import java.util.ArrayList; -import java.util.List; - -import com.google.protobuf.InvalidProtocolBufferException; -import dev.langchain4j.data.embedding.Embedding; -import io.milvus.param.dml.InsertParam; -import org.apache.camel.Exchange; -import org.apache.camel.Handler; - -public class LangchainEmbeddingsTestSupport { - - private LangchainEmbeddingsTestSupport() { - } - - public static class AsInsertParam { - @Handler - public InsertParam AsInsertParam(Exchange e) throws InvalidProtocolBufferException { - Embedding embedding = e.getMessage().getHeader(LangchainEmbeddings.Headers.VECTOR, Embedding.class); - List<InsertParam.Field> fields = new ArrayList<>(); - ArrayList list = new ArrayList<>(); - list.add(embedding.vectorAsList()); - fields.add(new InsertParam.Field("vector", list)); - - InsertParam insertParam = InsertParam.newBuilder() - .withCollectionName("embeddings") - .withFields(fields) - .build(); - - return insertParam; - } - } -} diff --git a/components/camel-milvus/pom.xml b/components/camel-milvus/pom.xml index 7e707de0f28..f8a8e088c79 100644 --- a/components/camel-milvus/pom.xml +++ b/components/camel-milvus/pom.xml @@ -48,6 +48,12 @@ <artifactId>camel-support</artifactId> </dependency> + <dependency> + <groupId>dev.langchain4j</groupId> + <artifactId>langchain4j-core</artifactId> + <version>${langchain4j.version}</version> + </dependency> + <dependency> <groupId>io.milvus</groupId> <artifactId>milvus-sdk-java</artifactId> diff --git a/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json b/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json index 4e470ce0c9c..014942d2d91 100644 --- a/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json +++ b/components/camel-milvus/src/generated/resources/META-INF/org/apache/camel/component/milvus/milvus.json @@ -34,7 +34,10 @@ "headers": { "CamelMilvusAction": { "index": 0, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "enum": [ "CREATE_COLLECTION", "CREATE_INDEX", "UPSERT", "INSERT", "SEARCH", "DELETE" ], "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The action to be performed.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#ACTION" }, "CamelMilvusOperationStatus": { "index": 1, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS" }, - "CamelMilvusOperationStatusValue": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status Value.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS_VALUE" } + "CamelMilvusOperationStatusValue": { "index": 2, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "int", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Operation Status Value.", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#OPERATION_STATUS_VALUE" }, + "CamelMilvusTextFieldName": { "index": 3, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Text Field Name for Insert\/Upsert operation", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#TEXT_FIELD_NAME" }, + "CamelMilvusVectorFieldName": { "index": 4, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Vector Field Name for Insert\/Upsert operation", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#VECTOR_FIELD_NAME" }, + "CamelMilvusCollectionName": { "index": 5, "kind": "header", "displayName": "", "group": "producer", "label": "", "required": false, "javaType": "String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "Collection Name for Insert\/Upsert operation", "constantName": "org.apache.camel.component.milvus.Milvus$Headers#COLLECTION_NAME" } }, "properties": { "collection": { "index": 0, "kind": "path", "displayName": "Collection", "group": "producer", "label": "", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": false, "deprecationNote": "", "autowired": false, "secret": false, "description": "The collection Name" }, diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties new file mode 100644 index 00000000000..de216dd7f50 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer.properties @@ -0,0 +1,7 @@ +# Generated by camel build tools - do NOT edit this file! +transformers=milvus:embeddings +groupId=org.apache.camel +artifactId=camel-milvus +version=4.5.0-SNAPSHOT +projectName=Camel :: Milvus +projectDescription=Camel Milvus support diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings new file mode 100644 index 00000000000..ccd7a4be1ed --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings.json b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings.json new file mode 100644 index 00000000000..e04d45f50c7 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/milvus-embeddings.json @@ -0,0 +1,14 @@ +{ + "transformer": { + "kind": "transformer", + "name": "milvus:embeddings", + "title": "Milvus (Embeddings)", + "description": "Prepares the message to become an object writable by Milvus component", + "deprecated": false, + "javaType": "org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer", + "groupId": "org.apache.camel", + "artifactId": "camel-milvus", + "version": "4.5.0-SNAPSHOT" + } +} + diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings new file mode 100644 index 00000000000..ccd7a4be1ed --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings @@ -0,0 +1,2 @@ +# Generated by camel build tools - do NOT edit this file! +class=org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer diff --git a/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json new file mode 100644 index 00000000000..2eaeb37bfa7 --- /dev/null +++ b/components/camel-milvus/src/generated/resources/META-INF/services/org/apache/camel/transformer/qdrant-embeddings.json @@ -0,0 +1,14 @@ +{ + "transformer": { + "kind": "transformer", + "name": "qdrant:embeddings", + "title": "Qdrant (Embeddings)", + "description": "Prepares the message to become an object writable by Milvus component", + "deprecated": false, + "javaType": "org.apache.camel.component.milvus.transform.MilvusEmbeddingsDataTypeTransformer", + "groupId": "org.apache.camel", + "artifactId": "camel-milvus", + "version": "4.5.0-SNAPSHOT" + } +} + diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java index ebc92861673..7c2fc3bb7f1 100644 --- a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/Milvus.java @@ -34,5 +34,14 @@ public class Milvus { @Metadata(description = "Operation Status Value.", javaType = "int") public static final String OPERATION_STATUS_VALUE = "CamelMilvusOperationStatusValue"; + + @Metadata(description = "Text Field Name for Insert/Upsert operation", javaType = "String") + public static final String TEXT_FIELD_NAME = "CamelMilvusTextFieldName"; + + @Metadata(description = "Vector Field Name for Insert/Upsert operation", javaType = "String") + public static final String VECTOR_FIELD_NAME = "CamelMilvusVectorFieldName"; + + @Metadata(description = "Collection Name for Insert/Upsert operation", javaType = "String") + public static final String COLLECTION_NAME = "CamelMilvusCollectionName"; } } diff --git a/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/transform/MilvusEmbeddingsDataTypeTransformer.java b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/transform/MilvusEmbeddingsDataTypeTransformer.java new file mode 100644 index 00000000000..1aed3d370f7 --- /dev/null +++ b/components/camel-milvus/src/main/java/org/apache/camel/component/milvus/transform/MilvusEmbeddingsDataTypeTransformer.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.camel.component.milvus.transform; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import dev.langchain4j.data.embedding.Embedding; +import dev.langchain4j.data.segment.TextSegment; +import io.milvus.param.dml.InsertParam; +import org.apache.camel.Message; +import org.apache.camel.component.milvus.Milvus; +import org.apache.camel.spi.DataType; +import org.apache.camel.spi.DataTypeTransformer; +import org.apache.camel.spi.Transformer; + +/** + * Maps a Langchain Embeddings to a Milvus InsertParam/Upsert Param to write an embeddings vector on a Milvus Database. + */ +@DataTypeTransformer(name = "milvus:embeddings", + description = "Prepares the message to become an object writable by Milvus component") +public class MilvusEmbeddingsDataTypeTransformer extends Transformer { + + @Override + public void transform(Message message, DataType fromType, DataType toType) { + Embedding embedding = message.getHeader("CamelLangchainEmbeddingsVector", Embedding.class); + String textFieldName = message.getHeader(Milvus.Headers.TEXT_FIELD_NAME, () -> "text", String.class); + String vectorFieldName = message.getHeader(Milvus.Headers.VECTOR_FIELD_NAME, () -> "vector", String.class); + String collectionName = message.getHeader(Milvus.Headers.COLLECTION_NAME, () -> "embeddings", String.class); + TextSegment text = message.getBody(TextSegment.class); + List<InsertParam.Field> fields = new ArrayList<>(); + ArrayList list = new ArrayList<>(); + list.add(embedding.vectorAsList()); + fields.add(new InsertParam.Field(vectorFieldName, list)); + fields.add(new InsertParam.Field(textFieldName, Collections.singletonList(text.text()))); + + InsertParam insertParam = InsertParam.newBuilder() + .withCollectionName(collectionName) + .withFields(fields) + .build(); + + message.setBody(insertParam); + } +}
