This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 664692c4f6ed CAMEL-26300: camel-pinecone - pinecone:embeddings stores
the text pinecone:rag reads, and names a missing embedding header (#29213)
664692c4f6ed is described below
commit 664692c4f6edff3ab0f84c8465715664cf48c027
Author: Andrea Cosentino <[email protected]>
AuthorDate: Thu Nov 5 13:02:35 2026 +0100
CAMEL-26300: camel-pinecone - pinecone:embeddings stores the text
pinecone:rag reads, and names a missing embedding header (#29213)
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../apache/camel/catalog/components/pinecone.json | 3 +-
.../camel/catalog/docs/pinecone-component.adoc | 4 ++
.../catalog/transformers/pinecone-embeddings.json | 2 +-
.../apache/camel/component/pinecone/pinecone.json | 3 +-
.../camel/transformer/pinecone-embeddings.json | 2 +-
.../src/main/docs/pinecone-component.adoc | 4 ++
.../pinecone/PineconeVectorDbHeaders.java | 6 ++
.../pinecone/PineconeVectorDbProducer.java | 8 ++-
.../PineconeEmbeddingsDataTypeTransformer.java | 28 +++++++-
.../pinecone/PineconeVectorDbProducerTest.java | 83 ++++++++++++++++++++--
.../PineconeEmbeddingsDataTypeTransformerTest.java | 82 +++++++++++++++++----
.../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 12 ++++
.../PineconeVectorDbEndpointBuilderFactory.java | 15 ++++
13 files changed, 225 insertions(+), 27 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
index fe55acd56013..d0ca3048a279 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/pinecone.json
@@ -56,7 +56,8 @@
"CamelPineconeNamespace": { "index": 13, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Namespace for actions (query\/upsert\/etc)",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#NAMESPACE" },
"CamelPineconeQueryFilter": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "com.google.protobuf.Struct", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Query Filter", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_FILTER" },
"CamelPineconeQueryIncludeValues": { "index": 15, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Values", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_VALUES"
},
- "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Metadata",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
}
+ "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Metadata",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
},
+ "CamelPineconeMetadata": { "index": 17, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType":
"com.google.protobuf.Struct", "deprecated": false, "deprecationNote": "",
"autowired": false, "secret": false, "description": "The metadata UPSERT stores
with the vector, and the fields of the metadata UPDATE sets. The
pinecone:embeddings data type transformer puts the text of the embedding in its
text_segment field, which the pinecone:rag data [...]
},
"bodyTypes": {
"producerIn": { "carriers": [ "list", "null" ], "byOption": [ { "option":
"action", "value": "UPSERT", "carriers": [ "list" ] }, { "option": "action",
"value": "UPDATE", "carriers": [ "list" ] }, { "option": "action", "value":
"QUERY", "carriers": [ "list" ] }, { "option": "action", "value": "FETCH",
"carriers": [ "list" ] }, { "option": "action", "value": "DELETE_BY_ID",
"carriers": [ "list" ] }, { "option": "action", "value": "QUERY_BY_ID",
"carriers": [ "any" ] }, { "option": "act [...]
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
index 35a185854b8a..ae76560e4878 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/pinecone-component.adoc
@@ -76,4 +76,8 @@ The `action` option, or the `CamelPineconeAction` header,
selects the operation.
The vector of UPSERT, UPDATE and QUERY is the message body: a List of numbers,
such as the List of Double a JSON array unmarshals to, or a `float[]` or a
`double[]`; each number is converted to a float. Another body fails, and so
does a List with an element that is not a number. UPSERT stores the vector
under the id in the `CamelPineconeIndexId` header; without the header, it
generates a random UUID and sets the header to it.
+UPSERT stores the `CamelPineconeMetadata` header, a
`com.google.protobuf.Struct`, as the metadata of the vector, and UPDATE sets
the fields of that header in the metadata of the vector.
+
+The `pinecone:embeddings` data type transformer turns the embedding of the
`CamelLangChain4jEmbeddingsVector` header, which the langchain4j-embeddings
component sets, into the vector of the message body, and fails when the header
is missing. It puts the text of a LangChain4j `TextSegment` or `String` body in
the `text_segment` field of the `CamelPineconeMetadata` header, and keeps the
other fields of that header. Without the `CamelPineconeIndexName` and
`CamelPineconeIndexId` headers, it [...]
+
The `pinecone:rag` data type transformer turns the result of a QUERY or
QUERY_BY_ID into the List of the texts of its matches, read from their
`text_segment` metadata, which the query returns when the
`CamelPineconeQueryIncludeMetadata` header is `true`.
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-embeddings.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-embeddings.json
index 614b93a419d1..69346d3e34de 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-embeddings.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/transformers/pinecone-embeddings.json
@@ -15,7 +15,7 @@
"carriers": [
"any"
],
- "description": "The body is not read: the vector comes from the
CamelLangChain4jEmbeddingsVector header, which must be set, and the text is not
stored"
+ "description": "A LangChain4j TextSegment, or a String the converter of
camel-langchain4j-embeddings turns into one, whose text goes in the
text_segment field of the CamelPineconeMetadata header, which UPSERT and UPDATE
store with the vector; another body stores no text. The vector comes from the
CamelLangChain4jEmbeddingsVector header, which must be set"
},
"transformerOut": {
"carriers": [
diff --git
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
index fe55acd56013..d0ca3048a279 100644
---
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
+++
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/org/apache/camel/component/pinecone/pinecone.json
@@ -56,7 +56,8 @@
"CamelPineconeNamespace": { "index": 13, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType": "String",
"deprecated": false, "deprecationNote": "", "autowired": false, "secret":
false, "description": "Namespace for actions (query\/upsert\/etc)",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#NAMESPACE" },
"CamelPineconeQueryFilter": { "index": 14, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "com.google.protobuf.Struct", "deprecated": false,
"deprecationNote": "", "autowired": false, "secret": false, "description":
"Query Filter", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_FILTER" },
"CamelPineconeQueryIncludeValues": { "index": 15, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Values", "constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_VALUES"
},
- "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Metadata",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
}
+ "CamelPineconeQueryIncludeMetadata": { "index": 16, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "boolean", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "Query Include Metadata",
"constantName":
"org.apache.camel.component.pinecone.PineconeVectorDbHeaders#QUERY_INCLUDE_METADATA"
},
+ "CamelPineconeMetadata": { "index": 17, "kind": "header", "displayName":
"", "group": "producer", "label": "", "required": false, "javaType":
"com.google.protobuf.Struct", "deprecated": false, "deprecationNote": "",
"autowired": false, "secret": false, "description": "The metadata UPSERT stores
with the vector, and the fields of the metadata UPDATE sets. The
pinecone:embeddings data type transformer puts the text of the embedding in its
text_segment field, which the pinecone:rag data [...]
},
"bodyTypes": {
"producerIn": { "carriers": [ "list", "null" ], "byOption": [ { "option":
"action", "value": "UPSERT", "carriers": [ "list" ] }, { "option": "action",
"value": "UPDATE", "carriers": [ "list" ] }, { "option": "action", "value":
"QUERY", "carriers": [ "list" ] }, { "option": "action", "value": "FETCH",
"carriers": [ "list" ] }, { "option": "action", "value": "DELETE_BY_ID",
"carriers": [ "list" ] }, { "option": "action", "value": "QUERY_BY_ID",
"carriers": [ "any" ] }, { "option": "act [...]
diff --git
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-embeddings.json
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-embeddings.json
index 614b93a419d1..69346d3e34de 100644
---
a/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-embeddings.json
+++
b/components/camel-ai/camel-pinecone/src/generated/resources/META-INF/services/org/apache/camel/transformer/pinecone-embeddings.json
@@ -15,7 +15,7 @@
"carriers": [
"any"
],
- "description": "The body is not read: the vector comes from the
CamelLangChain4jEmbeddingsVector header, which must be set, and the text is not
stored"
+ "description": "A LangChain4j TextSegment, or a String the converter of
camel-langchain4j-embeddings turns into one, whose text goes in the
text_segment field of the CamelPineconeMetadata header, which UPSERT and UPDATE
store with the vector; another body stores no text. The vector comes from the
CamelLangChain4jEmbeddingsVector header, which must be set"
},
"transformerOut": {
"carriers": [
diff --git
a/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
b/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
index 35a185854b8a..ae76560e4878 100644
--- a/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
+++ b/components/camel-ai/camel-pinecone/src/main/docs/pinecone-component.adoc
@@ -76,4 +76,8 @@ The `action` option, or the `CamelPineconeAction` header,
selects the operation.
The vector of UPSERT, UPDATE and QUERY is the message body: a List of numbers,
such as the List of Double a JSON array unmarshals to, or a `float[]` or a
`double[]`; each number is converted to a float. Another body fails, and so
does a List with an element that is not a number. UPSERT stores the vector
under the id in the `CamelPineconeIndexId` header; without the header, it
generates a random UUID and sets the header to it.
+UPSERT stores the `CamelPineconeMetadata` header, a
`com.google.protobuf.Struct`, as the metadata of the vector, and UPDATE sets
the fields of that header in the metadata of the vector.
+
+The `pinecone:embeddings` data type transformer turns the embedding of the
`CamelLangChain4jEmbeddingsVector` header, which the langchain4j-embeddings
component sets, into the vector of the message body, and fails when the header
is missing. It puts the text of a LangChain4j `TextSegment` or `String` body in
the `text_segment` field of the `CamelPineconeMetadata` header, and keeps the
other fields of that header. Without the `CamelPineconeIndexName` and
`CamelPineconeIndexId` headers, it [...]
+
The `pinecone:rag` data type transformer turns the result of a QUERY or
QUERY_BY_ID into the List of the texts of its matches, read from their
`text_segment` metadata, which the query returns when the
`CamelPineconeQueryIncludeMetadata` header is `true`.
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
index 222d31fe84dc..d5d527ae52fe 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbHeaders.java
@@ -73,4 +73,10 @@ public class PineconeVectorDbHeaders {
@Metadata(description = "Query Include Metadata", javaType = "boolean")
public static final String QUERY_INCLUDE_METADATA =
"CamelPineconeQueryIncludeMetadata";
+ @Metadata(description = "The metadata UPSERT stores with the vector, and
the fields of the metadata UPDATE sets. The"
+ + " pinecone:embeddings data type transformer puts
the text of the embedding in its text_segment"
+ + " field, which the pinecone:rag data type
transformer reads",
+ javaType = "com.google.protobuf.Struct")
+ public static final String METADATA = "CamelPineconeMetadata";
+
}
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
index 4ea721b30af8..d5736e8f50ce 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/PineconeVectorDbProducer.java
@@ -224,10 +224,12 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
in.setHeader(PineconeVectorDbHeaders.INDEX_ID, indexId);
}
String namespace = in.getHeader(PineconeVectorDbHeaders.NAMESPACE,
String.class);
+ // such as the text_segment the pinecone:embeddings transformer sets,
which the pinecone:rag transformer reads
+ Struct metadata = in.getHeader(PineconeVectorDbHeaders.METADATA,
Struct.class);
Index index = this.client.getIndexConnection(indexName);
- UpsertResponse result = index.upsert(indexId, elements, null, null,
null, namespace);
+ UpsertResponse result = index.upsert(indexId, elements, null, null,
metadata, namespace);
populateUpsertResponse(result, exchange);
@@ -243,9 +245,11 @@ public class PineconeVectorDbProducer extends
DefaultProducer {
}
String indexId = in.getHeader(PineconeVectorDbHeaders.INDEX_ID,
String.class);
String namespace = in.getHeader(PineconeVectorDbHeaders.NAMESPACE,
String.class);
+ // the fields of the metadata to set, such as the text_segment the
pinecone:embeddings transformer sets
+ Struct metadata = in.getHeader(PineconeVectorDbHeaders.METADATA,
Struct.class);
Index index = this.client.getIndexConnection(indexName);
- UpdateResponse result = index.update(indexId, elements, namespace);
+ UpdateResponse result = index.update(indexId, elements, metadata,
namespace, null, null);
populateUpdateResponse(result, exchange);
diff --git
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformer.java
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformer.java
index 7c8dc627c3dc..4d0c6075a4e4 100644
---
a/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformer.java
+++
b/components/camel-ai/camel-pinecone/src/main/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformer.java
@@ -19,7 +19,10 @@ package org.apache.camel.component.pinecone.transform;
import java.util.UUID;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import dev.langchain4j.data.embedding.Embedding;
+import dev.langchain4j.data.segment.TextSegment;
import org.apache.camel.Message;
import org.apache.camel.ai.CamelLangchain4jAttributes;
import org.apache.camel.component.pinecone.PineconeVectorDbHeaders;
@@ -37,21 +40,40 @@ import org.apache.camel.spi.Transformer;
@DataTypeTransformer(name = "pinecone:embeddings",
description = "Prepares the message to become an object
writable by Pinecone component")
@BodyTypes(transformerIn = @BodyType(value = Carrier.ANY,
- description = "The body is not read: the
vector comes from the"
- + "
CamelLangChain4jEmbeddingsVector header, which must be set, and the"
- + " text is not stored"),
+ description = "A LangChain4j TextSegment,
or a String the converter of"
+ + "
camel-langchain4j-embeddings turns into one, whose text goes in the"
+ + " text_segment field of
the CamelPineconeMetadata header, which UPSERT"
+ + " and UPDATE store with
the vector; another body stores no text. The"
+ + " vector comes from the
CamelLangChain4jEmbeddingsVector header, which"
+ + " must be set"),
transformerOut = @BodyType(value = Carrier.LIST, javaType =
"java.util.List<java.lang.Float>",
description = "The vector of the
embedding"))
public class PineconeEmbeddingsDataTypeTransformer extends Transformer {
+ // the metadata field of the text, which the pinecone:rag transformer and
the Pinecone store of LangChain4j read
+ private static final String TEXT_SEGMENT = "text_segment";
+
@Override
public void transform(Message message, DataType fromType, DataType toType)
{
Embedding embedding =
message.getHeader(CamelLangchain4jAttributes.CAMEL_LANGCHAIN4J_EMBEDDING_VECTOR,
Embedding.class);
+ if (embedding == null) {
+ throw new IllegalArgumentException(
+ "Missing embedding vector header '" +
CamelLangchain4jAttributes.CAMEL_LANGCHAIN4J_EMBEDDING_VECTOR
+ + "'. Ensure the
langchain4j-embeddings component is called before this transformer.");
+ }
+ TextSegment text = message.getBody(TextSegment.class);
String indexId = message.getHeader(PineconeVectorDbHeaders.INDEX_ID,
UUID.randomUUID(), String.class);
String indexName =
message.getHeader(PineconeVectorDbHeaders.INDEX_NAME, "embeddings",
String.class);
message.setHeader(PineconeVectorDbHeaders.INDEX_NAME, indexName);
message.setHeader(PineconeVectorDbHeaders.INDEX_ID, indexId);
+ if (text != null) {
+ // added to the metadata the caller may have set, which is kept
+ Struct metadata =
message.getHeader(PineconeVectorDbHeaders.METADATA, Struct.class);
+ Struct.Builder builder = metadata != null ? metadata.toBuilder() :
Struct.newBuilder();
+ builder.putFields(TEXT_SEGMENT,
Value.newBuilder().setStringValue(text.text()).build());
+ message.setHeader(PineconeVectorDbHeaders.METADATA,
builder.build());
+ }
message.setBody(embedding.vectorAsList());
}
}
diff --git
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
index 9fa7f5f6ab5d..8213f1a4766d 100644
---
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
+++
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/PineconeVectorDbProducerTest.java
@@ -21,7 +21,10 @@ import java.math.BigDecimal;
import java.net.InetSocketAddress;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Stream;
@@ -30,6 +33,8 @@ import com.google.protobuf.Struct;
import com.google.protobuf.Value;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpServer;
+import dev.langchain4j.data.embedding.Embedding;
+import dev.langchain4j.data.segment.TextSegment;
import io.grpc.Server;
import io.grpc.ServerBuilder;
import io.grpc.stub.StreamObserver;
@@ -40,11 +45,13 @@ import io.pinecone.proto.UpdateRequest;
import io.pinecone.proto.UpdateResponse;
import io.pinecone.proto.UpsertRequest;
import io.pinecone.proto.UpsertResponse;
+import io.pinecone.proto.Vector;
import io.pinecone.proto.VectorServiceGrpc;
import io.pinecone.unsigned_indices_model.QueryResponseWithUnsignedIndices;
import org.apache.camel.CamelContext;
import org.apache.camel.Exchange;
import org.apache.camel.InvalidPayloadException;
+import org.apache.camel.ai.CamelLangchain4jAttributes;
import org.apache.camel.builder.RouteBuilder;
import org.apache.camel.spi.DataType;
import org.apache.camel.test.junit6.CamelTestSupport;
@@ -67,6 +74,9 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
* header, UPSERT without the CamelPineconeIndexId header stores the vector
under a random id, and the pinecone:rag data
* type transformer reads the matches of a QUERY. A fake Pinecone, its control
plane on a JDK HttpServer and the data
* plane of the index on a gRPC server, records the requests of the client, so
no Pinecone is needed.
+ * <p>
+ * CAMEL-26300: UPSERT and UPDATE store the CamelPineconeMetadata header, in
which the pinecone:embeddings data type
+ * transformer puts the text that the pinecone:rag transformer reads.
*/
class PineconeVectorDbProducerTest extends CamelTestSupport {
@@ -75,15 +85,19 @@ class PineconeVectorDbProducerTest extends CamelTestSupport
{
// the client keeps a connection per index name for the JVM, so each test
has its own index
private final String indexName = "camel-" + UUID.randomUUID();
private final List<Object> requests = new CopyOnWriteArrayList<>();
+ // the vectors of the fake index by id, which UPSERT stores and QUERY
returns as its matches
+ private final Map<String, Vector> vectors =
Collections.synchronizedMap(new LinkedHashMap<>());
private HttpServer controlPlane;
private Server dataPlane;
@Override
protected CamelContext createCamelContext() throws Exception {
+ vectors.put("vector-1",
Vector.newBuilder().setId("vector-1").addAllValues(VECTOR).setMetadata(text("hello")).build());
dataPlane = ServerBuilder.forPort(0).addService(new
VectorServiceGrpc.VectorServiceImplBase() {
@Override
public void upsert(UpsertRequest request,
StreamObserver<UpsertResponse> reply) {
requests.add(request);
+ request.getVectorsList().forEach(vector ->
vectors.put(vector.getId(), vector));
reply.onNext(UpsertResponse.newBuilder().setUpsertedCount(request.getVectorsCount()).build());
reply.onCompleted();
}
@@ -98,12 +112,12 @@ class PineconeVectorDbProducerTest extends
CamelTestSupport {
@Override
public void query(QueryRequest request,
StreamObserver<QueryResponse> reply) {
requests.add(request);
- Struct metadata = Struct.newBuilder()
- .putFields("text_segment",
Value.newBuilder().setStringValue("hello").build())
- .build();
- reply.onNext(QueryResponse.newBuilder()
-
.addMatches(ScoredVector.newBuilder().setId("vector-1").setScore(0.9f).setMetadata(metadata))
- .build());
+ QueryResponse.Builder response = QueryResponse.newBuilder();
+ synchronized (vectors) {
+ vectors.values().forEach(vector -> response.addMatches(
+
ScoredVector.newBuilder().setId(vector.getId()).setScore(0.9f).setMetadata(vector.getMetadata())));
+ }
+ reply.onNext(response.build());
reply.onCompleted();
}
}).build().start();
@@ -129,6 +143,9 @@ class PineconeVectorDbProducerTest extends CamelTestSupport
{
from("direct:rag")
.to("pinecone:rag?action=QUERY")
.transformDataType(new DataType("pinecone:rag"));
+ from("direct:embeddings")
+ .transformDataType(new DataType("pinecone:embeddings"))
+ .to("pinecone:embeddings?action=UPSERT");
}
};
}
@@ -161,6 +178,8 @@ class PineconeVectorDbProducerTest extends CamelTestSupport
{
UpsertRequest upsert = assertInstanceOf(UpsertRequest.class,
singleRequest());
assertEquals("vector-1", upsert.getVectors(0).getId());
assertEquals(VECTOR, upsert.getVectors(0).getValuesList());
+ // no metadata without the CamelPineconeMetadata header
+ assertFalse(upsert.getVectors(0).hasMetadata());
}
@ParameterizedTest
@@ -175,6 +194,8 @@ class PineconeVectorDbProducerTest extends CamelTestSupport
{
UpdateRequest update = assertInstanceOf(UpdateRequest.class,
singleRequest());
assertEquals("vector-1", update.getId());
assertEquals(VECTOR, update.getValuesList());
+ // no metadata without the CamelPineconeMetadata header
+ assertFalse(update.hasSetMetadata());
}
@ParameterizedTest
@@ -240,6 +261,52 @@ class PineconeVectorDbProducerTest extends
CamelTestSupport {
assertEquals(List.of("hello"), texts);
}
+ @Test
+ void upsertAndUpdateStoreTheMetadataHeader() {
+ // CAMEL-26300: UPSERT stores the CamelPineconeMetadata header as the
metadata of the vector, and UPDATE sets its
+ // fields in the metadata of the vector
+ Struct metadata = text("hello");
+ Exchange upsert = fluentTemplate.to("pinecone:test?action=UPSERT")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withHeader(PineconeVectorDbHeaders.METADATA, metadata)
+ .withBody(VECTOR)
+ .request(Exchange.class);
+ Exchange update = fluentTemplate.to("pinecone:test?action=UPDATE")
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withHeader(PineconeVectorDbHeaders.METADATA, metadata)
+ .withBody(VECTOR)
+ .request(Exchange.class);
+
+ assertNull(upsert.getException());
+ assertNull(update.getException());
+ assertEquals(2, requests.size(), requests::toString);
+ assertEquals(metadata, assertInstanceOf(UpsertRequest.class,
requests.get(0)).getVectors(0).getMetadata());
+ assertEquals(metadata, assertInstanceOf(UpdateRequest.class,
requests.get(1)).getSetMetadata());
+ }
+
+ @Test
+ void theRagTransformerGivesTheTextTheEmbeddingsTransformerStored() {
+ // CAMEL-26300: the pinecone:embeddings transformer puts the text in
the text_segment metadata that UPSERT stores,
+ // which the pinecone:rag transformer reads from the matches of a
QUERY. Before, the text was not stored, and the
+ // match of the vector gave an empty String.
+ Exchange upsert = fluentTemplate.to("direct:embeddings")
+
.withHeader(CamelLangchain4jAttributes.CAMEL_LANGCHAIN4J_EMBEDDING_VECTOR,
+ new Embedding(new float[] { 0.1f, 0.2f, 0.3f }))
+ // the index of the endpoint, as the transformer gives the
embeddings index without this header
+ .withHeader(PineconeVectorDbHeaders.INDEX_NAME, indexName)
+ .withHeader(PineconeVectorDbHeaders.INDEX_ID, "vector-1")
+ .withBody(TextSegment.from("the source passage"))
+ .request(Exchange.class);
+ assertNull(upsert.getException());
+
+ Object texts = fluentTemplate.to("direct:rag")
+ .withHeader(PineconeVectorDbHeaders.QUERY_INCLUDE_METADATA,
true)
+ .withBody(VECTOR)
+ .request();
+
+ assertEquals(List.of("the source passage"), texts);
+ }
+
@Test
void aBodyThatIsNotAVectorFails() {
Exchange text = fluentTemplate.to("pinecone:test?action=UPSERT")
@@ -271,6 +338,10 @@ class PineconeVectorDbProducerTest extends
CamelTestSupport {
assertEquals(List.of(), requests);
}
+ private static Struct text(String text) {
+ return Struct.newBuilder().putFields("text_segment",
Value.newBuilder().setStringValue(text).build()).build();
+ }
+
private Object singleRequest() {
assertEquals(1, requests.size(), requests::toString);
return requests.get(0);
diff --git
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformerTest.java
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformerTest.java
index dce686ced532..da84b739d659 100644
---
a/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformerTest.java
+++
b/components/camel-ai/camel-pinecone/src/test/java/org/apache/camel/component/pinecone/transform/PineconeEmbeddingsDataTypeTransformerTest.java
@@ -19,20 +19,30 @@ package org.apache.camel.component.pinecone.transform;
import java.util.Arrays;
import java.util.List;
+import com.google.protobuf.Struct;
+import com.google.protobuf.Value;
import dev.langchain4j.data.embedding.Embedding;
import dev.langchain4j.data.segment.TextSegment;
import org.apache.camel.Exchange;
+import org.apache.camel.Message;
import org.apache.camel.ai.CamelLangchain4jAttributes;
+import org.apache.camel.component.pinecone.PineconeVectorDbHeaders;
import org.apache.camel.impl.DefaultCamelContext;
import org.apache.camel.spi.BodyType.Carrier;
import org.apache.camel.spi.BodyTypes;
import org.apache.camel.spi.DataType;
import org.apache.camel.support.DefaultExchange;
+import org.apache.camel.support.SimpleTypeConverter;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
class PineconeEmbeddingsDataTypeTransformerTest {
@@ -40,28 +50,76 @@ class PineconeEmbeddingsDataTypeTransformerTest {
private final PineconeEmbeddingsDataTypeTransformer transformer = new
PineconeEmbeddingsDataTypeTransformer();
+ @BeforeEach
+ void startContext() {
+ // the converter camel-langchain4j-embeddings brings, which is not on
the classpath of this module
+
camelContext.getTypeConverterRegistry().addTypeConverter(TextSegment.class,
String.class,
+ new SimpleTypeConverter(false, (type, exchange, value) ->
TextSegment.from((String) value)));
+ camelContext.start();
+ }
+
+ @AfterEach
+ void stopContext() {
+ camelContext.stop();
+ }
+
@Test
void theBodyTypesSayTheBody() throws Exception {
- // CAMEL-25928: any body, as transformerIn says, as the vector comes
from a header, and the List of Float
- // transformerOut says
+ // CAMEL-25928: any body, as transformerIn says, of which a
TextSegment or a String is stored as the text, with the
+ // vector in a header, and the List of Float transformerOut says
BodyTypes bodyTypes =
PineconeEmbeddingsDataTypeTransformer.class.getAnnotation(BodyTypes.class);
assertArrayEquals(new Carrier[] { Carrier.ANY },
bodyTypes.transformerIn().value());
assertArrayEquals(new Carrier[] { Carrier.LIST },
bodyTypes.transformerOut().value());
assertEquals("java.util.List<java.lang.Float>",
bodyTypes.transformerOut().javaType());
for (Object body : Arrays.asList(TextSegment.from("hello"), "hello",
5, null)) {
- Exchange exchange = new DefaultExchange(camelContext);
-
exchange.getMessage().setHeader(CamelLangchain4jAttributes.CAMEL_LANGCHAIN4J_EMBEDDING_VECTOR,
- new Embedding(new float[] { 0.1f, 0.2f, 0.3f }));
- exchange.getMessage().setBody(body);
- transformer.transform(exchange.getMessage(), DataType.ANY,
DataType.ANY);
- assertEquals(List.of(0.1f, 0.2f, 0.3f),
exchange.getMessage().getBody(), String.valueOf(body));
+ Message message = transform(body, new Embedding(new float[] {
0.1f, 0.2f, 0.3f }));
+ assertEquals(List.of(0.1f, 0.2f, 0.3f), message.getBody(),
String.valueOf(body));
+ Struct metadata =
message.getHeader(PineconeVectorDbHeaders.METADATA, Struct.class);
+ if (body instanceof TextSegment || body instanceof String) {
+ // CAMEL-26300: the text, in the text_segment metadata the
pinecone:rag transformer reads
+ assertNotNull(metadata, String.valueOf(body));
+ assertEquals("hello",
metadata.getFieldsOrThrow("text_segment").getStringValue(),
String.valueOf(body));
+ } else {
+ // another body stores no text
+ assertNull(metadata, String.valueOf(body));
+ }
}
+ }
+
+ @Test
+ void theTextIsAddedToTheMetadataOfTheHeader() throws Exception {
+ // CAMEL-26300: the metadata the caller set is kept
+ Exchange exchange = new DefaultExchange(camelContext);
+
exchange.getMessage().setHeader(CamelLangchain4jAttributes.CAMEL_LANGCHAIN4J_EMBEDDING_VECTOR,
+ new Embedding(new float[] { 0.1f, 0.2f, 0.3f }));
+ exchange.getMessage().setHeader(PineconeVectorDbHeaders.METADATA,
Struct.newBuilder()
+ .putFields("source",
Value.newBuilder().setStringValue("doc.txt").build())
+ .build());
+ exchange.getMessage().setBody(TextSegment.from("hello"));
+ transformer.transform(exchange.getMessage(), DataType.ANY,
DataType.ANY);
+
+ Struct metadata =
exchange.getMessage().getHeader(PineconeVectorDbHeaders.METADATA, Struct.class);
+ assertEquals(2, metadata.getFieldsCount(), metadata::toString);
+ assertEquals("doc.txt",
metadata.getFieldsOrThrow("source").getStringValue());
+ assertEquals("hello",
metadata.getFieldsOrThrow("text_segment").getStringValue());
+ }
- // a message without the embedding fails
+ @Test
+ void aMessageWithoutTheEmbeddingFailsWithAClearMessage() {
+ // CAMEL-26300: an IllegalArgumentException that names the missing
header, as pgvector:embeddings gives, instead of
+ // a NullPointerException
+ IllegalArgumentException noEmbedding =
assertThrows(IllegalArgumentException.class, () -> transform("hello", null));
+
assertTrue(noEmbedding.getMessage().contains("CamelLangChain4jEmbeddingsVector"),
noEmbedding.getMessage());
+ }
+
+ private Message transform(Object body, Embedding embedding) throws
Exception {
Exchange exchange = new DefaultExchange(camelContext);
- exchange.getMessage().setBody("hello");
- assertThrows(NullPointerException.class,
- () -> transformer.transform(exchange.getMessage(),
DataType.ANY, DataType.ANY));
+ if (embedding != null) {
+
exchange.getMessage().setHeader(CamelLangchain4jAttributes.CAMEL_LANGCHAIN4J_EMBEDDING_VECTOR,
embedding);
+ }
+ exchange.getMessage().setBody(body);
+ transformer.transform(exchange.getMessage(), DataType.ANY,
DataType.ANY);
+ return exchange.getMessage();
}
}
diff --git
a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
index 4c5b28b9e0d2..9721138d273a 100644
--- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
+++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc
@@ -3646,6 +3646,18 @@ it stayed started without receiving any event. The
events of the time without a
`docker:events` or `docker:stats` consumer now closes its stream: before, the
stream stayed open, and a route that was
stopped and started again received every event once more for each restart.
+=== camel-pinecone - the pinecone:embeddings transformer stores the text
+
+The `pinecone:embeddings` data type transformer now reads the message body.
The text of a LangChain4j `TextSegment`,
+or of a `String` that camel-langchain4j-embeddings converts to one, goes in
the `text_segment` field of the new
+`CamelPineconeMetadata` header, which UPSERT stores as the metadata of the
vector and whose fields UPDATE sets, so the
+`pinecone:rag` transformer now gives the text back. Before, the text was not
stored, and `pinecone:rag` gave an empty
+String for the match of such a vector.
+
+The `pinecone:embeddings` transformer now fails with an
`IllegalArgumentException` that names the
+`CamelLangChain4jEmbeddingsVector` header when the message does not have it,
as `pgvector:embeddings` does. Before, it
+failed with a `NullPointerException`.
+
=== camel-docker - the documented default of the socket option is false
The `socket` option, and the `CamelDockerSocketEnabled` header, were
documented with the default `true`, but their
diff --git
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
index 4c64d5d39b6f..e66eea40e14a 100644
---
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
+++
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/PineconeVectorDbEndpointBuilderFactory.java
@@ -598,6 +598,21 @@ public interface PineconeVectorDbEndpointBuilderFactory {
public String pineconeQueryIncludeMetadata() {
return "CamelPineconeQueryIncludeMetadata";
}
+ /**
+ * The metadata UPSERT stores with the vector, and the fields of the
+ * metadata UPDATE sets. The pinecone:embeddings data type transformer
+ * puts the text of the embedding in its text_segment field, which the
+ * pinecone:rag data type transformer reads.
+ *
+ * The option is a: {@code com.google.protobuf.Struct} type.
+ *
+ * Group: producer
+ *
+ * @return the name of the header {@code PineconeMetadata}.
+ */
+ public String pineconeMetadata() {
+ return "CamelPineconeMetadata";
+ }
}
static PineconeVectorDbEndpointBuilder endpointBuilder(String
componentName, String path) {
class PineconeVectorDbEndpointBuilderImpl extends
AbstractEndpointBuilder implements PineconeVectorDbEndpointBuilder,
AdvancedPineconeVectorDbEndpointBuilder {
