This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 2de4d6d1dddd CAMEL-24753: camel-langchain4j-ingest - filter options
2de4d6d1dddd is described below
commit 2de4d6d1ddddeb84470a1bd1d06c9ea4253ada96
Author: Jiří Ondrušek <[email protected]>
AuthorDate: Thu Sep 17 21:23:22 2026 +0200
CAMEL-24753: camel-langchain4j-ingest - filter options
Not everything a consumer delivers belongs in a knowledge base. Adds
filter options to the langchain4j-ingest producer, grouped under the
new "filter" label:
- includeId / excludeId: comma-separated Ant-style patterns matched
against the document id, evaluated before the dedup claim and before
the body is read, so a rejected delivery never occupies its id and
never materializes the payload. Exclusion wins over inclusion.
- minDocumentSize: character minimum checked next to the existing
maxDocumentSize guard. Undersized documents answer a FILTERED result
(a data decision) whereas oversized ones still throw (a resource
guard). Validated >= 0 and <= maxDocumentSize at start.
- documentFilter: an org.apache.camel.Predicate bean evaluated with the
body available, after the claim. The predicate runs before the
blank-document check, so it must tolerate an absent body; rejecting
a document releases its claim.
IngestResult gains a FILTERED outcome that is reported like EMPTY.
Closes #26471
Co-Authored-By: Claude Fable 5 <[email protected]>
---
.../catalog/components/langchain4j-ingest.json | 12 +-
.../catalog/docs/langchain4j-ingest-component.adoc | 26 ++-
.../LangChain4jIngestComponentConfigurer.java | 24 +++
.../LangChain4jIngestConfigurationConfigurer.java | 24 +++
.../LangChain4jIngestEndpointConfigurer.java | 24 +++
.../LangChain4jIngestEndpointUriFactory.java | 6 +-
.../langchain4j/ingest/langchain4j-ingest.json | 12 +-
.../main/docs/langchain4j-ingest-component.adoc | 26 ++-
.../component/langchain4j/ingest/IngestResult.java | 4 +-
.../langchain4j/ingest/IngestService.java | 16 +-
.../ingest/LangChain4jIngestConfiguration.java | 78 +++++++++
.../ingest/LangChain4jIngestProducer.java | 88 +++++++++-
.../ingest/LangChain4jIngestFilterTest.java | 186 +++++++++++++++++++++
.../ingest/LangChain4jIngestValidationTest.java | 30 ++++
.../Langchain4jIngestComponentBuilderFactory.java | 85 ++++++++++
.../LangChain4jIngestEndpointBuilderFactory.java | 116 +++++++++++++
16 files changed, 737 insertions(+), 20 deletions(-)
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-ingest.json
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-ingest.json
index d0223f6b0fb2..6fe605a52e3d 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-ingest.json
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/components/langchain4j-ingest.json
@@ -35,7 +35,11 @@
"autowiredEnabled": { "index": 8, "kind": "property", "displayName":
"Autowired Enabled", "group": "advanced", "label": "advanced", "required":
false, "type": "boolean", "javaType": "boolean", "deprecated": false,
"autowired": false, "secret": false, "defaultValue": true, "description":
"Whether autowiring is enabled. This is used for automatic autowiring options
(the option must be marked as autowired) by looking up in the registry to find
if there is a single instance of matching t [...]
"documentSplitter": { "index": 9, "kind": "property", "displayName":
"Document Splitter", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"dev.langchain4j.data.document.DocumentSplitter", "deprecated": false,
"autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The DocumentSplitter
deciding how a docume [...]
"idempotentRepository": { "index": 10, "kind": "property", "displayName":
"Idempotent Repository", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"org.apache.camel.spi.IdempotentRepository", "deprecated": false, "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The IdempotentRepository
remembering a [...]
- "maxDocumentSize": { "index": 11, "kind": "property", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text a [...]
+ "maxDocumentSize": { "index": 11, "kind": "property", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text a [...]
+ "documentFilter": { "index": 12, "kind": "property", "displayName":
"Document Filter", "group": "filter", "label": "filter", "required": false,
"type": "object", "javaType": "org.apache.camel.Predicate", "deprecated":
false, "autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "A Predicate deciding
whether a delivery is ingested, referenced as #b [...]
+ "excludeId": { "index": 13, "kind": "property", "displayName": "Exclude
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns for document ids to skip. Exclusion wins over [...]
+ "includeId": { "index": 14, "kind": "property", "displayName": "Include
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns the document id must match to be ingested. A n [...]
+ "minDocumentSize": { "index": 15, "kind": "property", "displayName": "Min
Document Size", "group": "filter", "label": "filter", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Minimum size of one
document in characters; 0, the default, means no [...]
},
"headers": {
"CamelLangChain4jIngestDocumentId": { "index": 0, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "The stable document id of the ingested
payload, read when the documentIdHeader endpoint option does not name another
header. The CamelLangChain4jIngestDocumentId exchange property, when set, takes
precedence over any header.", "c [...]
@@ -51,6 +55,10 @@
"lazyStartProducer": { "index": 7, "kind": "parameter", "displayName":
"Lazy Start Producer", "group": "producer (advanced)", "label":
"producer,advanced", "required": false, "type": "boolean", "javaType":
"boolean", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": false, "description": "Whether the producer should be started
lazy (on the first message). By starting lazy you can use this to allow
CamelContext and routes to startup in situations where a produc [...]
"documentSplitter": { "index": 8, "kind": "parameter", "displayName":
"Document Splitter", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"dev.langchain4j.data.document.DocumentSplitter", "deprecated": false,
"autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The DocumentSplitter
deciding how a docum [...]
"idempotentRepository": { "index": 9, "kind": "parameter", "displayName":
"Idempotent Repository", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"org.apache.camel.spi.IdempotentRepository", "deprecated": false, "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The IdempotentRepository
remembering a [...]
- "maxDocumentSize": { "index": 10, "kind": "parameter", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text [...]
+ "maxDocumentSize": { "index": 10, "kind": "parameter", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text [...]
+ "documentFilter": { "index": 11, "kind": "parameter", "displayName":
"Document Filter", "group": "filter", "label": "filter", "required": false,
"type": "object", "javaType": "org.apache.camel.Predicate", "deprecated":
false, "autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "A Predicate deciding
whether a delivery is ingested, referenced as # [...]
+ "excludeId": { "index": 12, "kind": "parameter", "displayName": "Exclude
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns for document ids to skip. Exclusion wins over [...]
+ "includeId": { "index": 13, "kind": "parameter", "displayName": "Include
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns the document id must match to be ingested. A [...]
+ "minDocumentSize": { "index": 14, "kind": "parameter", "displayName": "Min
Document Size", "group": "filter", "label": "filter", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Minimum size of one
document in characters; 0, the default, means no [...]
}
}
diff --git
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
index aca861bf4995..b56060446aa9 100644
---
a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
+++
b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/langchain4j-ingest-component.adoc
@@ -96,8 +96,30 @@ replacement.
After ingestion the message body is replaced with the `IngestResult`, so a
request-reply
caller receives the outcome of its call: the pipeline, the document id, the
number of segments
-written and the outcome — `ingested`, `empty` (blank document, nothing
written) or `skipped`
-(duplicate id, see below).
+written and the outcome — `ingested`, `empty` (blank document, nothing
written), `skipped`
+(duplicate id, see below) or `filtered` (rejected by a filter option, see
below).
+
+=== Filtering deliveries
+
+Not everything a consumer delivers belongs in a knowledge base. `includeId`
and `excludeId`
+take comma-separated Ant-style patterns matched against the document id —
exclusion wins —
+and act before the body is read and before the dedup claim, so a rejected
delivery costs
+nothing and never occupies its id. A pattern and an id must agree on a leading
`/`:
+`docs/**` never matches `/docs/guide.md`, so ids derived from absolute paths
need patterns
+starting with `/`. `minDocumentSize` filters documents too short to carry
+retrievable content, and `documentFilter` names a `Predicate` bean evaluated
with the body
+available; both answer `filtered` and release the dedup claim like a blank
document does, so
+a later, acceptable delivery under the same id still ingests. A duplicate id
is answered
+`skipped` before the content filters run, and the predicate runs before the
blank-document
+check — it must tolerate an absent body, and rejecting a blank delivery
answers `filtered`
+rather than `empty`.
+
+[source,java]
+----
+from("file:knowledge-base?noop=true")
+ .to("langchain4j-ingest:docs?documentIdHeader=CamelFileName"
+ +
"&includeId=**/*.md,**/*.txt&excludeId=**/draft-*&minDocumentSize=100");
+----
=== Deduplicating deliveries
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestComponentConfigurer.java
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestComponentConfigurer.java
index 27e12e8a3904..300e1661b56e 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestComponentConfigurer.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestComponentConfigurer.java
@@ -33,6 +33,8 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "autowiredenabled":
case "autowiredEnabled":
target.setAutowiredEnabled(property(camelContext, boolean.class, value));
return true;
case "configuration": target.setConfiguration(property(camelContext,
org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration.class,
value)); return true;
+ case "documentfilter":
+ case "documentFilter":
getOrCreateConfiguration(target).setDocumentFilter(property(camelContext,
org.apache.camel.Predicate.class, value)); return true;
case "documentidheader":
case "documentIdHeader":
getOrCreateConfiguration(target).setDocumentIdHeader(property(camelContext,
java.lang.String.class, value)); return true;
case "documentsplitter":
@@ -43,8 +45,12 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "embeddingModel":
getOrCreateConfiguration(target).setEmbeddingModel(property(camelContext,
dev.langchain4j.model.embedding.EmbeddingModel.class, value)); return true;
case "embeddingstore":
case "embeddingStore":
getOrCreateConfiguration(target).setEmbeddingStore(property(camelContext,
dev.langchain4j.store.embedding.EmbeddingStore.class, value)); return true;
+ case "excludeid":
+ case "excludeId":
getOrCreateConfiguration(target).setExcludeId(property(camelContext,
java.lang.String.class, value)); return true;
case "idempotentrepository":
case "idempotentRepository":
getOrCreateConfiguration(target).setIdempotentRepository(property(camelContext,
org.apache.camel.spi.IdempotentRepository.class, value)); return true;
+ case "includeid":
+ case "includeId":
getOrCreateConfiguration(target).setIncludeId(property(camelContext,
java.lang.String.class, value)); return true;
case "lazystartproducer":
case "lazyStartProducer":
target.setLazyStartProducer(property(camelContext, boolean.class, value));
return true;
case "maxdocumentsize":
@@ -53,6 +59,8 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "maxOverlapSize":
getOrCreateConfiguration(target).setMaxOverlapSize(property(camelContext,
int.class, value)); return true;
case "maxsegmentsize":
case "maxSegmentSize":
getOrCreateConfiguration(target).setMaxSegmentSize(property(camelContext,
int.class, value)); return true;
+ case "mindocumentsize":
+ case "minDocumentSize":
getOrCreateConfiguration(target).setMinDocumentSize(property(camelContext,
int.class, value)); return true;
default: return false;
}
}
@@ -68,6 +76,8 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "autowiredenabled":
case "autowiredEnabled": return boolean.class;
case "configuration": return
org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration.class;
+ case "documentfilter":
+ case "documentFilter": return org.apache.camel.Predicate.class;
case "documentidheader":
case "documentIdHeader": return java.lang.String.class;
case "documentsplitter":
@@ -78,8 +88,12 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "embeddingModel": return
dev.langchain4j.model.embedding.EmbeddingModel.class;
case "embeddingstore":
case "embeddingStore": return
dev.langchain4j.store.embedding.EmbeddingStore.class;
+ case "excludeid":
+ case "excludeId": return java.lang.String.class;
case "idempotentrepository":
case "idempotentRepository": return
org.apache.camel.spi.IdempotentRepository.class;
+ case "includeid":
+ case "includeId": return java.lang.String.class;
case "lazystartproducer":
case "lazyStartProducer": return boolean.class;
case "maxdocumentsize":
@@ -88,6 +102,8 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "maxOverlapSize": return int.class;
case "maxsegmentsize":
case "maxSegmentSize": return int.class;
+ case "mindocumentsize":
+ case "minDocumentSize": return int.class;
default: return null;
}
}
@@ -99,6 +115,8 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "autowiredenabled":
case "autowiredEnabled": return target.isAutowiredEnabled();
case "configuration": return target.getConfiguration();
+ case "documentfilter":
+ case "documentFilter": return
getOrCreateConfiguration(target).getDocumentFilter();
case "documentidheader":
case "documentIdHeader": return
getOrCreateConfiguration(target).getDocumentIdHeader();
case "documentsplitter":
@@ -109,8 +127,12 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "embeddingModel": return
getOrCreateConfiguration(target).getEmbeddingModel();
case "embeddingstore":
case "embeddingStore": return
getOrCreateConfiguration(target).getEmbeddingStore();
+ case "excludeid":
+ case "excludeId": return
getOrCreateConfiguration(target).getExcludeId();
case "idempotentrepository":
case "idempotentRepository": return
getOrCreateConfiguration(target).getIdempotentRepository();
+ case "includeid":
+ case "includeId": return
getOrCreateConfiguration(target).getIncludeId();
case "lazystartproducer":
case "lazyStartProducer": return target.isLazyStartProducer();
case "maxdocumentsize":
@@ -119,6 +141,8 @@ public class LangChain4jIngestComponentConfigurer extends
PropertyConfigurerSupp
case "maxOverlapSize": return
getOrCreateConfiguration(target).getMaxOverlapSize();
case "maxsegmentsize":
case "maxSegmentSize": return
getOrCreateConfiguration(target).getMaxSegmentSize();
+ case "mindocumentsize":
+ case "minDocumentSize": return
getOrCreateConfiguration(target).getMinDocumentSize();
default: return null;
}
}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfigurationConfigurer.java
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfigurationConfigurer.java
index 81d822dbb447..5e62f6cc245f 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfigurationConfigurer.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfigurationConfigurer.java
@@ -23,6 +23,8 @@ public class LangChain4jIngestConfigurationConfigurer extends
org.apache.camel.s
public boolean configure(CamelContext camelContext, Object obj, String
name, Object value, boolean ignoreCase) {
org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration
target =
(org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration)
obj;
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "documentfilter":
+ case "documentFilter": target.setDocumentFilter(property(camelContext,
org.apache.camel.Predicate.class, value)); return true;
case "documentidheader":
case "documentIdHeader":
target.setDocumentIdHeader(property(camelContext, java.lang.String.class,
value)); return true;
case "documentsplitter":
@@ -33,14 +35,20 @@ public class LangChain4jIngestConfigurationConfigurer
extends org.apache.camel.s
case "embeddingModel": target.setEmbeddingModel(property(camelContext,
dev.langchain4j.model.embedding.EmbeddingModel.class, value)); return true;
case "embeddingstore":
case "embeddingStore": target.setEmbeddingStore(property(camelContext,
dev.langchain4j.store.embedding.EmbeddingStore.class, value)); return true;
+ case "excludeid":
+ case "excludeId": target.setExcludeId(property(camelContext,
java.lang.String.class, value)); return true;
case "idempotentrepository":
case "idempotentRepository":
target.setIdempotentRepository(property(camelContext,
org.apache.camel.spi.IdempotentRepository.class, value)); return true;
+ case "includeid":
+ case "includeId": target.setIncludeId(property(camelContext,
java.lang.String.class, value)); return true;
case "maxdocumentsize":
case "maxDocumentSize":
target.setMaxDocumentSize(property(camelContext, int.class, value)); return
true;
case "maxoverlapsize":
case "maxOverlapSize": target.setMaxOverlapSize(property(camelContext,
int.class, value)); return true;
case "maxsegmentsize":
case "maxSegmentSize": target.setMaxSegmentSize(property(camelContext,
int.class, value)); return true;
+ case "mindocumentsize":
+ case "minDocumentSize":
target.setMinDocumentSize(property(camelContext, int.class, value)); return
true;
default: return false;
}
}
@@ -48,6 +56,8 @@ public class LangChain4jIngestConfigurationConfigurer extends
org.apache.camel.s
@Override
public Class<?> getOptionType(String name, boolean ignoreCase) {
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "documentfilter":
+ case "documentFilter": return org.apache.camel.Predicate.class;
case "documentidheader":
case "documentIdHeader": return java.lang.String.class;
case "documentsplitter":
@@ -58,14 +68,20 @@ public class LangChain4jIngestConfigurationConfigurer
extends org.apache.camel.s
case "embeddingModel": return
dev.langchain4j.model.embedding.EmbeddingModel.class;
case "embeddingstore":
case "embeddingStore": return
dev.langchain4j.store.embedding.EmbeddingStore.class;
+ case "excludeid":
+ case "excludeId": return java.lang.String.class;
case "idempotentrepository":
case "idempotentRepository": return
org.apache.camel.spi.IdempotentRepository.class;
+ case "includeid":
+ case "includeId": return java.lang.String.class;
case "maxdocumentsize":
case "maxDocumentSize": return int.class;
case "maxoverlapsize":
case "maxOverlapSize": return int.class;
case "maxsegmentsize":
case "maxSegmentSize": return int.class;
+ case "mindocumentsize":
+ case "minDocumentSize": return int.class;
default: return null;
}
}
@@ -74,6 +90,8 @@ public class LangChain4jIngestConfigurationConfigurer extends
org.apache.camel.s
public Object getOptionValue(Object obj, String name, boolean ignoreCase) {
org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration
target =
(org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration)
obj;
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "documentfilter":
+ case "documentFilter": return target.getDocumentFilter();
case "documentidheader":
case "documentIdHeader": return target.getDocumentIdHeader();
case "documentsplitter":
@@ -84,14 +102,20 @@ public class LangChain4jIngestConfigurationConfigurer
extends org.apache.camel.s
case "embeddingModel": return target.getEmbeddingModel();
case "embeddingstore":
case "embeddingStore": return target.getEmbeddingStore();
+ case "excludeid":
+ case "excludeId": return target.getExcludeId();
case "idempotentrepository":
case "idempotentRepository": return target.getIdempotentRepository();
+ case "includeid":
+ case "includeId": return target.getIncludeId();
case "maxdocumentsize":
case "maxDocumentSize": return target.getMaxDocumentSize();
case "maxoverlapsize":
case "maxOverlapSize": return target.getMaxOverlapSize();
case "maxsegmentsize":
case "maxSegmentSize": return target.getMaxSegmentSize();
+ case "mindocumentsize":
+ case "minDocumentSize": return target.getMinDocumentSize();
default: return null;
}
}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointConfigurer.java
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointConfigurer.java
index 24d1ca23ec15..5c2f4ce5dcba 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointConfigurer.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointConfigurer.java
@@ -23,6 +23,8 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
public boolean configure(CamelContext camelContext, Object obj, String
name, Object value, boolean ignoreCase) {
LangChain4jIngestEndpoint target = (LangChain4jIngestEndpoint) obj;
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "documentfilter":
+ case "documentFilter":
target.getConfiguration().setDocumentFilter(property(camelContext,
org.apache.camel.Predicate.class, value)); return true;
case "documentidheader":
case "documentIdHeader":
target.getConfiguration().setDocumentIdHeader(property(camelContext,
java.lang.String.class, value)); return true;
case "documentsplitter":
@@ -33,8 +35,12 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
case "embeddingModel":
target.getConfiguration().setEmbeddingModel(property(camelContext,
dev.langchain4j.model.embedding.EmbeddingModel.class, value)); return true;
case "embeddingstore":
case "embeddingStore":
target.getConfiguration().setEmbeddingStore(property(camelContext,
dev.langchain4j.store.embedding.EmbeddingStore.class, value)); return true;
+ case "excludeid":
+ case "excludeId":
target.getConfiguration().setExcludeId(property(camelContext,
java.lang.String.class, value)); return true;
case "idempotentrepository":
case "idempotentRepository":
target.getConfiguration().setIdempotentRepository(property(camelContext,
org.apache.camel.spi.IdempotentRepository.class, value)); return true;
+ case "includeid":
+ case "includeId":
target.getConfiguration().setIncludeId(property(camelContext,
java.lang.String.class, value)); return true;
case "lazystartproducer":
case "lazyStartProducer":
target.setLazyStartProducer(property(camelContext, boolean.class, value));
return true;
case "maxdocumentsize":
@@ -43,6 +49,8 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
case "maxOverlapSize":
target.getConfiguration().setMaxOverlapSize(property(camelContext, int.class,
value)); return true;
case "maxsegmentsize":
case "maxSegmentSize":
target.getConfiguration().setMaxSegmentSize(property(camelContext, int.class,
value)); return true;
+ case "mindocumentsize":
+ case "minDocumentSize":
target.getConfiguration().setMinDocumentSize(property(camelContext, int.class,
value)); return true;
default: return false;
}
}
@@ -55,6 +63,8 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
@Override
public Class<?> getOptionType(String name, boolean ignoreCase) {
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "documentfilter":
+ case "documentFilter": return org.apache.camel.Predicate.class;
case "documentidheader":
case "documentIdHeader": return java.lang.String.class;
case "documentsplitter":
@@ -65,8 +75,12 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
case "embeddingModel": return
dev.langchain4j.model.embedding.EmbeddingModel.class;
case "embeddingstore":
case "embeddingStore": return
dev.langchain4j.store.embedding.EmbeddingStore.class;
+ case "excludeid":
+ case "excludeId": return java.lang.String.class;
case "idempotentrepository":
case "idempotentRepository": return
org.apache.camel.spi.IdempotentRepository.class;
+ case "includeid":
+ case "includeId": return java.lang.String.class;
case "lazystartproducer":
case "lazyStartProducer": return boolean.class;
case "maxdocumentsize":
@@ -75,6 +89,8 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
case "maxOverlapSize": return int.class;
case "maxsegmentsize":
case "maxSegmentSize": return int.class;
+ case "mindocumentsize":
+ case "minDocumentSize": return int.class;
default: return null;
}
}
@@ -83,6 +99,8 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
public Object getOptionValue(Object obj, String name, boolean ignoreCase) {
LangChain4jIngestEndpoint target = (LangChain4jIngestEndpoint) obj;
switch (ignoreCase ? name.toLowerCase() : name) {
+ case "documentfilter":
+ case "documentFilter": return
target.getConfiguration().getDocumentFilter();
case "documentidheader":
case "documentIdHeader": return
target.getConfiguration().getDocumentIdHeader();
case "documentsplitter":
@@ -93,8 +111,12 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
case "embeddingModel": return
target.getConfiguration().getEmbeddingModel();
case "embeddingstore":
case "embeddingStore": return
target.getConfiguration().getEmbeddingStore();
+ case "excludeid":
+ case "excludeId": return target.getConfiguration().getExcludeId();
case "idempotentrepository":
case "idempotentRepository": return
target.getConfiguration().getIdempotentRepository();
+ case "includeid":
+ case "includeId": return target.getConfiguration().getIncludeId();
case "lazystartproducer":
case "lazyStartProducer": return target.isLazyStartProducer();
case "maxdocumentsize":
@@ -103,6 +125,8 @@ public class LangChain4jIngestEndpointConfigurer extends
PropertyConfigurerSuppo
case "maxOverlapSize": return
target.getConfiguration().getMaxOverlapSize();
case "maxsegmentsize":
case "maxSegmentSize": return
target.getConfiguration().getMaxSegmentSize();
+ case "mindocumentsize":
+ case "minDocumentSize": return
target.getConfiguration().getMinDocumentSize();
default: return null;
}
}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointUriFactory.java
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointUriFactory.java
index 26be06f15843..3d7678324afd 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointUriFactory.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/generated/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestEndpointUriFactory.java
@@ -24,17 +24,21 @@ public class LangChain4jIngestEndpointUriFactory extends
org.apache.camel.suppor
private static final Set<String> ENDPOINT_IDENTITY_PROPERTY_NAMES;
private static final Map<String, String> MULTI_VALUE_PREFIXES;
static {
- Set<String> props = new HashSet<>(11);
+ Set<String> props = new HashSet<>(15);
+ props.add("documentFilter");
props.add("documentIdHeader");
props.add("documentSplitter");
props.add("embeddingBatchSize");
props.add("embeddingModel");
props.add("embeddingStore");
+ props.add("excludeId");
props.add("idempotentRepository");
+ props.add("includeId");
props.add("lazyStartProducer");
props.add("maxDocumentSize");
props.add("maxOverlapSize");
props.add("maxSegmentSize");
+ props.add("minDocumentSize");
props.add("pipelineName");
PROPERTY_NAMES = Collections.unmodifiableSet(props);
SECRET_PROPERTY_NAMES = Collections.emptySet();
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/ingest/langchain4j-ingest.json
b/components/camel-ai/camel-langchain4j-ingest/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/ingest/langchain4j-ingest.json
index d0223f6b0fb2..6fe605a52e3d 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/ingest/langchain4j-ingest.json
+++
b/components/camel-ai/camel-langchain4j-ingest/src/generated/resources/META-INF/org/apache/camel/component/langchain4j/ingest/langchain4j-ingest.json
@@ -35,7 +35,11 @@
"autowiredEnabled": { "index": 8, "kind": "property", "displayName":
"Autowired Enabled", "group": "advanced", "label": "advanced", "required":
false, "type": "boolean", "javaType": "boolean", "deprecated": false,
"autowired": false, "secret": false, "defaultValue": true, "description":
"Whether autowiring is enabled. This is used for automatic autowiring options
(the option must be marked as autowired) by looking up in the registry to find
if there is a single instance of matching t [...]
"documentSplitter": { "index": 9, "kind": "property", "displayName":
"Document Splitter", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"dev.langchain4j.data.document.DocumentSplitter", "deprecated": false,
"autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The DocumentSplitter
deciding how a docume [...]
"idempotentRepository": { "index": 10, "kind": "property", "displayName":
"Idempotent Repository", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"org.apache.camel.spi.IdempotentRepository", "deprecated": false, "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The IdempotentRepository
remembering a [...]
- "maxDocumentSize": { "index": 11, "kind": "property", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text a [...]
+ "maxDocumentSize": { "index": 11, "kind": "property", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text a [...]
+ "documentFilter": { "index": 12, "kind": "property", "displayName":
"Document Filter", "group": "filter", "label": "filter", "required": false,
"type": "object", "javaType": "org.apache.camel.Predicate", "deprecated":
false, "autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "A Predicate deciding
whether a delivery is ingested, referenced as #b [...]
+ "excludeId": { "index": 13, "kind": "property", "displayName": "Exclude
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns for document ids to skip. Exclusion wins over [...]
+ "includeId": { "index": 14, "kind": "property", "displayName": "Include
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns the document id must match to be ingested. A n [...]
+ "minDocumentSize": { "index": 15, "kind": "property", "displayName": "Min
Document Size", "group": "filter", "label": "filter", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Minimum size of one
document in characters; 0, the default, means no [...]
},
"headers": {
"CamelLangChain4jIngestDocumentId": { "index": 0, "kind": "header",
"displayName": "", "group": "producer", "label": "", "required": false,
"javaType": "String", "deprecated": false, "deprecationNote": "", "autowired":
false, "secret": false, "description": "The stable document id of the ingested
payload, read when the documentIdHeader endpoint option does not name another
header. The CamelLangChain4jIngestDocumentId exchange property, when set, takes
precedence over any header.", "c [...]
@@ -51,6 +55,10 @@
"lazyStartProducer": { "index": 7, "kind": "parameter", "displayName":
"Lazy Start Producer", "group": "producer (advanced)", "label":
"producer,advanced", "required": false, "type": "boolean", "javaType":
"boolean", "deprecated": false, "autowired": false, "secret": false,
"defaultValue": false, "description": "Whether the producer should be started
lazy (on the first message). By starting lazy you can use this to allow
CamelContext and routes to startup in situations where a produc [...]
"documentSplitter": { "index": 8, "kind": "parameter", "displayName":
"Document Splitter", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"dev.langchain4j.data.document.DocumentSplitter", "deprecated": false,
"autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The DocumentSplitter
deciding how a docum [...]
"idempotentRepository": { "index": 9, "kind": "parameter", "displayName":
"Idempotent Repository", "group": "advanced", "label": "advanced", "required":
false, "type": "object", "javaType":
"org.apache.camel.spi.IdempotentRepository", "deprecated": false, "autowired":
false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "The IdempotentRepository
remembering a [...]
- "maxDocumentSize": { "index": 10, "kind": "parameter", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text [...]
+ "maxDocumentSize": { "index": 10, "kind": "parameter", "displayName": "Max
Document Size", "group": "advanced", "label": "advanced", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Maximum size of one
document in characters, applied to the text [...]
+ "documentFilter": { "index": 11, "kind": "parameter", "displayName":
"Document Filter", "group": "filter", "label": "filter", "required": false,
"type": "object", "javaType": "org.apache.camel.Predicate", "deprecated":
false, "autowired": false, "secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "A Predicate deciding
whether a delivery is ingested, referenced as # [...]
+ "excludeId": { "index": 12, "kind": "parameter", "displayName": "Exclude
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns for document ids to skip. Exclusion wins over [...]
+ "includeId": { "index": 13, "kind": "parameter", "displayName": "Include
Id", "group": "filter", "label": "filter", "required": false, "type": "string",
"javaType": "java.lang.String", "deprecated": false, "autowired": false,
"secret": false, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Comma-separated list of
Ant-style patterns the document id must match to be ingested. A [...]
+ "minDocumentSize": { "index": 14, "kind": "parameter", "displayName": "Min
Document Size", "group": "filter", "label": "filter", "required": false,
"type": "integer", "javaType": "int", "deprecated": false, "autowired": false,
"secret": false, "defaultValue": 0, "configurationClass":
"org.apache.camel.component.langchain4j.ingest.LangChain4jIngestConfiguration",
"configurationField": "configuration", "description": "Minimum size of one
document in characters; 0, the default, means no [...]
}
}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
b/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
index aca861bf4995..b56060446aa9 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/docs/langchain4j-ingest-component.adoc
@@ -96,8 +96,30 @@ replacement.
After ingestion the message body is replaced with the `IngestResult`, so a
request-reply
caller receives the outcome of its call: the pipeline, the document id, the
number of segments
-written and the outcome — `ingested`, `empty` (blank document, nothing
written) or `skipped`
-(duplicate id, see below).
+written and the outcome — `ingested`, `empty` (blank document, nothing
written), `skipped`
+(duplicate id, see below) or `filtered` (rejected by a filter option, see
below).
+
+=== Filtering deliveries
+
+Not everything a consumer delivers belongs in a knowledge base. `includeId`
and `excludeId`
+take comma-separated Ant-style patterns matched against the document id —
exclusion wins —
+and act before the body is read and before the dedup claim, so a rejected
delivery costs
+nothing and never occupies its id. A pattern and an id must agree on a leading
`/`:
+`docs/**` never matches `/docs/guide.md`, so ids derived from absolute paths
need patterns
+starting with `/`. `minDocumentSize` filters documents too short to carry
+retrievable content, and `documentFilter` names a `Predicate` bean evaluated
with the body
+available; both answer `filtered` and release the dedup claim like a blank
document does, so
+a later, acceptable delivery under the same id still ingests. A duplicate id
is answered
+`skipped` before the content filters run, and the predicate runs before the
blank-document
+check — it must tolerate an absent body, and rejecting a blank delivery
answers `filtered`
+rather than `empty`.
+
+[source,java]
+----
+from("file:knowledge-base?noop=true")
+ .to("langchain4j-ingest:docs?documentIdHeader=CamelFileName"
+ +
"&includeId=**/*.md,**/*.txt&excludeId=**/draft-*&minDocumentSize=100");
+----
=== Deduplicating deliveries
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestResult.java
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestResult.java
index 0556447f30a1..820caadbf32d 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestResult.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestResult.java
@@ -29,7 +29,9 @@ public record IngestResult(String pipeline, String
documentId, int segmentsWritt
/** Blank document, nothing written. */
EMPTY,
/** Already ingested under the same key, nothing written. */
- SKIPPED;
+ SKIPPED,
+ /** Rejected by includeId/excludeId, minDocumentSize or
documentFilter, nothing written. */
+ FILTERED;
/** The stable wire/log form; not used in this component itself, but
consumed by downstream runtimes. */
public String label() {
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestService.java
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestService.java
index 0a7ef716cad9..a5322d1726f9 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestService.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/IngestService.java
@@ -60,21 +60,25 @@ class IngestService {
private final DocumentSplitter splitter;
private final int embeddingBatchSize;
private final int maxDocumentSize;
+ private final int minDocumentSize;
public IngestService(String pipeline, EmbeddingStore<TextSegment> store,
EmbeddingModel model,
- int maxSegmentSize, int maxOverlapSize, int
embeddingBatchSize, int maxDocumentSize) {
+ int maxSegmentSize, int maxOverlapSize, int
embeddingBatchSize, int maxDocumentSize,
+ int minDocumentSize) {
this(pipeline, store, model,
DocumentSplitters.recursive(maxSegmentSize, maxOverlapSize), embeddingBatchSize,
- maxDocumentSize);
+ maxDocumentSize, minDocumentSize);
}
public IngestService(String pipeline, EmbeddingStore<TextSegment> store,
EmbeddingModel model,
- DocumentSplitter splitter, int embeddingBatchSize,
int maxDocumentSize) {
+ DocumentSplitter splitter, int embeddingBatchSize,
int maxDocumentSize,
+ int minDocumentSize) {
this.pipeline = pipeline;
this.store = store;
this.model = model;
this.splitter = splitter;
this.embeddingBatchSize = embeddingBatchSize;
this.maxDocumentSize = maxDocumentSize;
+ this.minDocumentSize = minDocumentSize;
}
public IngestResult ingest(String documentId, String text) {
@@ -84,6 +88,12 @@ class IngestService {
if (text == null || text.isBlank()) {
return new IngestResult(pipeline, documentId, 0,
IngestResult.Outcome.EMPTY);
}
+ // a benign filter, not a guard: a too-short document answers
FILTERED, which releases
+ // a dedup claim like EMPTY does. Deliberately soft where the max
check below throws -
+ // undersized is a data decision, oversized a resource risk - so do
not align the two
+ if (minDocumentSize > 0 && text.length() < minDocumentSize) {
+ return new IngestResult(pipeline, documentId, 0,
IngestResult.Outcome.FILTERED);
+ }
// the pipeline is whole-document-in-memory by design, so the cap is
the protection
// against oversized - and, on a consumer-fed pipeline, attacker-sized
- payloads. The
// failure releases a dedup claim like any other, so a trimmed
re-delivery still ingests
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfiguration.java
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfiguration.java
index 58cc5975f764..600fb9263529 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfiguration.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestConfiguration.java
@@ -20,6 +20,7 @@ import dev.langchain4j.data.document.DocumentSplitter;
import dev.langchain4j.data.segment.TextSegment;
import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.store.embedding.EmbeddingStore;
+import org.apache.camel.Predicate;
import org.apache.camel.RuntimeCamelException;
import org.apache.camel.spi.Configurer;
import org.apache.camel.spi.IdempotentRepository;
@@ -86,6 +87,39 @@ public class LangChain4jIngestConfiguration implements
Cloneable {
defaultValue = LangChain4jIngestHeaders.DOCUMENT_ID)
private String documentIdHeader = LangChain4jIngestHeaders.DOCUMENT_ID;
+ @UriParam(description = "Comma-separated list of Ant-style patterns the
document id must match to be ingested."
+ + " A non-matching delivery is answered with a
filtered result, before the dedup claim"
+ + " and without reading the body. When not set,
every id is accepted. Matching is"
+ + " case-sensitive, and a pattern only matches an
id that agrees with it on a leading"
+ + " path separator - an id derived from an
absolute path needs a pattern starting with"
+ + " one. See the component documentation for
pattern examples.",
+ label = "filter")
+ private String includeId;
+
+ @UriParam(description = "Comma-separated list of Ant-style patterns for
document ids to skip. Exclusion wins"
+ + " over includeId. A matching delivery is
answered with a filtered result, before the"
+ + " dedup claim and without reading the body.
Matching is case-sensitive, and a pattern"
+ + " only matches an id that agrees with it on a
leading path separator. See the"
+ + " component documentation for pattern examples.",
+ label = "filter")
+ private String excludeId;
+
+ @UriParam(description = "Minimum size of one document in characters; 0,
the default, means no minimum. A"
+ + " shorter document - boilerplate too small to
carry retrievable content - is answered"
+ + " with a filtered result instead of being
written, and releases its dedup claim like"
+ + " a blank one.",
+ defaultValue = "0", label = "filter")
+ private int minDocumentSize;
+
+ @UriParam(description = "A Predicate deciding whether a delivery is
ingested, referenced as #bean:name and"
+ + " evaluated with the message body available. A
rejected delivery is answered with a"
+ + " filtered result and releases its dedup claim.
Runs after the id patterns and after"
+ + " the dedup claim, so a duplicate is answered
skipped without the filter being"
+ + " evaluated. Not looked up by type on purpose -
an application may hold unrelated"
+ + " predicates.",
+ label = "filter")
+ private Predicate documentFilter;
+
@UriParam(description = "The IdempotentRepository remembering already
ingested document ids, referenced as"
+ " #bean:name. When set, a delivery whose id was
already written is answered with a"
+ " skipped result instead of being re-ingested:
first write wins per id. A blank"
@@ -188,6 +222,50 @@ public class LangChain4jIngestConfiguration implements
Cloneable {
this.documentIdHeader = documentIdHeader;
}
+ public String getIncludeId() {
+ return includeId;
+ }
+
+ /**
+ * Sets the Ant-style patterns the document id must match to be ingested.
+ */
+ public void setIncludeId(String includeId) {
+ this.includeId = includeId;
+ }
+
+ public String getExcludeId() {
+ return excludeId;
+ }
+
+ /**
+ * Sets the Ant-style patterns for document ids to skip; exclusion wins
over inclusion.
+ */
+ public void setExcludeId(String excludeId) {
+ this.excludeId = excludeId;
+ }
+
+ public int getMinDocumentSize() {
+ return minDocumentSize;
+ }
+
+ /**
+ * Sets the minimum size of one document in characters; 0 means no minimum.
+ */
+ public void setMinDocumentSize(int minDocumentSize) {
+ this.minDocumentSize = minDocumentSize;
+ }
+
+ public Predicate getDocumentFilter() {
+ return documentFilter;
+ }
+
+ /**
+ * Sets the predicate deciding whether a delivery is ingested.
+ */
+ public void setDocumentFilter(Predicate documentFilter) {
+ this.documentFilter = documentFilter;
+ }
+
public IdempotentRepository getIdempotentRepository() {
return idempotentRepository;
}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
index 2ee5bf1c6ae5..dc68870ede1c 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/main/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestProducer.java
@@ -16,6 +16,7 @@
*/
package org.apache.camel.component.langchain4j.ingest;
+import java.util.Arrays;
import java.util.Set;
import dev.langchain4j.data.segment.TextSegment;
@@ -23,9 +24,11 @@ import dev.langchain4j.model.embedding.EmbeddingModel;
import dev.langchain4j.store.embedding.EmbeddingStore;
import org.apache.camel.CamelContextAware;
import org.apache.camel.Exchange;
+import org.apache.camel.Predicate;
import org.apache.camel.spi.IdempotentRepository;
import org.apache.camel.support.DefaultProducer;
import org.apache.camel.support.service.ServiceHelper;
+import org.apache.camel.util.AntPathMatcher;
/**
* Splits the message body into segments, embeds them in batches and writes
them to the embedding store; the message
@@ -45,6 +48,8 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
private final LangChain4jIngestEndpoint endpoint;
private final LangChain4jIngestConfiguration configuration;
private IngestService service;
+ private String[] includeIds;
+ private String[] excludeIds;
public LangChain4jIngestProducer(LangChain4jIngestEndpoint endpoint) {
super(endpoint);
@@ -90,6 +95,28 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
+
configuration.getMaxDocumentSize() + ")");
}
+ if (configuration.getMinDocumentSize() < 0) {
+ throw new IllegalArgumentException(
+ "Ingestion pipeline '" + pipeline + "': minDocumentSize
must not be negative (got "
+ +
configuration.getMinDocumentSize() + ")");
+ }
+ if (configuration.getMaxDocumentSize() > 0
+ && configuration.getMinDocumentSize() >
configuration.getMaxDocumentSize()) {
+ throw new IllegalArgumentException(
+ "Ingestion pipeline '" + pipeline + "': minDocumentSize
must not exceed maxDocumentSize (got "
+ +
configuration.getMinDocumentSize() + " > "
+ +
configuration.getMaxDocumentSize() + ")");
+ }
+
+ includeIds = parsePatterns(configuration.getIncludeId());
+ excludeIds = parsePatterns(configuration.getExcludeId());
+ if (configuration.getDocumentFilter() != null) {
+ // initialised and started, but - like the repository below -
never stopped: the
+ // bean may be shared, and stopping it here would tear it down
under other users
+
configuration.getDocumentFilter().initPredicate(getEndpoint().getCamelContext());
+ ServiceHelper.startService(configuration.getDocumentFilter());
+ }
+
EmbeddingStore<TextSegment> store
= resolve(EmbeddingStore.class,
configuration.getEmbeddingStore(), "embedding store", "embeddingStore");
EmbeddingModel model
@@ -97,11 +124,12 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
service = configuration.getDocumentSplitter() != null
? new IngestService(
pipeline, store, model,
configuration.getDocumentSplitter(),
- configuration.getEmbeddingBatchSize(),
configuration.getMaxDocumentSize())
+ configuration.getEmbeddingBatchSize(),
configuration.getMaxDocumentSize(),
+ configuration.getMinDocumentSize())
: new IngestService(
pipeline, store, model,
configuration.getMaxSegmentSize(),
configuration.getMaxOverlapSize(),
configuration.getEmbeddingBatchSize(),
- configuration.getMaxDocumentSize());
+ configuration.getMaxDocumentSize(),
configuration.getMinDocumentSize());
IdempotentRepository repository =
configuration.getIdempotentRepository();
if (repository != null) {
@@ -118,9 +146,16 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
public void process(Exchange exchange) throws Exception {
String documentId = resolveDocumentId(exchange);
+ // before the claim and the body read: a filtered document never
occupies its id and
+ // never pays for materialising the payload
+ if (!idAccepted(documentId)) {
+ exchange.getMessage().setBody(filtered(documentId));
+ return;
+ }
+
IdempotentRepository repository =
configuration.getIdempotentRepository();
if (repository == null) {
- exchange.getMessage().setBody(service.ingest(documentId,
exchange.getMessage().getBody(String.class)));
+ exchange.getMessage().setBody(ingestUnlessFiltered(exchange,
documentId));
return;
}
@@ -136,7 +171,7 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
}
IngestResult result;
try {
- result = service.ingest(documentId,
exchange.getMessage().getBody(String.class));
+ result = ingestUnlessFiltered(exchange, documentId);
} catch (Exception e) {
// a failed write must not keep the claim, or the delivery could
never be retried.
// An Error (an OutOfMemoryError, say) is deliberately not caught:
under a VM-level
@@ -151,9 +186,9 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
}
throw e;
}
- if (result.outcome() == IngestResult.Outcome.EMPTY) {
- // a blank document wrote nothing, so it must not keep the claim -
a later, populated
- // delivery under the same id would be answered SKIPPED
+ if (result.outcome() == IngestResult.Outcome.EMPTY || result.outcome()
== IngestResult.Outcome.FILTERED) {
+ // a blank or filtered document wrote nothing, so it must not keep
the claim - a
+ // later delivery under the same id would be answered SKIPPED
repository.remove(exchange, documentId);
} else {
repository.confirm(exchange, documentId);
@@ -161,6 +196,45 @@ public class LangChain4jIngestProducer extends
DefaultProducer {
exchange.getMessage().setBody(result);
}
+ /**
+ * The content filters, in claim scope: the documentFilter predicate sees
the body, and minDocumentSize (inside
+ * {@link IngestService}) needs the text - both run after the dedup claim,
unlike the id patterns.
+ */
+ private IngestResult ingestUnlessFiltered(Exchange exchange, String
documentId) {
+ Predicate filter = configuration.getDocumentFilter();
+ if (filter != null && !filter.matches(exchange)) {
+ return filtered(documentId);
+ }
+ return service.ingest(documentId,
exchange.getMessage().getBody(String.class));
+ }
+
+ private IngestResult filtered(String documentId) {
+ // the endpoint's pipeline name, not service.pipeline(): same value,
but valid even
+ // if a harness invokes the producer before doStart
+ return new IngestResult(endpoint.getPipelineName(), documentId, 0,
IngestResult.Outcome.FILTERED);
+ }
+
+ /** Exclusion wins over inclusion; with includeId set, only matching ids
pass. */
+ private boolean idAccepted(String documentId) {
+ if (AntPathMatcher.INSTANCE.anyMatch(excludeIds, documentId)) {
+ return false;
+ }
+ return includeIds == null ||
AntPathMatcher.INSTANCE.anyMatch(includeIds, documentId);
+ }
+
+ // normalised to null when no pattern remains: an all-blank includeId must
accept
+ // everything, while an empty array handed to anyMatch would reject
everything
+ private static String[] parsePatterns(String patterns) {
+ if (patterns == null || patterns.isBlank()) {
+ return null;
+ }
+ String[] parsed = Arrays.stream(patterns.split(","))
+ .map(String::trim)
+ .filter(pattern -> !pattern.isEmpty())
+ .toArray(String[]::new);
+ return parsed.length == 0 ? null : parsed;
+ }
+
/**
* The exchange property wins over the header: a route that parses
documents captures the id into the property
* before the parse, and a parser (Tika) copies document metadata over the
headers, so a header read here could be
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestFilterTest.java
b/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestFilterTest.java
new file mode 100644
index 000000000000..2942534bdd08
--- /dev/null
+++
b/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestFilterTest.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.langchain4j.ingest;
+
+import dev.langchain4j.data.segment.TextSegment;
+import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore;
+import org.apache.camel.BindToRegistry;
+import org.apache.camel.CamelExecutionException;
+import org.apache.camel.Predicate;
+import org.apache.camel.builder.RouteBuilder;
+import
org.apache.camel.support.processor.idempotent.MemoryIdempotentRepository;
+import org.apache.camel.test.junit6.CamelTestSupport;
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+/**
+ * The filter options: id patterns act before the dedup claim, content filters
release theirs, and a duplicate is
+ * answered before any filter runs.
+ */
+class LangChain4jIngestFilterTest extends CamelTestSupport {
+
+ @BindToRegistry("store")
+ private final InMemoryEmbeddingStore<TextSegment> store = new
InMemoryEmbeddingStore<>();
+
+ @BindToRegistry("model")
+ private final DeterministicEmbeddingModel model = new
DeterministicEmbeddingModel(16);
+
+ @BindToRegistry("register")
+ private final MemoryIdempotentRepository register = new
MemoryIdempotentRepository();
+
+ @BindToRegistry("confidential")
+ private final Predicate confidential
+ = exchange ->
!exchange.getMessage().getBody(String.class).contains("CONFIDENTIAL");
+
+ @BindToRegistry("explosive")
+ private final Predicate explosive = exchange -> {
+ if (exchange.getMessage().getBody(String.class).contains("BOOM")) {
+ throw new IllegalStateException("filter blew up");
+ }
+ return true;
+ };
+
+ @BindToRegistry("requireContent")
+ private final Predicate requireContent = exchange -> {
+ String body = exchange.getMessage().getBody(String.class);
+ return body != null && !body.isBlank();
+ };
+
+ @Override
+ protected RouteBuilder createRouteBuilder() {
+ return new RouteBuilder() {
+ @Override
+ public void configure() {
+
from("direct:globs").to("langchain4j-ingest:globs?includeId=docs/**,*.md&excludeId=**/draft-*"
+ +
"&idempotentRepository=#bean:register");
+
from("direct:min").to("langchain4j-ingest:min?minDocumentSize=25"
+ +
"&idempotentRepository=#bean:register");
+
from("direct:predicate").to("langchain4j-ingest:predicate?documentFilter=#bean:confidential"
+ +
"&idempotentRepository=#bean:register");
+
from("direct:norepo").to("langchain4j-ingest:norepo?documentFilter=#bean:confidential"
+ + "&minDocumentSize=25");
+
from("direct:throwing").to("langchain4j-ingest:throwing?documentFilter=#bean:explosive"
+ +
"&idempotentRepository=#bean:register");
+
from("direct:requireContent").to("langchain4j-ingest:require-content"
+ +
"?documentFilter=#bean:requireContent"
+ +
"&idempotentRepository=#bean:register");
+ }
+ };
+ }
+
+ @Test
+ void idPatternsGovernIngestionWithoutClaiming() {
+ IngestResult matching = ingest("direct:globs", "docs/guide.md", "a
matching document");
+ IngestResult wrongExtension = ingest("direct:globs", "notes.txt", "a
non-matching document");
+ IngestResult excluded = ingest("direct:globs", "docs/draft-plan.md",
"included but excluded");
+
+
assertThat(matching.outcome()).isEqualTo(IngestResult.Outcome.INGESTED);
+
assertThat(wrongExtension.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+ assertThat(wrongExtension.segmentsWritten()).isZero();
+ // exclusion wins over a matching include
+
assertThat(excluded.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+
+ // a filtered id never touched the register
+ assertThat(register.contains("docs/guide.md")).isTrue();
+ assertThat(register.contains("notes.txt")).isFalse();
+ assertThat(register.contains("docs/draft-plan.md")).isFalse();
+ }
+
+ @Test
+ void shortDocumentIsFilteredAndReleasesItsClaim() {
+ IngestResult tooShort = ingest("direct:min", "doc-min", "too short");
+
assertThat(tooShort.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+ assertThat(register.contains("doc-min")).isFalse();
+
+ IngestResult populated = ingest("direct:min", "doc-min", "long enough
to carry retrievable content");
+
assertThat(populated.outcome()).isEqualTo(IngestResult.Outcome.INGESTED);
+ }
+
+ @Test
+ void rejectedByThePredicateReleasesItsClaim() {
+ IngestResult rejected = ingest("direct:predicate", "doc-p1", "the
CONFIDENTIAL relay spec");
+
assertThat(rejected.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+ assertThat(register.contains("doc-p1")).isFalse();
+
+ IngestResult accepted = ingest("direct:predicate", "doc-p1", "the
public relay spec");
+
assertThat(accepted.outcome()).isEqualTo(IngestResult.Outcome.INGESTED);
+ }
+
+ @Test
+ void duplicateIsAnsweredBeforeTheFilterRuns() {
+ IngestResult first = ingest("direct:predicate", "doc-p2", "the public
pump spec");
+ // the claim is held, so the duplicate is SKIPPED - the filter never
sees it
+ IngestResult duplicate = ingest("direct:predicate", "doc-p2", "the
CONFIDENTIAL pump spec");
+
+ assertThat(first.outcome()).isEqualTo(IngestResult.Outcome.INGESTED);
+
assertThat(duplicate.outcome()).isEqualTo(IngestResult.Outcome.SKIPPED);
+ }
+
+ @Test
+ void filtersApplyWithoutARepository() {
+ IngestResult rejected = ingest("direct:norepo", "doc-n1", "the
CONFIDENTIAL relay spec");
+ IngestResult tooShort = ingest("direct:norepo", "doc-n2", "too short");
+ IngestResult accepted = ingest("direct:norepo", "doc-n3", "long enough
to carry retrievable content");
+
+
assertThat(rejected.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+
assertThat(tooShort.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+
assertThat(accepted.outcome()).isEqualTo(IngestResult.Outcome.INGESTED);
+ }
+
+ @Test
+ void blankBodyAnswersEmptyNotFiltered() {
+ // the blank check outranks minDocumentSize on purpose: an empty
delivery is EMPTY
+ IngestResult blank = ingest("direct:min", "doc-blank", " ");
+ assertThat(blank.outcome()).isEqualTo(IngestResult.Outcome.EMPTY);
+ }
+
+ @Test
+ void throwingFilterReleasesItsClaim() {
+ assertThatThrownBy(() -> ingest("direct:throwing", "doc-t1", "BOOM"))
+ .isInstanceOf(CamelExecutionException.class);
+ assertThat(register.contains("doc-t1")).isFalse();
+
+ IngestResult retried = ingest("direct:throwing", "doc-t1", "recovered
content");
+ assertThat(retried.outcome()).isEqualTo(IngestResult.Outcome.INGESTED);
+ }
+
+ /**
+ * The predicate runs before the blank-document check and must tolerate an
absent body: a predicate rejecting a
+ * blank delivery answers FILTERED, one accepting it leaves the blank
check to answer EMPTY. Both release the claim.
+ */
+ @Test
+ void blankBodyMeetsThePredicateFirst() {
+ IngestResult rejected = ingest("direct:requireContent", "doc-b1", "
");
+
assertThat(rejected.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+ assertThat(register.contains("doc-b1")).isFalse();
+
+ IngestResult absent = ingest("direct:requireContent", "doc-b2", null);
+ assertThat(absent.outcome()).isEqualTo(IngestResult.Outcome.FILTERED);
+ assertThat(register.contains("doc-b2")).isFalse();
+
+ // the confidential predicate tolerates and accepts a blank body, so
EMPTY wins
+ IngestResult accepted = ingest("direct:predicate", "doc-b3", " ");
+ assertThat(accepted.outcome()).isEqualTo(IngestResult.Outcome.EMPTY);
+ }
+
+ private IngestResult ingest(String uri, String documentId, String body) {
+ return template.requestBodyAndHeader(uri, body,
+ LangChain4jIngestHeaders.DOCUMENT_ID, documentId,
IngestResult.class);
+ }
+}
diff --git
a/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestValidationTest.java
b/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestValidationTest.java
index daa86ad45ddc..310a4d4dcbf9 100644
---
a/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestValidationTest.java
+++
b/components/camel-ai/camel-langchain4j-ingest/src/test/java/org/apache/camel/component/langchain4j/ingest/LangChain4jIngestValidationTest.java
@@ -91,6 +91,36 @@ class LangChain4jIngestValidationTest {
}
}
+ @Test
+ void negativeMinDocumentSizeFailsTheStart() throws Exception {
+ try (DefaultCamelContext context = new DefaultCamelContext()) {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+
from("direct:in").to("langchain4j-ingest:pipe?minDocumentSize=-1");
+ }
+ });
+
+ assertThatThrownBy(context::start)
+ .hasStackTraceContaining("minDocumentSize must not be
negative");
+ }
+ }
+
+ @Test
+ void minDocumentSizeAboveMaxFailsTheStart() throws Exception {
+ try (DefaultCamelContext context = new DefaultCamelContext()) {
+ context.addRoutes(new RouteBuilder() {
+ @Override
+ public void configure() {
+
from("direct:in").to("langchain4j-ingest:pipe?minDocumentSize=200&maxDocumentSize=100");
+ }
+ });
+
+ assertThatThrownBy(context::start)
+ .hasStackTraceContaining("minDocumentSize must not exceed
maxDocumentSize");
+ }
+ }
+
@Test
void overlapNotSmallerThanSegmentSizeFailsTheStart() throws Exception {
assertBoundsRejected("langchain4j-ingest:pipe?maxSegmentSize=100&maxOverlapSize=100");
diff --git
a/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Langchain4jIngestComponentBuilderFactory.java
b/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Langchain4jIngestComponentBuilderFactory.java
index 09c19fe0773e..5d2c81e2d0b0 100644
---
a/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Langchain4jIngestComponentBuilderFactory.java
+++
b/dsl/camel-componentdsl/src/generated/java/org/apache/camel/builder/component/dsl/Langchain4jIngestComponentBuilderFactory.java
@@ -301,6 +301,87 @@ public interface Langchain4jIngestComponentBuilderFactory {
doSetProperty("maxDocumentSize", maxDocumentSize);
return this;
}
+
+ /**
+ * A Predicate deciding whether a delivery is ingested, referenced as
+ * #bean:name and evaluated with the message body available. A rejected
+ * delivery is answered with a filtered result and releases its dedup
+ * claim. Runs after the id patterns and after the dedup claim, so a
+ * duplicate is answered skipped without the filter being evaluated.
Not
+ * looked up by type on purpose - an application may hold unrelated
+ * predicates.
+ *
+ * The option is a: <code>org.apache.camel.Predicate</code>
+ * type.
+ *
+ * Group: filter
+ *
+ * @param documentFilter the value to set
+ * @return the dsl builder
+ */
+ default Langchain4jIngestComponentBuilder
documentFilter(org.apache.camel.Predicate documentFilter) {
+ doSetProperty("documentFilter", documentFilter);
+ return this;
+ }
+
+ /**
+ * Comma-separated list of Ant-style patterns for document ids to skip.
+ * Exclusion wins over includeId. A matching delivery is answered with
a
+ * filtered result, before the dedup claim and without reading the
body.
+ * Matching is case-sensitive, and a pattern only matches an id that
+ * agrees with it on a leading path separator. See the component
+ * documentation for pattern examples.
+ *
+ * The option is a: <code>java.lang.String</code> type.
+ *
+ * Group: filter
+ *
+ * @param excludeId the value to set
+ * @return the dsl builder
+ */
+ default Langchain4jIngestComponentBuilder excludeId(java.lang.String
excludeId) {
+ doSetProperty("excludeId", excludeId);
+ return this;
+ }
+
+ /**
+ * Comma-separated list of Ant-style patterns the document id must
match
+ * to be ingested. A non-matching delivery is answered with a filtered
+ * result, before the dedup claim and without reading the body. When
not
+ * set, every id is accepted. Matching is case-sensitive, and a pattern
+ * only matches an id that agrees with it on a leading path separator -
+ * an id derived from an absolute path needs a pattern starting with
+ * one. See the component documentation for pattern examples.
+ *
+ * The option is a: <code>java.lang.String</code> type.
+ *
+ * Group: filter
+ *
+ * @param includeId the value to set
+ * @return the dsl builder
+ */
+ default Langchain4jIngestComponentBuilder includeId(java.lang.String
includeId) {
+ doSetProperty("includeId", includeId);
+ return this;
+ }
+
+ /**
+ * Minimum size of one document in characters; 0, the default, means no
+ * minimum. A shorter document - boilerplate too small to carry
+ * retrievable content - is answered with a filtered result instead of
+ * being written, and releases its dedup claim like a blank one.
+ *
+ * The option is a: <code>int</code> type.
+ *
+ * Group: filter
+ *
+ * @param minDocumentSize the value to set
+ * @return the dsl builder
+ */
+ default Langchain4jIngestComponentBuilder minDocumentSize(int
minDocumentSize) {
+ doSetProperty("minDocumentSize", minDocumentSize);
+ return this;
+ }
}
class Langchain4jIngestComponentBuilderImpl
@@ -334,6 +415,10 @@ public interface Langchain4jIngestComponentBuilderFactory {
case "documentSplitter":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setDocumentSplitter((dev.langchain4j.data.document.DocumentSplitter)
value); return true;
case "idempotentRepository":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setIdempotentRepository((org.apache.camel.spi.IdempotentRepository)
value); return true;
case "maxDocumentSize":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setMaxDocumentSize((int) value); return true;
+ case "documentFilter":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setDocumentFilter((org.apache.camel.Predicate) value); return true;
+ case "excludeId":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setExcludeId((java.lang.String) value); return true;
+ case "includeId":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setIncludeId((java.lang.String) value); return true;
+ case "minDocumentSize":
getOrCreateConfiguration((LangChain4jIngestComponent)
component).setMinDocumentSize((int) value); return true;
default: return false;
}
}
diff --git
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/LangChain4jIngestEndpointBuilderFactory.java
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/LangChain4jIngestEndpointBuilderFactory.java
index 2d15229b314b..fe09a62fb394 100644
---
a/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/LangChain4jIngestEndpointBuilderFactory.java
+++
b/dsl/camel-endpointdsl/src/generated/java/org/apache/camel/builder/endpoint/dsl/LangChain4jIngestEndpointBuilderFactory.java
@@ -235,6 +235,122 @@ public interface LangChain4jIngestEndpointBuilderFactory {
doSetProperty("maxSegmentSize", maxSegmentSize);
return this;
}
+ /**
+ * A Predicate deciding whether a delivery is ingested, referenced as
+ * #bean:name and evaluated with the message body available. A rejected
+ * delivery is answered with a filtered result and releases its dedup
+ * claim. Runs after the id patterns and after the dedup claim, so a
+ * duplicate is answered skipped without the filter being evaluated.
Not
+ * looked up by type on purpose - an application may hold unrelated
+ * predicates.
+ *
+ * The option is a: <code>org.apache.camel.Predicate</code> type.
+ *
+ * Group: filter
+ *
+ * @param documentFilter the value to set
+ * @return the dsl builder
+ */
+ default LangChain4jIngestEndpointBuilder
documentFilter(org.apache.camel.Predicate documentFilter) {
+ doSetProperty("documentFilter", documentFilter);
+ return this;
+ }
+ /**
+ * A Predicate deciding whether a delivery is ingested, referenced as
+ * #bean:name and evaluated with the message body available. A rejected
+ * delivery is answered with a filtered result and releases its dedup
+ * claim. Runs after the id patterns and after the dedup claim, so a
+ * duplicate is answered skipped without the filter being evaluated.
Not
+ * looked up by type on purpose - an application may hold unrelated
+ * predicates.
+ *
+ * The option will be converted to a
+ * <code>org.apache.camel.Predicate</code> type.
+ *
+ * Group: filter
+ *
+ * @param documentFilter the value to set
+ * @return the dsl builder
+ */
+ default LangChain4jIngestEndpointBuilder documentFilter(String
documentFilter) {
+ doSetProperty("documentFilter", documentFilter);
+ return this;
+ }
+ /**
+ * Comma-separated list of Ant-style patterns for document ids to skip.
+ * Exclusion wins over includeId. A matching delivery is answered with
a
+ * filtered result, before the dedup claim and without reading the
body.
+ * Matching is case-sensitive, and a pattern only matches an id that
+ * agrees with it on a leading path separator. See the component
+ * documentation for pattern examples.
+ *
+ * The option is a: <code>java.lang.String</code> type.
+ *
+ * Group: filter
+ *
+ * @param excludeId the value to set
+ * @return the dsl builder
+ */
+ default LangChain4jIngestEndpointBuilder excludeId(String excludeId) {
+ doSetProperty("excludeId", excludeId);
+ return this;
+ }
+ /**
+ * Comma-separated list of Ant-style patterns the document id must
match
+ * to be ingested. A non-matching delivery is answered with a filtered
+ * result, before the dedup claim and without reading the body. When
not
+ * set, every id is accepted. Matching is case-sensitive, and a pattern
+ * only matches an id that agrees with it on a leading path separator -
+ * an id derived from an absolute path needs a pattern starting with
+ * one. See the component documentation for pattern examples.
+ *
+ * The option is a: <code>java.lang.String</code> type.
+ *
+ * Group: filter
+ *
+ * @param includeId the value to set
+ * @return the dsl builder
+ */
+ default LangChain4jIngestEndpointBuilder includeId(String includeId) {
+ doSetProperty("includeId", includeId);
+ return this;
+ }
+ /**
+ * Minimum size of one document in characters; 0, the default, means no
+ * minimum. A shorter document - boilerplate too small to carry
+ * retrievable content - is answered with a filtered result instead of
+ * being written, and releases its dedup claim like a blank one.
+ *
+ * The option is a: <code>int</code> type.
+ *
+ * Default: 0
+ * Group: filter
+ *
+ * @param minDocumentSize the value to set
+ * @return the dsl builder
+ */
+ default LangChain4jIngestEndpointBuilder minDocumentSize(int
minDocumentSize) {
+ doSetProperty("minDocumentSize", minDocumentSize);
+ return this;
+ }
+ /**
+ * Minimum size of one document in characters; 0, the default, means no
+ * minimum. A shorter document - boilerplate too small to carry
+ * retrievable content - is answered with a filtered result instead of
+ * being written, and releases its dedup claim like a blank one.
+ *
+ * The option will be converted to a <code>int</code> type.
+ *
+ * Default: 0
+ * Group: filter
+ *
+ * @param minDocumentSize the value to set
+ * @return the dsl builder
+ */
+ default LangChain4jIngestEndpointBuilder minDocumentSize(String
minDocumentSize) {
+ doSetProperty("minDocumentSize", minDocumentSize);
+ return this;
+ }
}
/**
