This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch feature/CAMEL-24237-catalog-docs in repository https://gitbox.apache.org/repos/asf/camel.git
commit b8dbf1cb94748f6a166de104e0483ac26f58536e Author: Claus Ibsen <[email protected]> AuthorDate: Wed Jul 22 13:14:08 2026 +0200 CAMEL-24237: Add tokenizer EIP documentation and skip langchain4j sub-elements Co-Authored-By: Claude Opus 4.6 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../org/apache/camel/catalog/docs.properties | 1 + .../apache/camel/catalog/docs/tokenizer-eip.adoc | 109 +++++++++++++++++++++ .../docs/modules/eips/pages/tokenizer-eip.adoc | 109 +++++++++++++++++++++ .../dsl/jbang/core/commands/tui/CatalogTab.java | 5 +- 4 files changed, 223 insertions(+), 1 deletion(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs.properties b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs.properties index d3fcb4e4c3c9..766005ff6e6d 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs.properties +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs.properties @@ -640,6 +640,7 @@ timer-component to-eip toD-eip tokenize-language +tokenizer-eip topicLoadBalancer-eip tracing transactional-client diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenizer-eip.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenizer-eip.adoc new file mode 100644 index 000000000000..d36a81b504a9 --- /dev/null +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/tokenizer-eip.adoc @@ -0,0 +1,109 @@ += Tokenizer EIP +:doctitle: Tokenizer +:shortname: tokenizer +:description: Tokenizes the message body for AI processing, splitting text into chunks suitable for embedding or LLM input +:since: 4.8 +:supportlevel: Preview +:tabs-sync-option: + +The Tokenizer EIP provides support to tokenize (chunk) larger blocks of text into text segments +that can be used when interacting with LLMs. Tokenization is particularly helpful when used with +https://en.wikipedia.org/wiki/Vector_database[vector databases] to provide better and more contextual search results +for https://en.wikipedia.org/wiki/Retrieval-augmented_generation[retrieval-augmented generation (RAG)]. + +This EIP uses the https://docs.langchain4j.dev/tutorials/rag/#document-splitter[LangChain4j document splitter] +to handle chunking. + +== EIP options + +// eip options: START +include::partial$eip-options.adoc[] +// eip options: END + +== Supported Splitters + +The following type of splitters is supported: + +* By paragraph: using the DSL `tokenizer().byParagraph()` +* By sentence: using the DSL `tokenizer().bySentence()` +* By word: using the DSL `tokenizer().byWord()` +* By line: using the DSL `tokenizer().byLine()` +* By character: using the DSL `tokenizer().byCharacter()` + +== Supported Tokenizers + +The following tokenizers are supported: + +* OpenAI: using `LangChain4jTokenizerDefinition.TokenizerType.OPEN_AI` +* Azure: using `LangChain4jTokenizerDefinition.TokenizerType.AZURE` +* Qwen: using `LangChain4jTokenizerDefinition.TokenizerType.QWEN` + +== Example + +The tokenization creates a composite message (an array of Strings). This composite message can be split +using the xref:split-eip.adoc[Split EIP] so that each text segment is sent separately to an endpoint. + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .tokenize(tokenizer() + .byParagraph() + .maxSegmentSize(1024) + .maxOverlap(10) + .using(LangChain4jTokenizerDefinition.TokenizerType.OPEN_AI) + .end()) + .split().body() + .to("mock:result"); +---- +==== + +=== Tokenize by tokens + +Starting with Camel 4.12, the code defaults to using segment sizes for determining the maximum amount of data to tokenize. +To tokenize by tokens, you must specify the underlying model: + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .tokenize(tokenizer() + .byParagraph() + .maxTokens(1024, "gpt-4o-mini") + .maxOverlap(10) + .using(LangChain4jTokenizerDefinition.TokenizerType.OPEN_AI) + .end()) + .split().body() + .to("mock:result"); +---- +==== + +== Dependencies + +Maven users will need to add the following dependency: + +[source,xml] +---- +<dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-langchain4j-tokenizer</artifactId> + <version>x.x.x</version> + <!-- use the same version as your Camel core version --> +</dependency> +---- + +The application must also provide the specific tokenizer implementation from LangChain4j: + +* OpenAI: `dev.langchain4j:langchain4j-open-ai` +* Azure: `dev.langchain4j:langchain4j-azure-open-ai` +* Qwen: `dev.langchain4j:langchain4j-dashscope` + +== See Also + +* xref:split-eip.adoc[Split EIP] diff --git a/core/camel-core-engine/src/main/docs/modules/eips/pages/tokenizer-eip.adoc b/core/camel-core-engine/src/main/docs/modules/eips/pages/tokenizer-eip.adoc new file mode 100644 index 000000000000..d36a81b504a9 --- /dev/null +++ b/core/camel-core-engine/src/main/docs/modules/eips/pages/tokenizer-eip.adoc @@ -0,0 +1,109 @@ += Tokenizer EIP +:doctitle: Tokenizer +:shortname: tokenizer +:description: Tokenizes the message body for AI processing, splitting text into chunks suitable for embedding or LLM input +:since: 4.8 +:supportlevel: Preview +:tabs-sync-option: + +The Tokenizer EIP provides support to tokenize (chunk) larger blocks of text into text segments +that can be used when interacting with LLMs. Tokenization is particularly helpful when used with +https://en.wikipedia.org/wiki/Vector_database[vector databases] to provide better and more contextual search results +for https://en.wikipedia.org/wiki/Retrieval-augmented_generation[retrieval-augmented generation (RAG)]. + +This EIP uses the https://docs.langchain4j.dev/tutorials/rag/#document-splitter[LangChain4j document splitter] +to handle chunking. + +== EIP options + +// eip options: START +include::partial$eip-options.adoc[] +// eip options: END + +== Supported Splitters + +The following type of splitters is supported: + +* By paragraph: using the DSL `tokenizer().byParagraph()` +* By sentence: using the DSL `tokenizer().bySentence()` +* By word: using the DSL `tokenizer().byWord()` +* By line: using the DSL `tokenizer().byLine()` +* By character: using the DSL `tokenizer().byCharacter()` + +== Supported Tokenizers + +The following tokenizers are supported: + +* OpenAI: using `LangChain4jTokenizerDefinition.TokenizerType.OPEN_AI` +* Azure: using `LangChain4jTokenizerDefinition.TokenizerType.AZURE` +* Qwen: using `LangChain4jTokenizerDefinition.TokenizerType.QWEN` + +== Example + +The tokenization creates a composite message (an array of Strings). This composite message can be split +using the xref:split-eip.adoc[Split EIP] so that each text segment is sent separately to an endpoint. + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .tokenize(tokenizer() + .byParagraph() + .maxSegmentSize(1024) + .maxOverlap(10) + .using(LangChain4jTokenizerDefinition.TokenizerType.OPEN_AI) + .end()) + .split().body() + .to("mock:result"); +---- +==== + +=== Tokenize by tokens + +Starting with Camel 4.12, the code defaults to using segment sizes for determining the maximum amount of data to tokenize. +To tokenize by tokens, you must specify the underlying model: + +[tabs] +==== +Java:: ++ +[source,java] +---- +from("direct:start") + .tokenize(tokenizer() + .byParagraph() + .maxTokens(1024, "gpt-4o-mini") + .maxOverlap(10) + .using(LangChain4jTokenizerDefinition.TokenizerType.OPEN_AI) + .end()) + .split().body() + .to("mock:result"); +---- +==== + +== Dependencies + +Maven users will need to add the following dependency: + +[source,xml] +---- +<dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-langchain4j-tokenizer</artifactId> + <version>x.x.x</version> + <!-- use the same version as your Camel core version --> +</dependency> +---- + +The application must also provide the specific tokenizer implementation from LangChain4j: + +* OpenAI: `dev.langchain4j:langchain4j-open-ai` +* Azure: `dev.langchain4j:langchain4j-azure-open-ai` +* Qwen: `dev.langchain4j:langchain4j-dashscope` + +== See Also + +* xref:split-eip.adoc[Split EIP] diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CatalogTab.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CatalogTab.java index 113b4a8d269a..18759397d305 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CatalogTab.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/CatalogTab.java @@ -578,7 +578,10 @@ class CatalogTab extends AbstractTableTab { } } - Set<String> skipEips = Set.of("when", "otherwise"); + Set<String> skipEips = Set.of("when", "otherwise", + "langChain4jCharacterTokenizer", "langChain4jLineTokenizer", + "langChain4jParagraphTokenizer", "langChain4jSentenceTokenizer", + "langChain4jWordTokenizer"); for (String name : eipNames) { if (skipEips.contains(name)) { continue;
