This is an automated email from the ASF dual-hosted git repository. luigidemasi pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel.git
commit 75d7e32c61a40e7393c4311eccf1f86c7ec6a90d Author: Luigi De Masi <[email protected]> AuthorDate: Wed Oct 14 23:24:55 2026 +0200 CAMEL-25669: Enforce expert-configured image limits across runtime and tooling Configure maxImageBytes on the expert bean with a 10 MB OpenAI default. Publish the limit in metadata, bound CLI/TUI file reads and route stream caching, and reject oversized inline images before evaluation. Document the validation boundary and cover malformed base64 bodies before inference. Co-authored-by: OpenAI Codex <[email protected]> Signed-off-by: Luigi De Masi <[email protected]> --- .../apache/camel/catalog/docs/semantic-openai.adoc | 18 ++++ .../src/main/docs/semantic-openai.adoc | 18 ++++ .../openai/OpenAIDecisionsSemanticAdapter.java | 16 +++ .../openai/OpenAIDecisionsSemanticAdapterTest.java | 109 +++++++++++++++++++++ .../camel/language/semantic/SemanticLanguage.java | 9 +- .../org/apache/camel/semantic/SemanticAdapter.java | 9 ++ .../org/apache/camel/semantic/SemanticInput.java | 71 +++++++++++++- .../camel/semantic/SemanticMetadataConsole.java | 18 ++-- .../ROOT/pages/camel-4x-upgrade-guide-4_24.adoc | 5 + .../modules/ROOT/pages/camel-jbang-managing.adoc | 9 ++ .../ROOT/pages/camel-jbang-tui-observe.adoc | 7 ++ .../jbang/core/commands/action/CamelSemantic.java | 1 + .../commands/action/SemanticActionCommand.java | 9 ++ .../core/commands/action/SemanticEvaluate.java | 48 ++++++++- .../dsl/jbang/core/common/SemanticMediaInput.java | 80 ++++++++++++++- .../core/commands/action/SemanticCommandTest.java | 77 +++++++++++++-- .../jbang/core/common/SemanticMediaInputTest.java | 44 ++++++++- .../jbang/core/commands/tui/SemanticDetails.java | 3 + .../core/commands/tui/SemanticEvaluationPopup.java | 9 +- .../core/commands/tui/SemanticPlayground.java | 9 +- .../core/commands/tui/SemanticImageViewTest.java | 49 ++++++++- 21 files changed, 587 insertions(+), 31 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc index 40d8325df015..eaeb98334c49 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/semantic-openai.adoc @@ -43,6 +43,7 @@ camel.component.openai.api-key={{env:OPENAI_API_KEY}} type: org.apache.camel.semantic.openai.OpenAIDecisionsSemanticAdapter properties: endpointUri: "openai:decisions?model=gpt-6-luna" + maxImageBytes: 10000000 - semantic: evaluation: damaged: @@ -73,6 +74,15 @@ When constructing the bean in Java, set its `CamelContext` before binding it to Every evaluation must explicitly name an expert. Capability inspection and declaration validation do not perform inference. +`maxImageBytes` is a positive integer configured on the expert bean, defaulting to 10 MB +(10,000,000 bytes) per image before base64 encoding. For example, set `maxImageBytes: 20000000` +to permit images up to 20 MB. Configure it before evaluation. The expert enforces this limit +for route input and direct calls, including inline data URLs and mixed media envelopes. +The CLI and TUI obtain the value from the expert's runtime metadata and check it before +loading local files or submitting image input. There is no separate CLI or TUI setting. +A batch uses the smallest declared image limit among its experts when normalizing a shared body. +Base64 encoding uses additional memory; provider input limits still apply. + == Input Supported modalities are text, images and ordered combinations of text and images. @@ -95,6 +105,9 @@ data:image/png;base64,iVBORw0KGgo... The example is abbreviated. Plain strings without an image content type remain text; Camel does not guess whether an arbitrary string is base64. Content-Type applies when the selected state is `$\{body}`; it does not reclassify a separately selected header or variable. +An image String body must contain base64-encoded bytes, not ordinary text. The Semantic +input helper creates the media envelope; the OpenAI expert validates base64 syntax and +provider input limits before inference, rejecting empty or malformed payloads locally. Streams are cached and reset before evaluation. Downstream steps receive a replayable stream cache containing the same bytes. Byte arrays and strings keep their original body identity. @@ -174,6 +187,11 @@ fits the budget. A descriptor is not a replayable image. Captured content remain `--input=image:/path/picture.png` for a direct expert call. Use `json:` for an ordered envelope. The TUI playground accepts `image:/path/picture.png` and JSON envelopes when the expert advertises image input. Files are read only on an explicit evaluation request. +For local `image:` input, the CLI checks the selected runtime expert's metadata first; +the TUI uses the selected operation's advertised limit. Both check file size and bound the +read using the expert's `maxImageBytes`, including when a file grows during reading. +If an expert does not advertise image support and a positive limit, local image loading +is rejected. These checks do not change the separate audit-preview bounds. The TUI Audit inspector (`u`) renders bounded inline colour previews from retained image data. The CLI renders a plain-terminal preview. Decoding uses locally available ImageIO readers; diff --git a/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc b/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc index 40d8325df015..eaeb98334c49 100644 --- a/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc +++ b/components/camel-ai/camel-semantic-openai/src/main/docs/semantic-openai.adoc @@ -43,6 +43,7 @@ camel.component.openai.api-key={{env:OPENAI_API_KEY}} type: org.apache.camel.semantic.openai.OpenAIDecisionsSemanticAdapter properties: endpointUri: "openai:decisions?model=gpt-6-luna" + maxImageBytes: 10000000 - semantic: evaluation: damaged: @@ -73,6 +74,15 @@ When constructing the bean in Java, set its `CamelContext` before binding it to Every evaluation must explicitly name an expert. Capability inspection and declaration validation do not perform inference. +`maxImageBytes` is a positive integer configured on the expert bean, defaulting to 10 MB +(10,000,000 bytes) per image before base64 encoding. For example, set `maxImageBytes: 20000000` +to permit images up to 20 MB. Configure it before evaluation. The expert enforces this limit +for route input and direct calls, including inline data URLs and mixed media envelopes. +The CLI and TUI obtain the value from the expert's runtime metadata and check it before +loading local files or submitting image input. There is no separate CLI or TUI setting. +A batch uses the smallest declared image limit among its experts when normalizing a shared body. +Base64 encoding uses additional memory; provider input limits still apply. + == Input Supported modalities are text, images and ordered combinations of text and images. @@ -95,6 +105,9 @@ data:image/png;base64,iVBORw0KGgo... The example is abbreviated. Plain strings without an image content type remain text; Camel does not guess whether an arbitrary string is base64. Content-Type applies when the selected state is `$\{body}`; it does not reclassify a separately selected header or variable. +An image String body must contain base64-encoded bytes, not ordinary text. The Semantic +input helper creates the media envelope; the OpenAI expert validates base64 syntax and +provider input limits before inference, rejecting empty or malformed payloads locally. Streams are cached and reset before evaluation. Downstream steps receive a replayable stream cache containing the same bytes. Byte arrays and strings keep their original body identity. @@ -174,6 +187,11 @@ fits the budget. A descriptor is not a replayable image. Captured content remain `--input=image:/path/picture.png` for a direct expert call. Use `json:` for an ordered envelope. The TUI playground accepts `image:/path/picture.png` and JSON envelopes when the expert advertises image input. Files are read only on an explicit evaluation request. +For local `image:` input, the CLI checks the selected runtime expert's metadata first; +the TUI uses the selected operation's advertised limit. Both check file size and bound the +read using the expert's `maxImageBytes`, including when a file grows during reading. +If an expert does not advertise image support and a positive limit, local image loading +is rejected. These checks do not change the separate audit-preview bounds. The TUI Audit inspector (`u`) renders bounded inline colour previews from retained image data. The CLI renders a plain-terminal preview. Decoding uses locally available ImageIO readers; diff --git a/components/camel-ai/camel-semantic-openai/src/main/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapter.java b/components/camel-ai/camel-semantic-openai/src/main/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapter.java index d1c477e9b3fa..b0c36dfe9710 100644 --- a/components/camel-ai/camel-semantic-openai/src/main/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapter.java +++ b/components/camel-ai/camel-semantic-openai/src/main/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapter.java @@ -119,6 +119,7 @@ public class OpenAIDecisionsSemanticAdapter implements SemanticAdapter, CamelCon private static final SemanticCapabilities CAPABILITIES = SemanticCapabilities.from(OpenAIDecisionsSemanticAdapter.class); private CamelContext camelContext; private String endpointUri = "openai:decisions"; + private int maxImageBytes = 10_000_000; private volatile OpenAIEndpoint endpoint; @Override @@ -135,6 +136,19 @@ public class OpenAIDecisionsSemanticAdapter implements SemanticAdapter, CamelCon return endpointUri; } + @Override + public int getMaxImageBytes() { + return maxImageBytes; + } + + /** Maximum bytes per image before base64 encoding. Defaults to 10 MB. Configure before evaluation. */ + public void setMaxImageBytes(int maxImageBytes) { + if (maxImageBytes <= 0) { + throw new IllegalArgumentException("maxImageBytes must be greater than zero"); + } + this.maxImageBytes = maxImageBytes; + } + /** Configured Decisions endpoint, including model and transport options. Set before the first evaluation. */ public synchronized void setEndpointUri(String endpointUri) { if (endpoint != null || endpointUri == null || endpointUri.isBlank()) { @@ -212,6 +226,7 @@ public class OpenAIDecisionsSemanticAdapter implements SemanticAdapter, CamelCon @Override public void validateState(Object state) { + SemanticInput.validateImageSize(SemanticInput.normalize(state), maxImageBytes); OpenAIDecisionsSupport.validateInput(input(SemanticInput.normalize(state))); } @@ -227,6 +242,7 @@ public class OpenAIDecisionsSemanticAdapter implements SemanticAdapter, CamelCon throw new IllegalArgumentException("OpenAI Decisions requires 1 to 200 evaluations per batch"); } Object input = SemanticInput.normalize(state); + SemanticInput.validateImageSize(input, maxImageBytes); evaluations.forEach((name, evaluation) -> { text(name); validate(evaluation); diff --git a/components/camel-ai/camel-semantic-openai/src/test/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapterTest.java b/components/camel-ai/camel-semantic-openai/src/test/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapterTest.java index 0e328d589780..9fb44533ab13 100644 --- a/components/camel-ai/camel-semantic-openai/src/test/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapterTest.java +++ b/components/camel-ai/camel-semantic-openai/src/test/java/org/apache/camel/semantic/openai/OpenAIDecisionsSemanticAdapterTest.java @@ -18,9 +18,11 @@ package org.apache.camel.semantic.openai; import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.InputStream; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import com.openai.client.OpenAIClient; @@ -36,6 +38,7 @@ import org.apache.camel.component.openai.OpenAIDecisionRefusalException; import org.apache.camel.component.openai.OpenAIDecisionServiceException; import org.apache.camel.component.openai.OpenAIEndpoint; import org.apache.camel.component.openai.OpenAIOperations; +import org.apache.camel.console.DevConsole; import org.apache.camel.impl.DefaultCamelContext; import org.apache.camel.language.semantic.SemanticLanguage; import org.apache.camel.semantic.SemanticCapabilities; @@ -45,6 +48,8 @@ import org.apache.camel.semantic.SemanticInput; import org.apache.camel.semantic.SemanticResult; import org.apache.camel.semantic.internal.SemanticAuditExtensions; import org.apache.camel.support.DefaultExchange; +import org.apache.camel.support.PluginHelper; +import org.apache.camel.util.json.JsonObject; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -133,6 +138,97 @@ class OpenAIDecisionsSemanticAdapterTest { verify(client.decisions(), times(5)).create(any(DecisionCreateParams.class)); } + @Test + void imageLimitAppliesToRouteBodiesAndDirectCalls() throws Exception { + assertThat(adapter.getMaxImageBytes()).isEqualTo(10_000_000); + adapter.setMaxImageBytes(2); + Expression expression = expression(evaluation("boolean", Map.of())); + byte[] bytes = { 1, 2, 3 }; + for (Object body : List.of(bytes, new ByteArrayInputStream(bytes), "AQID", "data:image/png;base64,AQID")) { + var exchange = new DefaultExchange(context); + exchange.getMessage().setBody(body); + exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "image/png"); + assertThatThrownBy(() -> expression.evaluate(exchange, Boolean.class)).hasMessageContaining("maxImageBytes"); + } + var media = SemanticInput.ofParts(List.of(SemanticInput.image("data:image/png;base64,AQID"))); + assertThatThrownBy(() -> adapter.evaluate(evaluation("boolean", Map.of()), media)) + .hasMessageContaining("maxImageBytes"); + verify(client.decisions(), never()).create(any(DecisionCreateParams.class)); + adapter.setMaxImageBytes(3); + respond = request -> response(predicate("question", 0.9)); + assertThat(adapter.evaluate(evaluation("boolean", Map.of()), media).getValue()).isEqualTo(true); + adapter.setMaxImageBytes(1); + var padded = SemanticInput.ofParts(List.of(SemanticInput.image("data:image/png;base64,AQ=="))); + assertThat(adapter.evaluate(evaluation("boolean", Map.of()), padded).getValue()).isEqualTo(true); + } + + @Test + void metadataAdvertisesEachExpertConfiguredImageLimitWithoutInference() { + adapter.setMaxImageBytes(3); + var other = new OpenAIDecisionsSemanticAdapter(); + other.setMaxImageBytes(7); + context.getRegistry().bind("other", other); + DevConsole console = PluginHelper.getDevConsoleResolver(context).resolveDevConsole("semantic-metadata"); + for (String reference : List.of("expert", "other")) { + JsonObject response = (JsonObject) console.call(DevConsole.MediaType.JSON, Map.of("expert", reference)); + for (Object value : response.getCollection("operations")) { + assertThat(((JsonObject) value).getJsonObject("contract").getInteger("maxImageBytes")) + .isEqualTo("expert".equals(reference) ? 3 : 7); + } + } + JsonObject overview = (JsonObject) console.call(DevConsole.MediaType.JSON, Map.of("overview", true)); + assertThat(overview.toJson()).contains("maxImageBytes"); + verify(client.decisions(), never()).create(any(DecisionCreateParams.class)); + } + + @Test + void imageLimitMustBePositive() { + assertThatThrownBy(() -> adapter.setMaxImageBytes(0)).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> adapter.setMaxImageBytes(-1)).isInstanceOf(IllegalArgumentException.class); + assertThat(adapter.getMaxImageBytes()).isEqualTo(10_000_000); + } + + @Test + void oversizedStreamsStopBeforeBeingFullyCached() { + adapter.setMaxImageBytes(3); + var consumed = new AtomicInteger(); + var exchange = new DefaultExchange(context); + exchange.getMessage().setBody(new InputStream() { + @Override + public int read() { + if (consumed.incrementAndGet() > 64) { + throw new AssertionError("Image stream was read without a bound"); + } + return 0; + } + }); + exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "image/png"); + Expression expression = expression(evaluation("boolean", Map.of())); + assertThatThrownBy(() -> expression.evaluate(exchange, Boolean.class)).hasStackTraceContaining("maxImageBytes"); + assertThat(consumed.get()).isEqualTo(4); + verify(client.decisions(), never()).create(any(DecisionCreateParams.class)); + } + + @Test + void sharedBodyUsesTheSmallestExpertLimitBeforeAnyBatchInference() { + adapter.setMaxImageBytes(3); + var strict = new OpenAIDecisionsSemanticAdapter(); + strict.setCamelContext(context); + strict.setMaxImageBytes(2); + context.getRegistry().bind("strict", strict); + var evaluations = new LinkedHashMap<String, SemanticEvaluation>(); + evaluations.put("first", evaluation("boolean", Map.of())); + evaluations.put("second", new SemanticEvaluation("boolean", "strict", null, Map.of("instructions", "Assess"))); + SemanticEvaluations.get(context).replace("batch", evaluations); + var expression = context.resolveLanguage("semantic").createExpression("refs:first,second"); + var exchange = new DefaultExchange(context); + exchange.getMessage().setBody(new byte[] { 1, 2, 3 }); + exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "image/png"); + assertThatThrownBy(() -> expression.evaluate(exchange, Map.class)) + .hasMessageContaining("maxImageBytes limit of 2 bytes"); + verify(client.decisions(), never()).create(any(DecisionCreateParams.class)); + } + @Test void selectedTextHeaderDoesNotInheritTheImageBodyContentType() { var evaluation = new SemanticEvaluation("boolean", "expert", "${header.evidence}", Map.of("instructions", "Assess")); @@ -288,6 +384,19 @@ class OpenAIDecisionsSemanticAdapterTest { verify(client.decisions(), never()).create(any(DecisionCreateParams.class)); } + @Test + void malformedBase64ImageBodiesAreRejectedBeforeInference() { + Expression expression = expression(evaluation("boolean", Map.of())); + for (String body : List.of("", " ", "invalid!", "hello world", "not valid image payload\n".repeat(100_000))) { + var exchange = new DefaultExchange(context); + exchange.getMessage().setBody(body); + exchange.getMessage().setHeader(Exchange.CONTENT_TYPE, "image/png"); + assertThatThrownBy(() -> expression.evaluate(exchange, Boolean.class)) + .hasRootCauseInstanceOf(IllegalArgumentException.class).hasMessageContaining("base64"); + } + verify(client.decisions(), never()).create(any(DecisionCreateParams.class)); + } + @Test void requiresAnExplicitModelBeforeInference() { context.getEndpoint("openai:decisions", OpenAIEndpoint.class).getConfiguration().setModel(null); diff --git a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/language/semantic/SemanticLanguage.java b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/language/semantic/SemanticLanguage.java index 19b7edbf8ef2..4bc97b8b3f9e 100644 --- a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/language/semantic/SemanticLanguage.java +++ b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/language/semantic/SemanticLanguage.java @@ -173,6 +173,11 @@ public class SemanticLanguage extends LanguageSupport { return describeExpert(expert).capabilities(); } + /** Read the named expert's configured image limit without performing inference. */ + public int getExpertMaxImageBytes(String expert) { + return expertBean(expertReference(expert)).getMaxImageBytes(); + } + /** The selected registry reference and its static contract. */ public record ExpertMetadata(String reference, SemanticCapabilities capabilities) { } @@ -442,7 +447,9 @@ public class SemanticLanguage extends LanguageSupport { active.addAll(invocations.values()); failureReason = "invalid_input"; Object state = current.body - ? SemanticInput.body(exchange) + ? SemanticInput.body(exchange, + current.groups.stream().mapToInt(g -> g.expert.provider.getMaxImageBytes()) + .filter(limit -> limit > 0).min().orElse(Integer.MAX_VALUE)) : SemanticInput.normalize(current.state.evaluate(exchange, Object.class)); invocations.values().forEach(invocation -> invocation.captureInput(state)); // A state selector can itself evaluate another semantic expression. diff --git a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticAdapter.java b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticAdapter.java index d3158424a767..1604ea712780 100644 --- a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticAdapter.java +++ b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticAdapter.java @@ -29,6 +29,15 @@ import java.util.Map; * does not instantiate, discover or manage provider services. */ public interface SemanticAdapter { + /** + * Configured maximum bytes per image before base64 encoding, or zero when no limit is advertised. Tooling uses this + * value before reading local files; image experts must also enforce it before inference. Inspection must not + * contact services. This is instance configuration, separate from the static operation contract. + */ + default int getMaxImageBytes() { + return 0; + } + /** Reject unsupported evaluation kinds, criteria, decision policies or input selectors before traffic starts. */ void validate(SemanticEvaluation evaluation); diff --git a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticInput.java b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticInput.java index b237100bdbae..4e9efcf8a4fa 100644 --- a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticInput.java +++ b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticInput.java @@ -16,6 +16,8 @@ */ package org.apache.camel.semantic; +import java.io.FilterInputStream; +import java.io.IOException; import java.io.InputStream; import java.util.Base64; import java.util.List; @@ -66,9 +68,15 @@ public final class SemanticInput { /** * Select an image body using Content-Type. Streams become replayable Camel stream caches before reading, so later - * route steps can still consume the body. Headers do not affect explicitly selected headers or variables. + * route steps can still consume the body. Headers do not affect explicitly selected headers or variables. String + * image bodies contain base64 or an image data URL; the selected expert validates the image data and limits. */ public static Object body(Exchange exchange) throws NoTypeConversionAvailableException { + return body(exchange, Integer.MAX_VALUE); + } + + /** Normalize an image body within the selected experts' per-image byte limit. */ + public static Object body(Exchange exchange, int maxImageBytes) throws NoTypeConversionAvailableException { Object input = exchange.getMessage().getBody(); String contentType = exchange.getMessage().getHeader(Exchange.CONTENT_TYPE, String.class); String mediaType = contentType == null ? "" : contentType.split(";", 2)[0].trim().toLowerCase(Locale.ROOT); @@ -81,23 +89,28 @@ public final class SemanticInput { if (!mediaType.matches("image/[a-z0-9.+-]{1,100}")) { throw new IllegalArgumentException("Invalid image Content-Type"); } - if (input instanceof InputStream && !(input instanceof StreamCache)) { - input = exchange.getContext().getTypeConverter().mandatoryConvertTo(StreamCache.class, exchange, input); + if (input instanceof InputStream stream && !(input instanceof StreamCache)) { + input = exchange.getContext().getTypeConverter().mandatoryConvertTo(StreamCache.class, exchange, + limitedImageStream(stream, maxImageBytes)); exchange.getMessage().setBody(input); } if (input instanceof StreamCache cache) { try { cache.reset(); + checkImageSize(cache.length(), maxImageBytes); byte[] data = exchange.getContext().getTypeConverter().mandatoryConvertTo(byte[].class, exchange, cache); + checkImageSize(data.length, maxImageBytes); return ofParts(List.of(image(data, mediaType))); } finally { cache.reset(); } } if (input instanceof byte[] data) { + checkImageSize(data.length, maxImageBytes); return ofParts(List.of(image(data, mediaType))); } if (input instanceof String text) { + checkImageSize(imageBytes(text), maxImageBytes); return text.startsWith("data:image/") ? normalize(text) : ofParts(List.of(image("data:" + mediaType + ";base64," + text))); @@ -108,6 +121,58 @@ public final class SemanticInput { throw new IllegalArgumentException("An image body requires bytes, an InputStream, or a base64 string"); } + /** + * Check decoded image sizes without decoding or duplicating payloads; encoding validation belongs to the expert. + */ + public static void validateImageSize(Object input, int maxImageBytes) { + if (isMedia(input)) { + for (Object item : parts(input)) { + if (item instanceof Map<?, ?> part && "image".equals(part.get("type")) + && part.get("data") instanceof String data) { + checkImageSize(imageBytes(data), maxImageBytes); + } + } + } + } + + private static long imageBytes(String data) { + int start = data.startsWith("data:") ? data.indexOf(';') : -1; + start = start >= 0 && data.startsWith(";base64,", start) ? start + 8 : 0; + int padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; + return (data.length() - start) * 3L / 4 - padding; + } + + private static void checkImageSize(long bytes, int maxImageBytes) { + if (bytes > maxImageBytes) { + throw new IllegalArgumentException("Image exceeds the expert maxImageBytes limit of " + maxImageBytes + " bytes"); + } + } + + private static InputStream limitedImageStream(InputStream stream, int maxImageBytes) { + return new FilterInputStream(stream) { + private long count; + + @Override + public int read() throws IOException { + int value = in.read(); + if (value != -1) { + checkImageSize(++count, maxImageBytes); + } + return value; + } + + @Override + public int read(byte[] bytes, int offset, int length) throws IOException { + int read = in.read(bytes, offset, (int) Math.min(length, maxImageBytes - count + 1)); + if (read > 0) { + count += read; + checkImageSize(count, maxImageBytes); + } + return read; + } + }; + } + public static boolean isMedia(Object input) { return input instanceof Map<?, ?> map && map.containsKey(PARTS); } diff --git a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticMetadataConsole.java b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticMetadataConsole.java index 3b84ab61f114..fbc41df5a293 100644 --- a/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticMetadataConsole.java +++ b/components/camel-ai/camel-semantic/src/main/java/org/apache/camel/semantic/SemanticMetadataConsole.java @@ -29,7 +29,7 @@ import org.apache.camel.spi.annotations.DevConsole; import org.apache.camel.support.console.AbstractDevConsole; import org.apache.camel.util.json.JsonRecordSupport; -/** Static operation metadata and published declarations for runtime tooling. */ +/** Operation metadata, configured input limits and published declarations for runtime tooling. */ @DevConsole(name = "semantic-metadata", displayName = "Semantic Metadata", description = "Semantic experts and evaluation definitions") public class SemanticMetadataConsole extends AbstractDevConsole { @@ -90,7 +90,8 @@ public class SemanticMetadataConsole extends AbstractDevConsole { } List<Operation> operations; try { - operations = operations(language.getExpertCapabilities(optionString(options, EXPERT))); + String reference = optionString(options, EXPERT); + operations = operations(language.getExpertCapabilities(reference), language.getExpertMaxImageBytes(reference)); } catch (IllegalArgumentException unavailable) { operations = List.of(); } @@ -101,7 +102,7 @@ public class SemanticMetadataConsole extends AbstractDevConsole { Map<String, Expert> experts = new TreeMap<>(); getCamelContext().getRegistry().findByTypeWithName(SemanticAdapter.class).keySet().forEach(reference -> { try { - addExpert(experts, language.describeExpert(reference)); + addExpert(experts, language.describeExpert(reference), language.getExpertMaxImageBytes(reference)); } catch (IllegalArgumentException e) { experts.put(reference, new Expert(reference, null, null, null, null, List.of(), e.getMessage())); } @@ -115,7 +116,7 @@ public class SemanticMetadataConsole extends AbstractDevConsole { String error = null; try { var selected = language.describeExpert(reference); - reference = addExpert(experts, selected); + reference = addExpert(experts, selected, language.getExpertMaxImageBytes(selected.reference())); resultType = selected.capabilities().operation(evaluation.getOperation()).getResultType().name() .toLowerCase(Locale.ROOT); } catch (IllegalArgumentException e) { @@ -130,20 +131,23 @@ public class SemanticMetadataConsole extends AbstractDevConsole { return new Response(null, List.copyOf(experts.values()), definitions); } - private static String addExpert(Map<String, Expert> experts, SemanticLanguage.ExpertMetadata metadata) { + private static String addExpert(Map<String, Expert> experts, SemanticLanguage.ExpertMetadata metadata, int maxImageBytes) { SemanticCapabilities capabilities = metadata.capabilities(); experts.putIfAbsent(metadata.reference(), new Expert( metadata.reference(), capabilities.getName(), capabilities.getDescription(), capabilities.getProvider(), capabilities.getArtifactId(), - operations(capabilities), null)); + operations(capabilities, maxImageBytes), null)); return metadata.reference(); } - private static List<Operation> operations(SemanticCapabilities capabilities) { + private static List<Operation> operations(SemanticCapabilities capabilities, int maxImageBytes) { return capabilities.getOperations().values().stream().map(op -> { Map<String, Object> contract = new LinkedHashMap<>(); contract.put("inputTypes", op.getInputTypes().stream().map(type -> type.name().toLowerCase(Locale.ROOT)).sorted().toList()); + if (maxImageBytes > 0 && op.getInputTypes().contains(SemanticExpert.InputType.IMAGE)) { + contract.put("maxImageBytes", maxImageBytes); + } contract.put("inputRequirements", op.getInputRequirements()); contract.put("resultMeaning", op.getResultMeaning()); if (!op.getScoreLevelsParameter().isEmpty()) { diff --git a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc index 01548ec434a0..50374dc80224 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-4x-upgrade-guide-4_24.adoc @@ -743,6 +743,11 @@ all questions for one expert. Existing per-evaluation `validateInput` hooks stil for every expert before any provider performs inference. A shared validation failure marks that expert's group as invalid input in the audit. +The additive `SemanticAdapter.getMaxImageBytes()` method exposes an expert's configured per-image +byte limit, with zero meaning no advertised limit. Image-capable adapters should advertise a +positive limit and enforce it before inference. CLI and TUI local image loading now requires +that metadata; the configured value appears as `maxImageBytes` in each image operation's contract. + Semantic audit backends can opt into the additive `inputType` extension. Readers that do not support it keep their existing projection. The `camel semantic audit --json` display omits image base64 and reports metadata instead. Use `--output=<new-file>` to export the full returned JSON, diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-managing.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-managing.adoc index 43e56e14f43b..bd7ccaabf7a5 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-managing.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-managing.adoc @@ -343,6 +343,15 @@ example `--parameter=threshold=json:0.7`, `--variable=enabled=json:true`, or with `json:` or `file:`. `--body=file:sample.txt` and `--input=file:sample.txt` read the file as UTF-8 text. +For an expert accepting images, `--body=image:/path/picture.png` or +`--input=image:/path/picture.png` reads a local PNG, JPEG, GIF or WebP file. Before reading, +the CLI queries the selected expert's runtime metadata for its `maxImageBytes` limit. +It checks both the file size and the actual read, and checks inline images before submitting +an evaluation. Configure the limit on the expert bean; the OpenAI Decisions expert defaults +to 10 MB (10,000,000 bytes) per image before base64 encoding. Both the CLI and TUI use that +same configuration. Missing image support or limit metadata prevents local image loading; +no evaluation is submitted. Use a JSON string to pass a literal `image:` prefix. + Evaluation prints the value, available probability or probabilities, confidence, provider metadata, elapsed milliseconds and any error. It invokes the configured provider and may incur charges. `--evaluation-timeout` bounds runtime evaluation diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc index 9e5ab04496cf..2907c2e4b798 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-tui-observe.adoc @@ -552,6 +552,13 @@ envelope; the file is read only when Run is requested. Unchanged text loaded fro literal text, including an `image:` prefix. Definition samples also accept `"body":"image:/path/picture.png"`. +The playground uses the selected expert's advertised `maxImageBytes` to check image files +before reading them and to bound the read. Inline image payloads are also checked before +submission. Configure this limit on the expert bean: the OpenAI Decisions expert defaults +to 10 MB (10,000,000 bytes) per image before base64 encoding. The CLI and route evaluations +use the same expert configuration. Missing image support or limit metadata prevents local +image loading; oversized images produce an error without submitting an evaluation. + The latest page refreshes every second; `Space` pauses or resumes polling. Older pages stay fixed until explicitly refreshed. Filters use exact values; `definition` and `operation` apply to all retained history. `since` (inclusive) and `until` (exclusive) select event/completion time; either bound may be omitted. diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelSemantic.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelSemantic.java index 5956701f1341..e0f76a0293fa 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelSemantic.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/CamelSemantic.java @@ -111,6 +111,7 @@ public class CamelSemantic extends SemanticActionCommand { continue; } renderField("Input", contract.get("inputTypes"), " "); + renderField("Maximum image bytes", contract.get("maxImageBytes"), " "); renderField("Requirements", contract.get("inputRequirements"), " "); renderField("Result", contract.get("resultMeaning"), " "); renderField("Labels", contract.get("labels"), " "); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticActionCommand.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticActionCommand.java index 5d447044bf10..75925e316cbd 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticActionCommand.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticActionCommand.java @@ -76,6 +76,11 @@ public abstract class SemanticActionCommand extends ActionBaseCommand implements + " runs Camel " + version, null); } + try { + prepareRequest(pid, request); + } catch (IllegalArgumentException e) { + return error(2, e.getMessage(), null); + } JsonObject response = RuntimeHelper.executeAction(pid, request, timeout); if (response == null) { return error(4, "No reply from integration within " + timeout + " milliseconds", null); @@ -178,6 +183,10 @@ public abstract class SemanticActionCommand extends ActionBaseCommand implements return Character.toUpperCase(label.charAt(0)) + label.substring(1).replace(" Id", " ID"); } + /** Prepare input requiring the selected runtime's metadata before submitting an action. */ + protected void prepareRequest(long pid, JsonObject request) throws Exception { + } + protected abstract JsonObject request(); protected abstract void render(JsonObject response); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java index 94ddf41c8e21..95bba0f098e9 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticEvaluate.java @@ -20,11 +20,13 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import org.apache.camel.dsl.jbang.core.commands.CamelJBangMain; +import org.apache.camel.dsl.jbang.core.common.RuntimeHelper; import org.apache.camel.dsl.jbang.core.common.SemanticMediaInput; import org.apache.camel.util.json.JsonObject; import picocli.CommandLine; @@ -124,6 +126,49 @@ public class SemanticEvaluate extends SemanticActionCommand { return request; } + @Override + protected void prepareRequest(long pid, JsonObject request) throws Exception { + String selected = evaluation == null ? input : body; + String inputKey = evaluation == null ? "input" : "body"; + Object value = request.get(inputKey); + Object messageHeaders = evaluation == null ? null : request.get("headers"); + boolean localImage = selected != null && selected.startsWith("image:"); + if (!localImage && !SemanticMediaInput.hasImages(value, messageHeaders)) { + return; + } + JsonObject metadata = RuntimeHelper.executeAction(pid, + new JsonObject(Map.of("action", "semantic-metadata", "overview", true)), timeout); + if (metadata == null || metadata.isEmpty()) { + throw new IllegalArgumentException("Cannot read the selected expert's image limit from runtime metadata"); + } + String reference = expert; + String operationName = operation; + if (evaluation != null) { + JsonObject definition = named(metadata, "evaluations", "name", evaluation); + reference = definition.getString("expert"); + operationName = definition.getString("operation"); + } + JsonObject selectedExpert = named(metadata, "experts", "reference", reference); + JsonObject selectedOperation = named(selectedExpert, "operations", "name", operationName); + int maxBytes = SemanticMediaInput.imageMaxBytes(selectedOperation); + if (localImage) { + value = SemanticMediaInput.readImage(selected, maxBytes); + request.put(inputKey, value); + } + SemanticMediaInput.validateImageSize(value, messageHeaders, maxBytes); + } + + private static JsonObject named(JsonObject metadata, String collection, String key, String name) { + if (name != null && metadata.getCollection(collection) != null) { + for (Object item : metadata.getCollection(collection)) { + if (item instanceof JsonObject value && name.equals(value.getString(key))) { + return value; + } + } + } + throw new IllegalArgumentException("Selected semantic " + collection + " unavailable in runtime metadata"); + } + private static void requireText(String value, String option) { if (value == null || value.isBlank()) { throw new IllegalArgumentException(option + " is required and must not be blank"); @@ -147,9 +192,6 @@ public class SemanticEvaluate extends SemanticActionCommand { } private static Object value(String value, boolean allowFile) { - if (allowFile && value.startsWith("image:")) { - return SemanticMediaInput.readImage(value); - } if (value.startsWith("json:")) { try { return JSON_VALUE.readValue(value.substring(5)); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java index 8d332eb02188..d66cc5420cfb 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInput.java @@ -18,6 +18,7 @@ package org.apache.camel.dsl.jbang.core.common; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Base64; @@ -43,8 +44,55 @@ public final class SemanticMediaInput { return isMedia(input) ? (List<?>) ((Map<?, ?>) input).get(PARTS) : List.of(); } + public static boolean hasImages(Object input) { + return input instanceof String text && text.startsWith("data:image/") + || parts(input).stream().anyMatch(item -> item instanceof Map<?, ?> part && "image".equals(part.get("type"))); + } + + public static boolean hasImages(Object body, Object headers) { + return hasImages(body) || body instanceof String && hasImageContentType(headers); + } + + public static void validateImageSize(Object body, Object headers, int maxBytes) { + if (body instanceof String text && !text.startsWith("data:image/") && hasImageContentType(headers)) { + validateBase64Size(text, 0, maxBytes); + } else { + validateImageSize(body, maxBytes); + } + } + + private static boolean hasImageContentType(Object headers) { + return headers instanceof Map<?, ?> values && values.entrySet().stream() + .anyMatch(entry -> "Content-Type".equalsIgnoreCase(entry.getKey().toString()) + && entry.getValue() instanceof String type + && type.trim().toLowerCase(Locale.ROOT).startsWith("image/")); + } + + /** Check encoded images before sending a tooling request, without decoding their payloads. */ + public static void validateImageSize(Object input, int maxBytes) { + if (input instanceof String data && data.startsWith("data:image/")) { + int offset = imageDataOffset(data); + if (offset >= 0) { + validateBase64Size(data, offset, maxBytes); + } + } else { + for (Object item : parts(input)) { + if (item instanceof Map<?, ?> part && "image".equals(part.get("type"))) { + validateImageSize(part.get("data"), maxBytes); + } + } + } + } + + private static void validateBase64Size(String data, int offset, int maxBytes) { + int padding = data.endsWith("==") ? 2 : data.endsWith("=") ? 1 : 0; + if ((data.length() - offset) * 3L / 4 - padding > maxBytes) { + throw new IllegalArgumentException("Image exceeds the expert maxImageBytes limit of " + maxBytes + " bytes"); + } + } + /** Read only an explicitly requested local image. This must run off the TUI render thread. */ - public static Object readImage(String value) { + public static Object readImage(String value, int maxBytes) { if (!value.startsWith("image:")) { return value; } @@ -56,14 +104,42 @@ public final class SemanticMediaInput { if (type == null) { throw new IllegalArgumentException("image: requires a local PNG, JPEG, GIF or WebP file"); } + if (maxBytes <= 0) { + throw new IllegalArgumentException("The selected expert must declare a positive maxImageBytes limit"); + } + String tooLarge = "Image file exceeds the expert maxImageBytes limit of " + maxBytes + " bytes"; try { - String data = "data:" + type + ";base64," + Base64.getEncoder().encodeToString(Files.readAllBytes(path)); + if (Files.size(path) > maxBytes) { + throw new IllegalArgumentException(tooLarge); + } + byte[] bytes; + try (InputStream input = Files.newInputStream(path)) { + // Bound the read too: a file can grow after the size check. + bytes = input.readNBytes(maxBytes); + if (input.read() != -1) { + throw new IllegalArgumentException(tooLarge); + } + } + String data = "data:" + type + ";base64," + Base64.getEncoder().encodeToString(bytes); return Map.of(PARTS, List.of(Map.of("type", "image", "data", data))); } catch (IOException e) { throw new IllegalArgumentException("Cannot read the selected image file", e); } } + /** Read the configured limit advertised by an image-capable expert operation. */ + public static int imageMaxBytes(Map<?, ?> operation) { + if (operation != null && operation.get("contract") instanceof Map<?, ?> contract + && contract.get("inputTypes") instanceof List<?> types && types.contains("image") + && contract.get("maxImageBytes") instanceof Number limit + && limit.longValue() > 0 && limit.longValue() <= Integer.MAX_VALUE + && limit.doubleValue() == limit.longValue()) { + return limit.intValue(); + } + throw new IllegalArgumentException( + "The selected expert must advertise image support and a positive maxImageBytes limit"); + } + /** Preserve metadata in terminal/JSON displays without printing image payloads. */ public static Object withoutImageData(Object value) { if (value instanceof Map<?, ?> map) { diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java index 5d01d33b5d1f..18b0a44951a4 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/SemanticCommandTest.java @@ -29,6 +29,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -62,11 +63,61 @@ class SemanticCommandTest { void imageFileBodyIsSentAsExplicitMedia() throws Exception { Path image = home.resolve("input.png"); Files.write(image, new byte[] { 1, 2, 3 }); - assertEquals(0, respond(new SemanticEvaluate(main()), request -> { + assertEquals(0, respondImage(new SemanticEvaluate(main()), request -> { assertEquals(Map.of("$semanticInput", List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))), request.get("body")); return jsonUnchecked("{\"status\":\"success\",\"value\":true}"); - }, "--evaluation=inspect", "--body=image:" + image)); + }, 10_000_000, "--evaluation=inspect", "--body=image:" + image)); + } + + @ParameterizedTest + @ValueSource(booleans = { true, false }) + void expertImageLimitRejectsBeforeSendingAnEvaluation(boolean named) throws Exception { + Path image = home.resolve("large.png"); + Files.write(image, new byte[] { 1, 2, 3 }); + String[] args = named + ? new String[] { "--evaluation=inspect", "--body=image:" + image, "--json" } + : new String[] { "--expert=inspector", "--operation=boolean", "--input=image:" + image, "--json" }; + assertEquals(2, respondImage(new SemanticEvaluate(main()), request -> { + throw new AssertionError("Oversized image must not reach evaluation"); + }, 2, args)); + assertError(2, "expert maxImageBytes limit of 2 bytes"); + } + + @ParameterizedTest + @ValueSource(strings = { + "data:image/png;base64,AQID", "AQID", + "json:{\"$semanticInput\":[{\"type\":\"image\",\"data\":\"data:image/png;base64,AQID\"}]}" }) + void expertLimitAlsoRejectsInlineImageBodies(String body) throws Exception { + assertEquals(2, respondImage(new SemanticEvaluate(main()), request -> { + throw new AssertionError("Oversized image must not reach evaluation"); + }, 2, "--evaluation=inspect", "--body=" + body, "--header=content-type=image/png", "--json")); + assertError(2, "expert maxImageBytes limit of 2 bytes"); + } + + @Test + void missingExpertImageLimitRejectsWithoutOpeningTheFile() throws Exception { + assertEquals(2, respondImage(new SemanticEvaluate(main()), request -> { + throw new AssertionError("Unbounded image must not reach evaluation"); + }, 0, "--expert=inspector", "--operation=boolean", "--input=image:" + home.resolve("missing.png"), "--json")); + assertError(2, "positive maxImageBytes"); + } + + private int respondImage( + SemanticActionCommand command, Function<JsonObject, JsonObject> response, int maxBytes, + String... options) + throws Exception { + return respond(command, request -> { + if ("semantic-metadata".equals(request.getString("action"))) { + return new JsonObject( + Map.of("experts", List.of(Map.of("reference", "inspector", "operations", List.of( + Map.of("name", "boolean", "contract", + Map.of("inputTypes", List.of("image"), "maxImageBytes", maxBytes))))), + "evaluations", + List.of(Map.of("name", "inspect", "expert", "inspector", "operation", "boolean")))); + } + return response.apply(request); + }, options); } @ParameterizedTest @@ -834,17 +885,29 @@ class SemanticCommandTest { } private int respond(IntSupplier call, Function<JsonObject, JsonObject> response) throws Exception { + var completed = new AtomicBoolean(); var responder = CompletableFuture.runAsync(() -> { try { - await().atMost(5, TimeUnit.SECONDS).until(() -> !actionFiles().isEmpty()); - Path request = actionFiles().get(0); - await().atMost(5, TimeUnit.SECONDS).until(() -> Files.size(request) > 0); - reply(request, response.apply(json(Files.readString(request)))); + while (!completed.get()) { + await().atMost(5, TimeUnit.SECONDS).until(() -> completed.get() || !actionFiles().isEmpty()); + if (completed.get()) { + break; + } + Path request = actionFiles().get(0); + await().atMost(5, TimeUnit.SECONDS).until(() -> Files.size(request) > 0); + reply(request, response.apply(json(Files.readString(request)))); + await().atMost(5, TimeUnit.SECONDS).until(() -> !Files.exists(request)); + } } catch (Exception e) { throw new RuntimeException(e); } }, executor); - int result = call.getAsInt(); + int result; + try { + result = call.getAsInt(); + } finally { + completed.set(true); + } responder.get(5, TimeUnit.SECONDS); assertTrue(actionFiles().isEmpty()); return result; diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java index 68cbd070f22d..6217cb3a13cf 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/common/SemanticMediaInputTest.java @@ -19,6 +19,7 @@ package org.apache.camel.dsl.jbang.core.common; import java.awt.image.BufferedImage; import java.io.ByteArrayOutputStream; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Base64; import java.util.List; @@ -31,7 +32,10 @@ import org.junit.jupiter.api.io.TempDir; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mockStatic; class SemanticMediaInputTest { @TempDir @@ -43,7 +47,7 @@ class SemanticMediaInputTest { image.setRGB(0, 0, 0xff0000); Path path = directory.resolve("sample.png"); ImageIO.write(image, "png", path.toFile()); - Object input = SemanticMediaInput.readImage("image:" + path); + Object input = SemanticMediaInput.readImage("image:" + path, 10_000_000); assertTrue(SemanticMediaInput.isMedia(input)); var description = SemanticMediaInput.withoutImageData(input).toString(); assertTrue(description.contains("image/png")); @@ -87,10 +91,46 @@ class SemanticMediaInputTest { assertEquals(List.of(0xffffff, 0x7f7f7f), preview.pixels().get(0)); } + @Test + void expertLimitIncludesTheBoundaryAndRejectsLargerFiles() throws Exception { + Path path = directory.resolve("sample.png"); + Files.write(path, new byte[] { 1, 2, 3 }); + assertTrue(SemanticMediaInput.isMedia(SemanticMediaInput.readImage("image:" + path, 3))); + var failure = assertThrows(IllegalArgumentException.class, () -> SemanticMediaInput.readImage("image:" + path, 2)); + assertTrue(failure.getMessage().contains("expert maxImageBytes limit of 2 bytes")); + assertThrows(IllegalArgumentException.class, () -> SemanticMediaInput.readImage("image:" + path, 0)); + } + + @Test + void readRemainsBoundedWhenTheFileGrowsAfterTheSizeCheck() throws Exception { + Path path = directory.resolve("growing.png"); + Files.write(path, new byte[] { 1, 2, 3, 4 }); + try (var files = mockStatic(Files.class, CALLS_REAL_METHODS)) { + files.when(() -> Files.size(path)).thenReturn(3L); + assertThrows(IllegalArgumentException.class, () -> SemanticMediaInput.readImage("image:" + path, 3)); + } + } + + @Test + void usesOnlyAdvertisedImageLimitsAndChecksEncodedImages() { + assertEquals(3, SemanticMediaInput + .imageMaxBytes(Map.of("contract", Map.of("inputTypes", List.of("image"), "maxImageBytes", 3)))); + for (Object limit : List.of(0, -1, 1.5, Long.MAX_VALUE, "3")) { + assertThrows(IllegalArgumentException.class, () -> SemanticMediaInput.imageMaxBytes( + Map.of("contract", Map.of("inputTypes", List.of("image"), "maxImageBytes", limit)))); + } + assertThrows(IllegalArgumentException.class, () -> SemanticMediaInput.imageMaxBytes(Map.of())); + Object input = Map.of("$semanticInput", List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))); + assertTrue(SemanticMediaInput.hasImages(input)); + SemanticMediaInput.validateImageSize(input, 3); + assertThrows(IllegalArgumentException.class, () -> SemanticMediaInput.validateImageSize(input, 2)); + } + @Test void ordinaryStateAndTextKeepTheirMeaning() { Object input = Map.of("type", "image", "data", "application state"); assertEquals(input, SemanticMediaInput.withoutImageData(input)); - assertEquals("https://example.com/picture.png";, SemanticMediaInput.readImage("https://example.com/picture.png";)); + assertEquals("https://example.com/picture.png";, + SemanticMediaInput.readImage("https://example.com/picture.png";, 10_000_000)); } } diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticDetails.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticDetails.java index 49ab089ffc55..06a01d0f5581 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticDetails.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticDetails.java @@ -183,6 +183,9 @@ final class SemanticDetails { result.add(Line.from(Span.styled("Operation contract", Theme.label()))); result.add(Line.empty()); add(result, "Input", inputTypes(op)); + if (contract(op).containsKey("maxImageBytes")) { + add(result, "Maximum image bytes", text(contract(op), "maxImageBytes")); + } add(result, "Accepts", text(contract(op), "inputRequirements")); result.add(Line.empty()); result.addAll(result(op, definition.get("parameters"))); diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java index 87314cd114e3..89f98f33a9a4 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticEvaluationPopup.java @@ -204,8 +204,13 @@ final class SemanticEvaluationPopup { pending = CompletableFuture .supplyAsync(() -> { JsonObject prepared = new JsonObject(request); - if (prepared.get("body") instanceof String text) { - prepared.put("body", SemanticMediaInput.readImage(text)); + if (prepared.get("body") instanceof String text && text.startsWith("image:")) { + prepared.put("body", + SemanticMediaInput.readImage(text, SemanticMediaInput.imageMaxBytes(submittedOperation))); + } + if (SemanticMediaInput.hasImages(prepared.get("body"), prepared.get("headers"))) { + SemanticMediaInput.validateImageSize(prepared.get("body"), prepared.get("headers"), + SemanticMediaInput.imageMaxBytes(submittedOperation)); } return ctx.executeIndependentAction(pid, prepared, 60000); }, ctx.backgroundExecutor) diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java index 54c760bd48dd..45cb462cc41b 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/main/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticPlayground.java @@ -377,8 +377,13 @@ final class SemanticPlayground { pending = CompletableFuture .supplyAsync(() -> { JsonObject prepared = new JsonObject(request); - if (readLocalImage && prepared.get("input") instanceof String text) { - prepared.put("input", SemanticMediaInput.readImage(text)); + if (readLocalImage && prepared.get("input") instanceof String text && text.startsWith("image:")) { + prepared.put("input", + SemanticMediaInput.readImage(text, SemanticMediaInput.imageMaxBytes(operation))); + } + if (SemanticMediaInput.hasImages(prepared.get("input"))) { + SemanticMediaInput.validateImageSize(prepared.get("input"), + SemanticMediaInput.imageMaxBytes(operation)); } return ctx.executeIndependentAction(pid, prepared, 60000); }, ctx.backgroundExecutor) diff --git a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java index a293f8df8b44..e8a019beec6b 100644 --- a/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java +++ b/dsl/camel-jbang/camel-jbang-plugin-tui/src/test/java/org/apache/camel/dsl/jbang/core/commands/tui/SemanticImageViewTest.java @@ -81,7 +81,8 @@ class SemanticImageViewTest { runtime.selectedPid = "image"; var operation = new JsonObject( Map.of("name", "boolean", "contract", new JsonObject( - Map.of("inputTypes", List.of("text", "image"), "parameters", List.of())))); + Map.of("inputTypes", List.of("text", "image"), "maxImageBytes", 10_000_000, "parameters", + List.of())))); var playground = new SemanticPlayground(runtime, "image", "expert", operation); var input = Map.of("$semanticInput", List.of(Map.of("type", "image", "data", "data:image/png;base64,AQID"))); playground.loadAuditInput(new JsonObject(Map.of("eventId", "record", "input", input))); @@ -116,7 +117,8 @@ class SemanticImageViewTest { runtime.selectedPid = "image"; var operation = new JsonObject( Map.of("name", "boolean", "contract", new JsonObject( - Map.of("inputTypes", List.of("text", "image"), "parameters", List.of())))); + Map.of("inputTypes", List.of("text", "image"), "maxImageBytes", 10_000_000, "parameters", + List.of())))); var playground = new SemanticPlayground(runtime, "image", "expert", operation); String input = "image:" + file; playground.loadAuditInput(new JsonObject(Map.of("eventId", "record", "input", input))); @@ -134,4 +136,47 @@ class SemanticImageViewTest { runtime.backgroundExecutor.shutdownNow(); } } + + @Test + void expertLimitRejectsLocalAndInlineImagesBeforeSubmitting(@TempDir Path directory) throws Exception { + Path file = directory.resolve("large.png"); + Files.write(file, new byte[] { 1, 2, 3 }); + var runtime = new MonitorContext(new AtomicReference<>(List.of()), new AtomicReference<>(List.of())) { + final List<JsonObject> calls = new CopyOnWriteArrayList<>(); + + @Override + JsonObject executeIndependentAction(String pid, JsonObject request, long timeout) { + calls.add(request); + return new JsonObject(Map.of("status", "success")); + } + }; + try { + var integration = new IntegrationInfo(); + integration.pid = "image"; + runtime.data.set(List.of(integration)); + runtime.selectedPid = "image"; + var operation = new JsonObject( + Map.of("name", "boolean", "contract", new JsonObject( + Map.of("inputTypes", List.of("text", "image"), "parameters", List.of(), "maxImageBytes", 2)))); + for (String input : List.of("image:" + file, "data:image/png;base64,AQID")) { + var playground = new SemanticPlayground(runtime, "image", "expert", operation); + assertThat(playground.setInputValue("input", input)).isTrue(); + playground.handleKeyEvent(KeyEvent.ofChar('r', KeyModifiers.CTRL)); + await().atMost(5, TimeUnit.SECONDS).untilAsserted( + () -> assertThat(playground.snapshot().toJson()).contains("expert maxImageBytes limit of 2 bytes")); + assertThat(runtime.calls).isEmpty(); + } + for (String body : List.of("image:" + file, "data:image/png;base64,AQID", "AQID")) { + var popup = new SemanticEvaluationPopup(runtime, "image", new JsonObject(Map.of("name", "inspect")), operation); + assertThat(popup.setInputValue("sample", new JsonObject( + Map.of("body", body, "headers", Map.of("Content-Type", "image/png"))).toJson())).isTrue(); + popup.handleKeyEvent(KeyEvent.ofChar('r', KeyModifiers.CTRL)); + await().atMost(5, TimeUnit.SECONDS).untilAsserted( + () -> assertThat(popup.snapshot().toJson()).contains("expert maxImageBytes limit of 2 bytes")); + assertThat(runtime.calls).isEmpty(); + } + } finally { + runtime.backgroundExecutor.shutdownNow(); + } + } }
![https://example.com/picture.png"](https://example.com/picture.png"){kind=link}