This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch feature/CAMEL-25750-validator-observed in repository https://gitbox.apache.org/repos/asf/camel.git
commit fcbaf9e1a1608148c3a6783919cb2bd79f9117c7 Author: Claus Ibsen <[email protected]> AuthorDate: Tue Oct 20 16:01:18 2026 +0200 CAMEL-25750: camel-jbang - the validator fills the gaps of its body walk with the body types a running app saw The YAML validator walks the body through the steps of a route and says nothing where the file does not tell what the body is: after a consumer other than the file consumer, or a to: it cannot follow. When an integration runs routes of the file, camel-jbang asks it for its route structure and passes what each consumer gave and each producer returned to the validator (ProjectFacts.observed). Where all those bodies were of one kind, the walk knows the body there: text for a payload, parsed data for data or a collection, and the existing checks apply with the origin in the message ("in the running app, from: platform-http:/orders gave a ByteArrayInputStreamCache"). Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../apache/camel/catalog/docs/message-history.adoc | 4 +- .../docs/modules/eips/pages/message-history.adoc | 4 +- .../modules/ROOT/pages/camel-jbang-validation.adoc | 22 +++ .../dsl/jbang/core/commands/ai/ObservedBodies.java | 210 +++++++++++++++++++++ .../jbang/core/commands/ai/SourceValidator.java | 5 +- .../dsl/jbang/core/commands/ai/ValidatorFacts.java | 13 +- .../action/ValidateWithObservedBodiesTest.java | 115 +++++++++++ .../jbang/core/commands/ai/ObservedBodiesTest.java | 127 +++++++++++++ .../camel/dsl/yaml/validator/ProjectFacts.java | 37 +++- .../camel/dsl/yaml/validator/TextBodyFlow.java | 109 ++++++++--- .../camel/dsl/yaml/validator/TextBodyFlowTest.java | 96 ++++++++++ 11 files changed, 711 insertions(+), 31 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/message-history.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/message-history.adoc index ab4767f818e1..fd86e95c95dc 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/message-history.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/message-history.adoc @@ -476,7 +476,9 @@ List<String> types = registry.getBodyTypes("orders", "to1"); List<BodyTypeRegistry.Kind> kinds = registry.getBodyKinds("orders", "to1"); ---- -The Camel CLI shows them as well, see xref:manual::camel-jbang-diagnostics.adoc[Camel CLI diagnostics]. +The Camel CLI shows them as well, see xref:manual::camel-jbang-diagnostics.adoc[Camel CLI diagnostics]. And its YAML +validator uses them where it cannot tell the body from the file itself, such as after an HTTP consumer, see +xref:manual::camel-jbang-validation.adoc[Camel CLI validation]. == Configuring MessageHistory format diff --git a/core/camel-core-engine/src/main/docs/modules/eips/pages/message-history.adoc b/core/camel-core-engine/src/main/docs/modules/eips/pages/message-history.adoc index ab4767f818e1..fd86e95c95dc 100644 --- a/core/camel-core-engine/src/main/docs/modules/eips/pages/message-history.adoc +++ b/core/camel-core-engine/src/main/docs/modules/eips/pages/message-history.adoc @@ -476,7 +476,9 @@ List<String> types = registry.getBodyTypes("orders", "to1"); List<BodyTypeRegistry.Kind> kinds = registry.getBodyKinds("orders", "to1"); ---- -The Camel CLI shows them as well, see xref:manual::camel-jbang-diagnostics.adoc[Camel CLI diagnostics]. +The Camel CLI shows them as well, see xref:manual::camel-jbang-diagnostics.adoc[Camel CLI diagnostics]. And its YAML +validator uses them where it cannot tell the body from the file itself, such as after an HTTP consumer, see +xref:manual::camel-jbang-validation.adoc[Camel CLI validation]. == Configuring MessageHistory format diff --git a/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc b/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc index 92459f1110d7..5553b58ff340 100644 --- a/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc +++ b/docs/user-manual/modules/ROOT/pages/camel-jbang-validation.adoc @@ -71,6 +71,28 @@ and checked as YAML routes are: These are what the Java compiler cannot see inside the strings. A part of a Java route the parser cannot work out (a value computed at runtime) is left unchecked. +=== What a running integration saw + +The YAML validator follows the message body through the steps of each route, and reports a step that cannot work on +it: a field read such as `+${body.qty}+` on text that was never unmarshalled, or a Map that a `to: file` cannot write. +Where the file does not tell what the body is, such as after a `platform-http` or `kafka` consumer, or after a call +with `to: http`, it says nothing. + +When an integration runs the routes of the file with the message history on, such as with `camel run --dev`, the +validator fills these gaps with what the integration saw at the endpoints of its routes: the body each consumer gave, +and each producer returned, when all those bodies were of one kind (see +xref:components:eips:message-history.adoc[Message History]). For example: + +[source,text] +---- +route orders: simple reads a field of the body (${body.qty}), but the body here is still text, not parsed data - in +the running app, from: platform-http:/orders gave a ByteArrayInputStreamCache: unmarshal it first with the data format +of the payload (json, jacksonXml, csv, ...) to read its fields +---- + +So the next version of a file is checked against what the integration does with the current one, before it is +written or reloaded. The tools that write files for AI agents validate the same way. + === Endpoints no route consumes As the routes of an application are spread over the files of its directory, `camel validate source` also reports diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ObservedBodies.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ObservedBodies.java new file mode 100644 index 000000000000..50bb337aff95 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ObservedBodies.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.ai; + +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.camel.dsl.jbang.core.common.ActionClient; +import org.apache.camel.dsl.jbang.core.common.CommandLineHelper; +import org.apache.camel.dsl.jbang.core.common.RuntimeHelper; +import org.apache.camel.dsl.yaml.validator.ProjectFacts; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; + +/** + * What a running integration of a route file saw at the endpoints of its routes (CAMEL-25750): the body each consumer + * gave, and the body each producer returned, as the step after it was reached with while the message history was on + * (the dev profile turns it on). The validator fills the gaps of its walk with them: what the walk alone does not know, + * such as the stream cache of an HTTP request, or the List a sql call returns. + * <p/> + * The facts are per endpoint, not per line: the validator checks the next version of the file while the integration + * runs the one before, and what an endpoint gives does not move with the lines. + */ +final class ObservedBodies { + + /** How long to wait for the integration to answer: validation does not wait for long. */ + private static final long TIMEOUT = 3000; + + /** How long the facts of a file are kept: a write validates the file before and after the change. */ + private static final long KEEP = 2000; + + private static final Map<Path, Facts> CACHE = new ConcurrentHashMap<>(); + + private record Facts(long time, Map<String, ProjectFacts.Observed> observed) { + } + + private ObservedBodies() { + } + + /** + * What a running integration of the file saw; nothing when no integration runs routes of the file, or it does not + * answer in time. + */ + static Map<String, ProjectFacts.Observed> read(String fileName, Path directory) { + if (fileName == null || fileName.isBlank()) { + return Map.of(); + } + Path file = (directory != null ? directory.resolve(fileName) : Path.of(fileName)).toAbsolutePath().normalize(); + Facts cached = CACHE.get(file); + if (cached != null && System.currentTimeMillis() - cached.time() < KEEP) { + return cached.observed(); + } + Map<String, ProjectFacts.Observed> answer = Map.of(); + // the processes are only looked for when an integration runs at all: most validations have none + String[] statusFiles = CommandLineHelper.getCamelDir().toFile().list((d, n) -> n.endsWith("-status.json")); + if (statusFiles == null || statusFiles.length == 0) { + return answer; + } + for (RuntimeHelper.ProcessInfo process : RuntimeHelper.discoverProcesses()) { + if (runsRoutesOf(RuntimeHelper.readStatus(process.pid()), file)) { + answer = ask(process.pid()); + break; + } + } + CACHE.put(file, new Facts(System.currentTimeMillis(), answer)); + return answer; + } + + private static Map<String, ProjectFacts.Observed> ask(long pid) { + try { + JsonObject request = new JsonObject(); + request.put("action", "route-structure"); + request.put("filter", "*"); + String answer = ActionClient.executeText(pid, request, TIMEOUT); + JsonObject structure = answer != null ? Jsoner.deserialize(answer, (JsonObject) null) : null; + return structure != null ? fromRouteStructure(structure) : Map.of(); + } catch (Exception e) { + // the validator then says only what it knows from the file + return Map.of(); + } + } + + /** Whether the status of an integration has a route loaded from the file. */ + static boolean runsRoutesOf(JsonObject status, Path file) { + JsonArray routes = status != null ? status.getCollection("routes") : null; + if (routes == null) { + return false; + } + for (Object o : routes) { + if (o instanceof JsonObject route && sameFile(route.getString("source"), file)) { + return true; + } + } + return false; + } + + /** + * Whether a route source (file:/work/orders.camel.yaml:12, classpath:orders.camel.yaml:3) is the file: by its path + * when it has one, by its name otherwise. + */ + static boolean sameFile(String source, Path file) { + if (source == null || file.getFileName() == null) { + return false; + } + String s = source.replaceFirst(":\\d+$", ""); + int colon = s.indexOf(':'); + if (colon > 1) { + // the scheme, not a drive letter of Windows + s = s.substring(colon + 1); + } + if (s.startsWith("/") || s.contains(":\\")) { + return Path.of(s).toAbsolutePath().normalize().equals(file); + } + String name = s.substring(Math.max(s.lastIndexOf('/'), s.lastIndexOf('\\')) + 1); + return name.equals(file.getFileName().toString()); + } + + /** + * The bodies of the endpoints, from the route-structure console: what the step after a from was reached with is + * what the consumer gave, and what the step after a to (the next step of its block) was reached with is what the + * producer returned. A to that ends its block has no step that only it reaches, and a toD no endpoint of its own. + */ + static Map<String, ProjectFacts.Observed> fromRouteStructure(JsonObject structure) { + Map<String, Set<String>> types = new LinkedHashMap<>(); + Map<String, Set<String>> kinds = new LinkedHashMap<>(); + JsonArray routes = structure.getCollection("routes"); + if (routes == null) { + return Map.of(); + } + for (Object r : routes) { + List<JsonObject> code = r instanceof JsonObject route ? route.getCollection("code") : null; + if (code == null) { + continue; + } + for (int i = 0; i < code.size(); i++) { + JsonObject line = code.get(i); + String type = line.getString("type"); + String uri = line.getString("uri"); + if (uri == null) { + continue; + } + uri = Jsoner.unescape(uri); + JsonObject after = null; + String key = null; + if ("from".equals(type)) { + after = firstStep(code, i); + key = ProjectFacts.consumerKey(uri); + } else if ("to".equals(type)) { + after = nextStep(code, i); + key = ProjectFacts.producerKey(uri); + } + List<String> seen = after != null ? after.getCollection("bodyTypes") : null; + List<String> seenKinds = after != null ? after.getCollection("bodyKinds") : null; + if (key == null || seen == null || seen.isEmpty() || seenKinds == null || seenKinds.isEmpty()) { + continue; + } + types.computeIfAbsent(key, k -> new LinkedHashSet<>()).addAll(seen); + kinds.computeIfAbsent(key, k -> new LinkedHashSet<>()).addAll(seenKinds); + } + } + Map<String, ProjectFacts.Observed> answer = new LinkedHashMap<>(); + types.forEach((key, t) -> answer.put(key, new ProjectFacts.Observed(List.copyOf(t), List.copyOf(kinds.get(key))))); + return answer; + } + + /** The first step of a route: the line after its from, one level deeper. */ + private static JsonObject firstStep(List<JsonObject> code, int from) { + int level = level(code.get(from)); + return from + 1 < code.size() && level(code.get(from + 1)) == level + 1 ? code.get(from + 1) : null; + } + + /** The step after a to in its block: the next line on its level, before the block ends. */ + private static JsonObject nextStep(List<JsonObject> code, int to) { + int level = level(code.get(to)); + for (int j = to + 1; j < code.size(); j++) { + int l = level(code.get(j)); + if (l == level) { + return code.get(j); + } else if (l < level) { + return null; + } + } + return null; + } + + private static int level(JsonObject line) { + Integer level = line.getInteger("level"); + return level != null ? level : 0; + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/SourceValidator.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/SourceValidator.java index 55f38efe889d..a7364ea2f183 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/SourceValidator.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/SourceValidator.java @@ -120,9 +120,10 @@ public final class SourceValidator { if (name.endsWith(".yaml") || name.endsWith(".yml")) { // a rest binding to an OpenAPI specification hides the verb, and the specification is a file beside the // route: read it, so that a GET operation is known to carry no body (CAMEL-24844), and with the REST binding - // mode of the properties, whether a Map sent or returned over REST is written for the route (CAMEL-25762) + // mode of the properties, whether a Map sent or returned over REST is written for the route (CAMEL-25762); + // and what a running integration of the file saw at the endpoints of its routes (CAMEL-25750) List<String> msgs = validateCamelYaml(content, catalog, schemaValidator, - ValidatorFacts.read(content, directory)); + ValidatorFacts.read(fileName, content, directory)); if (name.endsWith(".kamelet.yaml") || name.endsWith(".kamelet.yml")) { // the shape of a Kamelet file, which the schema of a route file does not know List<String> shape = KameletChecks.validateKameletFile(content); diff --git a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java index a3ff7ef6b033..735b9a15ffc9 100644 --- a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java +++ b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java @@ -30,7 +30,8 @@ import org.apache.camel.dsl.yaml.validator.ProjectFacts; /** * What the YAML validator cannot read from a route file, read from the files beside it (CAMEL-25762): the operations of - * the OpenAPI specifications the file uses, and the REST binding mode the project's properties set. + * the OpenAPI specifications the file uses, and the REST binding mode the project's properties set. And from a running + * integration of the file: the bodies the endpoints of its routes gave (CAMEL-25750). */ final class ValidatorFacts { @@ -47,14 +48,18 @@ final class ValidatorFacts { private ValidatorFacts() { } - /** The facts for a file in a directory, or nothing when there is no directory. */ - static ProjectFacts read(String content, Path directory) { + /** + * The facts for a file in a directory, or nothing when there is no directory: with what a running integration of + * the file saw at the endpoints of its routes (CAMEL-25750). + */ + static ProjectFacts read(String fileName, String content, Path directory) { if (content == null || directory == null) { return ProjectFacts.NONE; } return new ProjectFacts( OpenApiVerbs.bodylessEndpoints(content, directory), OpenApiVerbs.operations(content, directory), - restBindingMode(directory), OpenApiVerbs.requestFields(content, directory)); + restBindingMode(directory), OpenApiVerbs.requestFields(content, directory), + ObservedBodies.read(fileName, directory)); } /** diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/ValidateWithObservedBodiesTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/ValidateWithObservedBodiesTest.java new file mode 100644 index 000000000000..de6a3050c472 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/action/ValidateWithObservedBodiesTest.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.action; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; + +import org.apache.camel.catalog.DefaultCamelCatalog; +import org.apache.camel.dsl.jbang.core.commands.ai.SourceValidator; +import org.apache.camel.util.json.JsonArray; +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.junit.jupiter.MockitoExtension; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * The validator fills the gaps of its walk with what a running integration of the file saw at its endpoints + * (CAMEL-25750): here the stream cache that the HTTP consumer gives, which a field read cannot read. + */ +@ExtendWith(MockitoExtension.class) +class ValidateWithObservedBodiesTest extends ActionCommandTestSupport { + + private static final String YAML = """ + - route: + id: orders + from: + uri: platform-http:/orders + steps: + - setHeader: + name: qty + expression: + simple: ${body.qty} + """; + + @TempDir + Path dir; + + @Test + void theValidatorUsesWhatTheRunningAppSaw() throws Exception { + writeStatusFile(TEST_PID, status("file:" + dir.resolve("orders.camel.yaml") + ":1")); + + List<String> errors = new ArrayList<>(); + int exit = callWithResponse(() -> { + errors.addAll(SourceValidator.validate("orders.camel.yaml", YAML, new DefaultCamelCatalog(), null, dir)); + return 0; + }, routeStructure()); + + assertEquals(0, exit); + assertEquals("route-structure", readActionFile(TEST_PID).getString("action")); + assertTrue(errors.stream().anyMatch( + m -> m.contains("in the running app, from: platform-http:/orders gave a ByteArrayInputStreamCache")), + errors.toString()); + } + + @Test + void anIntegrationOfAnotherFileIsNotAsked() throws Exception { + writeStatusFile(TEST_PID, status("file:" + dir.resolve("invoices.camel.yaml") + ":1")); + + List<String> errors = new ArrayList<>(); + callWithResponse(() -> { + errors.addAll(SourceValidator.validate("orders.camel.yaml", YAML, new DefaultCamelCatalog(), null, dir)); + return 0; + }, routeStructure()); + + assertNull(readActionFile(TEST_PID)); + assertTrue(errors.isEmpty(), errors.toString()); + } + + private static JsonObject status(String source) { + JsonObject route = new JsonObject(); + route.put("routeId", "orders"); + route.put("source", source); + JsonArray routes = new JsonArray(); + routes.add(route); + JsonObject context = new JsonObject(); + context.put("name", "orders"); + JsonObject root = new JsonObject(); + root.put("context", context); + root.put("routes", routes); + return root; + } + + // what the route-structure console of the integration answers: the first step was reached with a stream cache + private static JsonObject routeStructure() { + return Jsoner.deserialize(""" + {"routes": [{"routeId": "orders", "code": [ + {"type": "route", "id": "orders", "level": 0}, + {"type": "from", "id": "orders", "level": 1, "uri": "platform-http:/orders"}, + {"type": "setHeader", "id": "setHeader1", "level": 2, + "bodyTypes": ["org.apache.camel.converter.stream.ByteArrayInputStreamCache"], "bodyKinds": ["payload"]} + ]}]} + """, (JsonObject) null); + } +} diff --git a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ObservedBodiesTest.java b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ObservedBodiesTest.java new file mode 100644 index 000000000000..18a465d84f13 --- /dev/null +++ b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ObservedBodiesTest.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.dsl.jbang.core.commands.ai; + +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import org.apache.camel.dsl.yaml.validator.ProjectFacts; +import org.apache.camel.util.json.JsonObject; +import org.apache.camel.util.json.Jsoner; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * The bodies the endpoints of a running integration gave, from its route-structure console (CAMEL-25750). + */ +class ObservedBodiesTest { + + @Test + void consumersAndProducers() { + // the shape of the route-structure console of Camel 4.24: a choice whose branch calls seda, then log and toD + JsonObject structure = Jsoner.deserialize( + """ + {"routes": [{"routeId": "orders", "code": [ + {"type": "route", "id": "orders", "level": 0}, + {"type": "from", "id": "orders", "level": 1, "uri": "direct:orders"}, + {"type": "choice", "id": "choice1", "level": 2, "bodyTypes": ["java.lang.String"], "bodyKinds": ["payload"]}, + {"type": "when", "id": "when1", "level": 3}, + {"type": "to", "id": "to1", "level": 4, "uri": "seda:stock?size=5"}, + {"type": "to", "id": "to2", "level": 2, "uri": "log:outbox", "bodyTypes": ["java.lang.String"], "bodyKinds": ["payload"]}, + {"type": "toD", "id": "toD1", "level": 2, "bodyTypes": ["java.lang.String"], "bodyKinds": ["payload"]} + ]}]} + """, + (JsonObject) null); + + Map<String, ProjectFacts.Observed> observed = ObservedBodies.fromRouteStructure(structure); + + // the first step was reached with what the consumer gave + assertThat(observed.get(ProjectFacts.consumerKey("direct:orders"))) + .isEqualTo(new ProjectFacts.Observed(List.of("java.lang.String"), List.of("payload"))); + // and the step after log:outbox with what it returned + assertThat(observed.get(ProjectFacts.producerKey("log:outbox"))) + .isEqualTo(new ProjectFacts.Observed(List.of("java.lang.String"), List.of("payload"))); + // the seda call ends its branch: the step after the choice is reached by both branches, not by it alone + assertThat(observed).doesNotContainKey(ProjectFacts.producerKey("seda:stock")); + assertThat(observed).hasSize(2); + } + + @Test + void theBodiesOfAnEndpointAcrossRoutes() { + JsonObject structure = Jsoner.deserialize( + """ + {"routes": [ + {"routeId": "report", "code": [ + {"type": "route", "id": "report", "level": 0}, + {"type": "from", "id": "report", "level": 1, "uri": "timer:report"}, + {"type": "to", "id": "to1", "level": 2, "uri": "sql:select * from orders", "bodyTypes": ["null"], "bodyKinds": ["null"]}, + {"type": "to", "id": "to2", "level": 2, "uri": "file:reports", "bodyTypes": ["java.util.ArrayList"], "bodyKinds": ["collection"]} + ]}, + {"routeId": "audit", "code": [ + {"type": "route", "id": "audit", "level": 0}, + {"type": "from", "id": "audit", "level": 1, "uri": "direct:audit"}, + {"type": "to", "id": "to3", "level": 2, "uri": "sql:select * from orders?outputType=SelectOne", "bodyTypes": ["java.lang.String"], "bodyKinds": ["payload"]}, + {"type": "log", "id": "log1", "level": 2, "bodyTypes": ["java.util.LinkedHashMap"], "bodyKinds": ["data"]} + ]}, + {"routeId": "idle", "code": [ + {"type": "route", "id": "idle", "level": 0}, + {"type": "from", "id": "idle", "level": 1, "uri": "direct:idle"}, + {"type": "to", "id": "to4", "level": 2, "uri": "seda:archive"}, + {"type": "log", "id": "log2", "level": 2} + ]} + ]} + """, + (JsonObject) null); + + Map<String, ProjectFacts.Observed> observed = ObservedBodies.fromRouteStructure(structure); + + // the two calls of one endpoint (without the query) gave a List and a Map + assertThat(observed.get(ProjectFacts.producerKey("sql:select * from orders"))) + .isEqualTo(new ProjectFacts.Observed( + List.of("java.util.ArrayList", "java.util.LinkedHashMap"), List.of("collection", "data"))); + assertThat(observed.get(ProjectFacts.consumerKey("timer:report"))) + .isEqualTo(new ProjectFacts.Observed(List.of("null"), List.of("null"))); + assertThat(observed.get(ProjectFacts.consumerKey("direct:audit"))) + .isEqualTo(new ProjectFacts.Observed(List.of("java.lang.String"), List.of("payload"))); + // a route no message has reached yet tells nothing + assertThat(observed).doesNotContainKeys( + ProjectFacts.consumerKey("direct:idle"), ProjectFacts.producerKey("seda:archive")); + } + + @Test + void theRoutesOfAFile() { + Path file = Path.of("/work/orders/orders.camel.yaml"); + + assertThat(ObservedBodies.sameFile("file:/work/orders/orders.camel.yaml:12", file)).isTrue(); + assertThat(ObservedBodies.sameFile("file:/other/orders.camel.yaml:12", file)) + .as("another project's file of the same name").isFalse(); + assertThat(ObservedBodies.sameFile("file:orders.camel.yaml:3", file)).isTrue(); + assertThat(ObservedBodies.sameFile("classpath:routes/orders.camel.yaml:3", file)).isTrue(); + assertThat(ObservedBodies.sameFile("file:invoices.camel.yaml:3", file)).isFalse(); + assertThat(ObservedBodies.sameFile(null, file)).isFalse(); + + JsonObject status = Jsoner.deserialize(""" + {"routes": [{"routeId": "invoices", "source": "file:invoices.camel.yaml:3"}, + {"routeId": "orders", "source": "file:orders.camel.yaml:5"}]} + """, (JsonObject) null); + assertThat(ObservedBodies.runsRoutesOf(status, file)).isTrue(); + assertThat(ObservedBodies.runsRoutesOf(status, Path.of("/work/orders/stock.camel.yaml"))).isFalse(); + assertThat(ObservedBodies.runsRoutesOf(null, file)).isFalse(); + } +} diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java index a45603d158bb..01b05c33b975 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java @@ -23,7 +23,8 @@ import java.util.Set; /** * What the validator cannot read from a route file, read by the caller from the files beside it (CAMEL-25762): the * OpenAPI operations the routes use, whose verb decides whether a call sends the body, and the REST binding mode of the - * project, which decides whether the binding writes a Map for the route. + * project, which decides whether the binding writes a Map for the route. And from a running integration of the routes: + * the bodies their endpoints gave (CAMEL-25750). * * @param bodylessEndpoints the {@code direct:} endpoints of the operations of a contract-first binding whose request * carries no body (CAMEL-24844) @@ -33,17 +34,39 @@ import java.util.Set; * null when not known: then nothing is said that a binding could change * @param requestFields per specification, keyed as {@link #operations()} is: the fields the JSON request of an * operationId needs, for the operations whose request schema requires some + * @param observed what a running integration of the routes saw at their endpoints (CAMEL-25750): the body a + * consumer gave, keyed as {@link #consumerKey(String)}, and the body a producer returned, + * keyed as {@link #producerKey(String)}: the types the step after it was reached with */ public record ProjectFacts(Set<String> bodylessEndpoints, Map<String, Map<String, String>> operations, - String restBindingMode, Map<String, Map<String, List<String>>> requestFields) { + String restBindingMode, Map<String, Map<String, List<String>>> requestFields, Map<String, Observed> observed) { /** Nothing known about the project. */ public static final ProjectFacts NONE = new ProjectFacts(Set.of(), Map.of(), null); + /** + * The bodies an endpoint gave in a running integration: their types, and their kinds (null, payload, data or + * collection, as the {@code BodyTypeRegistry} of Camel names them). + */ + public record Observed(List<String> types, List<String> kinds) { + + public Observed { + types = types != null ? List.copyOf(types) : List.of(); + kinds = kinds != null ? List.copyOf(kinds) : List.of(); + } + } + public ProjectFacts { bodylessEndpoints = bodylessEndpoints != null ? bodylessEndpoints : Set.of(); operations = operations != null ? operations : Map.of(); requestFields = requestFields != null ? requestFields : Map.of(); + observed = observed != null ? observed : Map.of(); + } + + /** Without what a running integration saw. */ + public ProjectFacts(Set<String> bodylessEndpoints, Map<String, Map<String, String>> operations, + String restBindingMode, Map<String, Map<String, List<String>>> requestFields) { + this(bodylessEndpoints, operations, restBindingMode, requestFields, Map.of()); } /** Without the fields of the requests. */ @@ -52,6 +75,16 @@ public record ProjectFacts(Set<String> bodylessEndpoints, Map<String, Map<String this(bodylessEndpoints, operations, restBindingMode, Map.of()); } + /** The key of the body a consumer gave: {@code from:} and the endpoint without its query. */ + public static String consumerKey(String uri) { + return "from:" + RouteGraph.normalize(uri); + } + + /** The key of the body a producer returned: {@code to:} and the endpoint without its query. */ + public static String producerKey(String uri) { + return "to:" + RouteGraph.normalize(uri); + } + /** * A specification as the routes name it, as {@link #operations()} keys it: the file without {@code file:} or * {@code classpath:}, and {@code openapi.json}, the default of rest-openapi, when none is named. diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java index 56a396ce1adf..7805e50f33ed 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; +import java.util.StringJoiner; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -50,6 +51,10 @@ import static org.apache.camel.dsl.yaml.validator.RouteGraph.scheme; * and a {@code to: file}, an http producer that sends it with POST, or the reply of a {@code platform-http} route fails * at runtime, because no type converter turns a Map into the stream they write: "No type converter available to convert * from type: java.util.LinkedHashMap to the required type: java.io.InputStream". + * <p/> + * Where the walk does not know the body - after a consumer other than the file consumer, or after a to: it cannot + * follow - what a running integration of the routes saw there fills the gap, when all the bodies it saw were of one + * kind (CAMEL-25750): the stream cache of an HTTP request, the List a sql call returns. */ final class TextBodyFlow { @@ -185,11 +190,12 @@ final class TextBodyFlow { * fields their JSON requests need. */ private record Walk(String routeId, Map<String, JsonNode> byFrom, List<Error> errors, Set<String> seen, boolean post, - Map<String, Map<String, String>> operations, Map<String, Map<String, List<String>>> requestFields) { + Map<String, Map<String, String>> operations, Map<String, Map<String, List<String>>> requestFields, + Map<String, ProjectFacts.Observed> observed) { /** The walk of a route a step calls: for its outcome alone. */ Walk called() { - return new Walk(null, byFrom, null, seen, false, operations, requestFields); + return new Walk(null, byFrom, null, seen, false, operations, requestFields, observed); } } @@ -202,6 +208,7 @@ final class TextBodyFlow { Map<String, Map<String, String>> operations = unbound ? facts.operations() : Map.of(); Map<String, Map<String, List<String>>> requestFields = unbound ? facts.requestFields() : Map.of(); Set<String> restReplies = unbound ? restReplies(target, facts) : Set.of(); + Map<String, ProjectFacts.Observed> observed = facts != null ? facts.observed() : Map.of(); Map<String, JsonNode> byFrom = new HashMap<>(); for (JsonNode entry : target) { JsonNode from = from(entry); @@ -233,9 +240,14 @@ final class TextBodyFlow { String uri = endpointOf(from); String consumer = scheme(uri); boolean post = NO_HTTP_HEADERS.contains(consumer) && !setsHttpHeaders(steps); - Body end = walk(steps, consumerText(from), at, new Walk( + Body start = consumerText(from); + if (start == null && uri != null) { + // what the consumer gave in a running integration of the route + start = observed(observed, ProjectFacts.consumerKey(uri), "from: " + normalize(uri) + " gave"); + } + Body end = walk(steps, start, at, new Walk( id, byFrom, errors, new HashSet<>(), post, operations, - requestFields)); + requestFields, observed)); // the body at the end of a platform-http route, or of the route of a REST call, is the reply boolean rest = uri != null && restReplies.contains(normalize(uri)); if (("platform-http".equals(consumer) || rest) && end instanceof Parsed parsed && steps != null @@ -396,23 +408,13 @@ final class TextBodyFlow { if (scheme != null && KEEPS_THE_BODY.contains(scheme)) { return body; } - JsonNode called = uri != null ? w.byFrom().get(normalize(uri)) : null; - if (called == null || !"to".equals(name) || !w.seen().add(normalize(uri))) { - return null; + Body returned = calledRoute(name, uri, body, w); + if (returned == null && uri != null) { + // what the producer returned in a running integration of the route + returned = observed(w.observed(), ProjectFacts.producerKey(uri), + name + ": " + normalize(uri) + " returned"); } - JsonNode from = from(called); - JsonNode steps = from.has("steps") ? from.get("steps") - : called.has("route") ? called.get("route").get("steps") : null; - Body end = walk(steps, body, new NodePath(PathType.JSON_POINTER), w.called()); - w.seen().remove(normalize(uri)); - if (end instanceof Text t) { - return new Text( - "to: " + normalize(uri) + " returns it as text (" + t.origin() + ")", t.format(), - t.file()); - } else if (end instanceof Parsed p) { - return new Parsed("to: " + normalize(uri) + " returns it as data (" + p.origin() + ")", p.format()); - } - return null; + return returned; } default -> { return KEEPING_STEPS.contains(name) ? body : null; @@ -420,6 +422,68 @@ final class TextBodyFlow { } } + /** + * What a direct: or seda: route of the file returns to the to: that calls it, as its walk ends; null when the walk + * does not know, or the endpoint is not a route of the file. + */ + private static Body calledRoute(String name, String uri, Body body, Walk w) { + JsonNode called = uri != null ? w.byFrom().get(normalize(uri)) : null; + if (called == null || !"to".equals(name) || !w.seen().add(normalize(uri))) { + return null; + } + JsonNode from = from(called); + JsonNode steps = from.has("steps") ? from.get("steps") + : called.has("route") ? called.get("route").get("steps") : null; + Body end = walk(steps, body, new NodePath(PathType.JSON_POINTER), w.called()); + w.seen().remove(normalize(uri)); + if (end instanceof Text t) { + return new Text( + "to: " + normalize(uri) + " returns it as text (" + t.origin() + ")", t.format(), + t.file()); + } else if (end instanceof Parsed p) { + return new Parsed("to: " + normalize(uri) + " returns it as data (" + p.origin() + ")", p.format()); + } + return null; + } + + /** + * What a running integration of the routes saw at an endpoint (CAMEL-25750), when all the bodies were of one kind: + * text for a payload, parsed data for data or a collection. Null when it was not seen, or seen with more than one + * kind, so the walk says nothing there. + */ + private static Body observed(Map<String, ProjectFacts.Observed> observed, String key, String what) { + ProjectFacts.Observed seen = observed.get(key); + if (seen == null || seen.kinds().size() != 1 || seen.types().isEmpty()) { + return null; + } + String origin = "in the running app, " + what + " " + inWords(seen.types()); + return switch (seen.kinds().get(0)) { + case "payload" -> new Text(origin, null, seen.types().stream().anyMatch(TextBodyFlow::isFile)); + case "data", "collection" -> new Parsed(origin, null); + default -> null; + }; + } + + /** Whether a body type is the file as a file consumer reads it. */ + private static boolean isFile(String type) { + String name = simpleName(type); + return name.endsWith("GenericFile") || name.endsWith("RemoteFile"); + } + + /** Body types in words: a String, or a String or a byte[]. */ + private static String inWords(List<String> types) { + StringJoiner sj = new StringJoiner(" or "); + for (String type : types) { + String name = simpleName(type); + sj.add(("AEIOU".indexOf(Character.toUpperCase(name.charAt(0))) >= 0 ? "an " : "a ") + name); + } + return sj.toString(); + } + + private static String simpleName(String type) { + return type.substring(type.lastIndexOf('.') + 1); + } + /** * The body after a setBody: text for a constant, and for a simple template that is more than one ${...}; not known * for any other language, which may return data. @@ -920,7 +984,10 @@ final class TextBodyFlow { * as it is. So the JSON text of the request comes first, with the fields the request needs. */ private static String fix(Need need, Parsed parsed) { - String marshal = "add marshal: " + parsed.format() + " " + need.fix(); + // data a running integration saw has no data format the walk knows of + String marshal = parsed.format() != null + ? "add marshal: " + parsed.format() + " " + need.fix() + : "add marshal with the data format to write (such as marshal: json) " + need.fix(); if (need.request() == null || !"json".equals(parsed.format())) { return marshal + restBinding(need, parsed); } diff --git a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java index e961585b3b6d..976f97598d03 100644 --- a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java +++ b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java @@ -774,6 +774,102 @@ public class TextBodyFlowTest { assertThat(lines(yaml.replace("direct:getStock", "direct:audit"), STOCK_API)).isEmpty(); } + @Test + public void testTheBodyAnHttpConsumerGaveInTheRunningApp() { + // CAMEL-25750: the request of a platform-http route is a stream cache, which the walk alone does not know + String yaml = """ + - route: + id: orders + from: + uri: platform-http:/orders + steps: + - setHeader: + name: qty + expression: + simple: ${body.qty} + """; + assertThat(lines(yaml)).isEmpty(); + + ProjectFacts seen = observed(ProjectFacts.consumerKey("platform-http:/orders"), + List.of("org.apache.camel.converter.stream.ByteArrayInputStreamCache"), List.of("payload")); + assertThat(lines(yaml, seen)).singleElement().satisfies(m -> { + assertThat(m).contains("simple reads a field of the body (${body.qty}), but the body here is still text"); + assertThat(m).contains("in the running app, from: platform-http:/orders gave a ByteArrayInputStreamCache"); + assertThat(m).contains("unmarshal it first with the data format of the payload"); + }); + } + + @Test + public void testTheBodyAProducerReturnedInTheRunningApp() { + // a sql call returns a List of rows, which a file write cannot write as it is + String yaml = """ + - route: + id: report + from: + uri: timer:report + steps: + - to: + uri: sql:select * from orders + - to: + uri: file:reports + """; + assertThat(lines(yaml)).isEmpty(); + + ProjectFacts seen = observed(ProjectFacts.producerKey("sql:select * from orders"), + List.of("java.util.ArrayList"), List.of("collection")); + assertThat(lines(yaml, seen)).singleElement().satisfies(m -> { + assertThat(m).contains("to: file writes the body to a file, but the body here is parsed data"); + assertThat(m).contains("in the running app, to: sql:select * from orders returned an ArrayList"); + // the walk knows no data format for what the running app saw + assertThat(m).contains("add marshal with the data format to write (such as marshal: json) before this step"); + }); + } + + @Test + public void testBodiesOfMoreThanOneKindSayNothing() { + String yaml = """ + - route: + id: report + from: + uri: timer:report + steps: + - to: + uri: sql:select * from orders + - to: + uri: file:reports + """; + // some calls returned rows, others text: the walk cannot tell which one this is + ProjectFacts seen = observed(ProjectFacts.producerKey("sql:select * from orders"), + List.of("java.util.ArrayList", "java.lang.String"), List.of("collection", "payload")); + assertThat(lines(yaml, seen)).isEmpty(); + } + + @Test + public void testWhatTheWalkKnowsComesFirst() { + // the file consumer gives the file as read, whatever a running app saw: the walk knows, and says so + String yaml = """ + - route: + id: orders + from: + uri: file:inbox + steps: + - setHeader: + name: qty + expression: + simple: ${body.qty} + """; + ProjectFacts seen = observed(ProjectFacts.consumerKey("file:inbox"), + List.of("java.util.LinkedHashMap"), List.of("data")); + assertThat(lines(yaml, seen)).singleElement().satisfies(m -> { + assertThat(m).contains("the file as it was read"); + assertThat(m).doesNotContain("running app"); + }); + } + + private static ProjectFacts observed(String key, List<String> types, List<String> kinds) { + return new ProjectFacts(Set.of(), Map.of(), null, Map.of(), Map.of(key, new ProjectFacts.Observed(types, kinds))); + } + private static List<String> lines(String yaml, ProjectFacts facts) { try { return YamlValidator.describeAll(yaml, validator.validate(yaml, facts));
