This is an automated email from the ASF dual-hosted git repository.
davsclaus pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel.git
The following commit(s) were added to refs/heads/main by this push:
new 3390c0cb4812 CAMEL-26416: camel-jbang - AI tools: a field read is
checked against the fields of the JSON data files the routes read (#28724)
3390c0cb4812 is described below
commit 3390c0cb4812523c7b378ebd10f3705b2814de5d
Author: Claus Ibsen <[email protected]>
AuthorDate: Sat Oct 31 07:47:35 2026 +0100
CAMEL-26416: camel-jbang - AI tools: a field read is checked against the
fields of the JSON data files the routes read (#28724)
Signed-off-by: Claus Ibsen <[email protected]>
Co-Authored-By: Claude Opus 5.5 <[email protected]>
---
.../dsl/jbang/core/commands/ai/ValidatorFacts.java | 135 +++++++++++++-
.../jbang/core/commands/ai/ValidatorFactsTest.java | 47 +++++
.../camel/dsl/yaml/validator/ProjectFacts.java | 45 ++++-
.../camel/dsl/yaml/validator/TextBodyFlow.java | 197 +++++++++++++++++++--
.../camel/dsl/yaml/validator/TextBodyFlowTest.java | 115 ++++++++++++
5 files changed, 525 insertions(+), 14 deletions(-)
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java
index 5f8fdbb55957..bb4dea6a8c17 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/main/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFacts.java
@@ -26,11 +26,18 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Stream;
import org.apache.camel.dsl.yaml.validator.ProjectFacts;
+import org.apache.camel.util.json.JsonArray;
+import org.apache.camel.util.json.JsonObject;
+import org.apache.camel.util.json.Jsoner;
+import org.yaml.snakeyaml.LoaderOptions;
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
/**
* What the YAML validator cannot read from a route file, read from the files
beside it (CAMEL-25762): the operations of
@@ -67,7 +74,8 @@ final class ValidatorFacts {
return new ProjectFacts(
OpenApiVerbs.bodylessEndpoints(content, directory),
OpenApiVerbs.operations(content, directory),
restBindingMode(directory),
OpenApiVerbs.requestFields(content, directory),
- ObservedBodies.read(fileName, directory), kamelets,
jsonArrays(content, directory));
+ ObservedBodies.read(fileName, directory), kamelets,
jsonArrays(content, directory),
+ jsonFields(content, directory));
}
/** resource:file:orders.json or resource:classpath:data/orders.json in a
route file. */
@@ -100,6 +108,131 @@ final class ValidatorFacts {
return answer;
}
+ /** The JSON files of one directory beyond this many are not read. */
+ private static final int MAX_JSON_FILES = 50;
+
+ /**
+ * The fields of the JSON data the routes read (r24 of the local-model
benchmark: ${body[lineCount]} on an order
+ * that has lines, and ${body[orderId]} inside the split of its lines): of
the file a resource: loads, and of the
+ * files in the directory a file consumer reads. Data whose files do not
all have one shape is left out.
+ */
+ static Map<String, ProjectFacts.JsonFields> jsonFields(String content,
Path directory) {
+ Map<String, ProjectFacts.JsonFields> answer = new HashMap<>();
+ Matcher m = JSON_RESOURCE.matcher(content);
+ while (m.find()) {
+ String resource = m.group(1);
+ String file = resource.substring(resource.indexOf(':') + 1);
+ List<Path> candidates = resource.startsWith("file:")
+ ? List.of(directory.resolve(file))
+ : List.of(directory.resolve(file),
directory.resolve("src/main/resources").resolve(file));
+ for (Path path : candidates) {
+ ProjectFacts.JsonFields fields = jsonFields(List.of(path),
file);
+ if (fields != null) {
+ answer.put(resource, fields);
+ break;
+ }
+ }
+ }
+ for (String consumed : fileConsumerDirectories(content)) {
+ Path dir = directory.resolve(consumed);
+ if (!Files.isDirectory(dir)) {
+ continue;
+ }
+ try (Stream<Path> files = Files.list(dir)) {
+ List<Path> json = files.filter(f ->
f.getFileName().toString().endsWith(".json")).sorted().toList();
+ // all of them or none: a field may be in a file not read
+ ProjectFacts.JsonFields fields = json.isEmpty() || json.size()
> MAX_JSON_FILES
+ ? null : jsonFields(json, "the files in " +
ProjectFacts.directoryKey(consumed).substring(4) + "/");
+ if (fields != null) {
+ answer.put(ProjectFacts.directoryKey(consumed), fields);
+ }
+ } catch (IOException e) {
+ // the directory is not readable: nothing is known
+ }
+ }
+ return answer;
+ }
+
+ /** The directories the file consumers of a YAML route file read: from:
file:orders, or directoryName: orders. */
+ static List<String> fileConsumerDirectories(String content) {
+ List<String> answer = new ArrayList<>();
+ Object doc;
+ try {
+ doc = new Yaml(new SafeConstructor(new
LoaderOptions())).load(content);
+ } catch (Exception e) {
+ return answer;
+ }
+ if (!(doc instanceof List<?> entries)) {
+ return answer;
+ }
+ for (Object entry : entries) {
+ Object from = entry instanceof Map<?, ?> map
+ ? map.get("route") instanceof Map<?, ?> route ?
route.get("from") : map.get("from")
+ : null;
+ if (!(from instanceof Map<?, ?> f) || !(f.get("uri") instanceof
String uri)) {
+ continue;
+ }
+ String dir = null;
+ if (uri.startsWith("file:")) {
+ dir = uri.substring("file:".length());
+ dir = dir.indexOf('?') >= 0 ? dir.substring(0,
dir.indexOf('?')) : dir;
+ } else if ("file".equals(uri) && f.get("parameters") instanceof
Map<?, ?> parameters
+ && parameters.get("directoryName") instanceof String name)
{
+ dir = name;
+ }
+ if (dir != null && !dir.isBlank() && !dir.contains("{{") &&
!dir.contains("${") && !dir.startsWith("/")) {
+ answer.add(dir.trim());
+ }
+ }
+ return answer;
+ }
+
+ /**
+ * The fields of the JSON in these files: the keys of their objects (of
the elements of a list), and for a field
+ * that holds a list of objects, the keys of those. Null when a file is
not read, is not JSON, or the files differ
+ * in shape (an object in one, a list in another).
+ */
+ static ProjectFacts.JsonFields jsonFields(List<Path> files, String where) {
+ Set<String> fields = new TreeSet<>();
+ Map<String, Set<String>> elementFields = new HashMap<>();
+ Boolean list = null;
+ for (Path path : files) {
+ Object json;
+ try {
+ if (!Files.isRegularFile(path) || Files.size(path) >
MAX_JSON_FILE) {
+ return null;
+ }
+ json = Jsoner.deserialize(Files.readString(path));
+ } catch (Exception e) {
+ return null;
+ }
+ List<JsonObject> objects = new ArrayList<>();
+ if (json instanceof JsonObject o) {
+ objects.add(o);
+ } else if (json instanceof JsonArray a && !a.isEmpty() &&
a.stream().allMatch(JsonObject.class::isInstance)) {
+ a.forEach(e -> objects.add((JsonObject) e));
+ } else {
+ return null;
+ }
+ boolean isList = json instanceof JsonArray;
+ if (list != null && list != isList) {
+ return null;
+ }
+ list = isList;
+ for (JsonObject o : objects) {
+ fields.addAll(o.keySet());
+ for (Map.Entry<String, Object> e : o.entrySet()) {
+ if (e.getValue() instanceof JsonArray a && !a.isEmpty()
+ &&
a.stream().allMatch(JsonObject.class::isInstance)) {
+ Set<String> keys =
elementFields.computeIfAbsent(e.getKey(), k -> new TreeSet<>());
+ a.forEach(x -> keys.addAll(((JsonObject) x).keySet()));
+ }
+ }
+ }
+ }
+ return list == null ? null : new ProjectFacts.JsonFields(where,
fields, elementFields, list, null);
+ }
+
private static boolean isJsonArray(Path path) {
try {
if (!Files.isRegularFile(path) || Files.size(path) >
MAX_JSON_FILE) {
diff --git
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFactsTest.java
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFactsTest.java
index 4c936479575e..efe8f35cccb8 100644
---
a/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFactsTest.java
+++
b/dsl/camel-jbang/camel-jbang-core/src/test/java/org/apache/camel/dsl/jbang/core/commands/ai/ValidatorFactsTest.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.camel.catalog.DefaultCamelCatalog;
+import org.apache.camel.dsl.yaml.validator.ProjectFacts;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -242,6 +243,52 @@ class ValidatorFactsTest {
.noneMatch(m -> m.contains("reply to the REST call"));
}
+ @Test
+ void theFieldsOfTheOrderFilesTheRoutesRead(@TempDir Path dir) throws
Exception {
+ // route-filter-and-multicast and connect-http-client of the
local-model benchmark (r24): ${body[lineCount]} on
+ // an order that has lines, and ${body[orderId]} inside the split of
its lines, were only null at runtime
+ String content = """
+ - route:
+ id: orders
+ from:
+ uri: file
+ parameters:
+ directoryName: orders
+ noop: true
+ steps:
+ - unmarshal:
+ json: {}
+ - log: "Pick ${body[lineCount]} line(s) for
${body[orderId]}"
+ """;
+ Files.createDirectories(dir.resolve("orders"));
+ Files.writeString(dir.resolve("orders/order-1.json"),
+ "{\"orderId\": \"ORD-1\", \"customer\": \"C-1\", \"lines\":
[{\"sku\": \"A\", \"qty\": 2}]}");
+ Files.writeString(dir.resolve("orders/order-2.json"),
+ "{\"orderId\": \"ORD-2\", \"status\": \"paid\", \"lines\":
[{\"sku\": \"B\", \"price\": 9.5}]}");
+
assertThat(ValidatorFacts.fileConsumerDirectories(content)).containsExactly("orders");
+ ProjectFacts.JsonFields fields = ValidatorFacts.jsonFields(content,
dir).get("dir:orders");
+ assertThat(fields.where()).isEqualTo("the files in orders/");
+ assertThat(fields.fields()).containsExactlyInAnyOrder("orderId",
"customer", "status", "lines");
+
assertThat(fields.elementFields().get("lines")).containsExactlyInAnyOrder("sku",
"qty", "price");
+ assertThat(fields.list()).isFalse();
+ assertThat(SourceValidator.validate("orders.camel.yaml", content, new
DefaultCamelCatalog(), null, dir))
+ .singleElement().asString().contains("but the data here has no
field lineCount");
+ // the uri form of the directory
+ assertThat(ValidatorFacts.fileConsumerDirectories(content.replace("""
+ uri: file
+ parameters:
+ directoryName: orders
+ noop: true
+ """, """
+ uri: file:orders?noop=true
+ """))).containsExactly("orders");
+ // a file of another shape among them: nothing is known
+ Files.writeString(dir.resolve("orders/all.json"), "[{\"orderId\":
\"ORD-3\"}]");
+ assertThat(ValidatorFacts.jsonFields(content, dir)).isEmpty();
+ Files.writeString(dir.resolve("orders/all.json"), "not json");
+ assertThat(ValidatorFacts.jsonFields(content, dir)).isEmpty();
+ }
+
@Test
void aJsonArrayFileTheRoutesLoadIsAList(@TempDir Path dir) throws
Exception {
// rest-code-first step 1 of the local-model benchmark:
${body.orderId} on the List of orders.json failed only at
diff --git
a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java
b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java
index 3a690086d4c4..f266612f0e56 100644
---
a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java
+++
b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/ProjectFacts.java
@@ -19,6 +19,7 @@ package org.apache.camel.dsl.yaml.validator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* What the validator cannot read from a route file, read by the caller from
the files beside it (CAMEL-25762): the
@@ -41,10 +42,13 @@ import java.util.Set;
* transformers Camel runs at the start and the end
of their route
* @param jsonArrays the files a resource: of the routes loads whose
JSON is an array, as the resource names them
* ({@code file:orders.json}): unmarshalled, such a
file is a List, not a Map
+ * @param jsonFields the fields of the JSON data the routes read: of
the file a resource: loads, keyed as the
+ * resource names it ({@code file:stock.json}), and
of the files in the directory a file
+ * consumer reads, keyed as {@link
#directoryKey(String)} names it
*/
public record ProjectFacts(Set<String> bodylessEndpoints, Map<String,
Map<String, String>> operations,
String restBindingMode, Map<String, Map<String, List<String>>>
requestFields, Map<String, Observed> observed,
- Map<String, KameletDataTypes> kamelets, Set<String> jsonArrays) {
+ Map<String, KameletDataTypes> kamelets, Set<String> jsonArrays,
Map<String, JsonFields> jsonFields) {
/** Nothing known about the project. */
public static final ProjectFacts NONE = new ProjectFacts(Set.of(),
Map.of(), null);
@@ -68,6 +72,37 @@ public record ProjectFacts(Set<String> bodylessEndpoints,
Map<String, Map<String
public record KameletDataTypes(String in, String out) {
}
+ /**
+ * The fields of JSON data, as the files of the project have them: the
keys of its objects (of the elements, for a
+ * list), and of each field that holds a list of objects, the keys of
those objects. Where says in words where the
+ * data is ("the files in orders/"); outer has the fields of the data an
element was split from, and is empty
+ * otherwise.
+ */
+ public record JsonFields(String where, Set<String> fields, Map<String,
Set<String>> elementFields, boolean list,
+ Set<String> outer) {
+
+ public JsonFields {
+ fields = fields != null ? Set.copyOf(fields) : Set.of();
+ elementFields = elementFields != null
+ ? elementFields.entrySet().stream()
+
.collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, e ->
Set.copyOf(e.getValue())))
+ : Map.of();
+ outer = outer != null ? Set.copyOf(outer) : Set.of();
+ }
+ }
+
+ /** The key of the JSON data in the directory a file consumer reads:
{@code dir:orders}. */
+ public static String directoryKey(String directory) {
+ String d = directory.trim();
+ while (d.startsWith("./")) {
+ d = d.substring(2);
+ }
+ while (d.endsWith("/") && d.length() > 1) {
+ d = d.substring(0, d.length() - 1);
+ }
+ return "dir:" + d;
+ }
+
public ProjectFacts {
bodylessEndpoints = bodylessEndpoints != null ? bodylessEndpoints :
Set.of();
operations = operations != null ? operations : Map.of();
@@ -75,6 +110,14 @@ public record ProjectFacts(Set<String> bodylessEndpoints,
Map<String, Map<String
observed = observed != null ? observed : Map.of();
kamelets = kamelets != null ? kamelets : Map.of();
jsonArrays = jsonArrays != null ? jsonArrays : Set.of();
+ jsonFields = jsonFields != null ? jsonFields : Map.of();
+ }
+
+ /** Without the fields of the JSON data of the project. */
+ public ProjectFacts(Set<String> bodylessEndpoints, Map<String, Map<String,
String>> operations,
+ String restBindingMode, Map<String, Map<String,
List<String>>> requestFields,
+ Map<String, Observed> observed, Map<String,
KameletDataTypes> kamelets, Set<String> jsonArrays) {
+ this(bodylessEndpoints, operations, restBindingMode, requestFields,
observed, kamelets, jsonArrays, Map.of());
}
/** Without the JSON arrays of the project. */
diff --git
a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java
b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java
index a5bba63c028c..c1b2806ebf05 100644
---
a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java
+++
b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/main/java/org/apache/camel/dsl/yaml/validator/TextBodyFlow.java
@@ -25,6 +25,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.StringJoiner;
+import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -194,11 +195,18 @@ final class TextBodyFlow {
String format();
}
- /** Parsed data: the Map or the List an unmarshal makes of the text; list
when it is known to be a List. */
- record Parsed(String origin, String format, boolean list) implements Body {
+ /**
+ * Parsed data: the Map or the List an unmarshal makes of the text; list
when it is known to be a List; the fields
+ * of the JSON data it was parsed from, when the files of the project say
them.
+ */
+ record Parsed(String origin, String format, boolean list,
ProjectFacts.JsonFields fields) implements Body {
Parsed(String origin, String format) {
- this(origin, format, false);
+ this(origin, format, false, null);
+ }
+
+ Parsed(String origin, String format, boolean list) {
+ this(origin, format, list, null);
}
}
@@ -254,7 +262,8 @@ final class TextBodyFlow {
private record Walk(String routeId, Map<String, JsonNode> byFrom,
List<Error> errors, Set<String> seen, boolean post,
Map<String, Map<String, String>> operations, Map<String,
Map<String, List<String>>> requestFields,
Map<String, ProjectFacts.Observed> observed, CatalogBodyTypes
bodyTypes,
- Map<String, ProjectFacts.KameletDataTypes> kamelets, Map<String,
Body> variables, Set<String> jsonArrays) {
+ Map<String, ProjectFacts.KameletDataTypes> kamelets, Map<String,
Body> variables, Set<String> jsonArrays,
+ Map<String, ProjectFacts.JsonFields> jsonFields, Map<String,
List<Body>> sent) {
/**
* The walk of a route a step calls: for its outcome alone, and the
variables of the message it changes. The
@@ -263,17 +272,21 @@ final class TextBodyFlow {
Walk called() {
return new Walk(
null, byFrom, null, seen, false, operations,
requestFields, observed, bodyTypes, kamelets, variables,
- jsonArrays);
+ jsonArrays, jsonFields, sent);
}
/** The walk of the steps of a block: what a branch puts in a variable
stays in the branch. */
Walk branch() {
return new Walk(
routeId, byFrom, errors, seen, post, operations,
requestFields, observed, bodyTypes, kamelets,
- new HashMap<>(variables), jsonArrays);
+ new HashMap<>(variables), jsonArrays, jsonFields, sent);
}
}
+ /** A direct: or seda: route whose body at the start is not known, to walk
again with what the file sends to it. */
+ private record Unknown(String uri, String routeId, JsonNode steps,
NodePath at, boolean post) {
+ }
+
static void check(JsonNode target, NodePath path, List<Error> errors,
ProjectFacts facts, CatalogBodyTypes bodyTypes) {
if (target == null || !target.isArray()) {
return;
@@ -286,6 +299,10 @@ final class TextBodyFlow {
Map<String, ProjectFacts.Observed> observed = facts != null ?
facts.observed() : Map.of();
Map<String, ProjectFacts.KameletDataTypes> kamelets = facts != null ?
facts.kamelets() : Map.of();
Set<String> jsonArrays = facts != null ? facts.jsonArrays() : Set.of();
+ Map<String, ProjectFacts.JsonFields> jsonFields = facts != null ?
facts.jsonFields() : Map.of();
+ // the bodies the routes send to each direct: or seda: endpoint, and
the routes whose start is not known
+ Map<String, List<Body>> sent = new HashMap<>();
+ List<Unknown> unknown = new ArrayList<>();
Map<String, JsonNode> byFrom = new HashMap<>();
for (JsonNode entry : target) {
JsonNode from = from(entry);
@@ -333,9 +350,12 @@ final class TextBodyFlow {
// the message the consumer receives is in the variable
put(variables, received, consumerBody(uri, from, bodyTypes,
"from: " + normalize(uri) + " gives it"));
}
+ if (start == null && !variable && INTERNAL.contains(consumer)) {
+ unknown.add(new Unknown(normalize(uri), id, steps, at, post));
+ }
Body end = walk(steps, start, at, new Walk(
id, byFrom, errors, new HashSet<>(), post, operations,
- requestFields, observed, bodyTypes, kamelets, variables,
jsonArrays));
+ requestFields, observed, bodyTypes, kamelets, variables,
jsonArrays, jsonFields, sent));
// the body at the end of the route of a consumer that replies
with text (platform-http), or of the route of
// a REST call, is the reply
boolean rest = uri != null && restReplies.contains(normalize(uri));
@@ -351,8 +371,37 @@ final class TextBodyFlow {
parsed));
}
}
+ // a direct: or seda: route that every call of the file sends the same
JSON data to: what it reads of that data
+ // is checked against its fields (r24 route-filter-and-multicast step
2: ${body[lineCount]} in the route the
+ // multicast sends the order to). Only those reports: the route was
walked already
+ for (Unknown route : unknown) {
+ List<Body> bodies = sent.get(route.uri());
+ if (bodies == null || bodies.isEmpty() || !(bodies.get(0)
instanceof Parsed first) || first.fields() == null
+ || first.list()
+ || bodies.stream().anyMatch(b -> !(b instanceof Parsed p)
|| !first.fields().equals(p.fields()))) {
+ continue;
+ }
+ List<Error> found = new ArrayList<>();
+ walk(route.steps(), new Parsed(
+ "every call of " + route.uri() + " in the file sends it",
first.format(), false,
+ first.fields()), route.at(),
+ new Walk(
+ route.routeId(), byFrom, found, new HashSet<>(),
route.post(), operations, requestFields,
+ observed, bodyTypes, kamelets, new HashMap<>(),
jsonArrays, jsonFields, new HashMap<>()));
+ for (Error e : found) {
+ if (String.valueOf(e.getMessage()).contains(NO_FIELD)) {
+ errors.add(e);
+ }
+ }
+ }
}
+ /** The components a route of the file is reached through, with the body
of its caller. */
+ private static final Set<String> INTERNAL = Set.of("direct", "seda",
"direct-vm");
+
+ /** What a report of a field the JSON data does not have says. */
+ private static final String NO_FIELD = ", but the data here has no field ";
+
/**
* Whether the steps set or read an HTTP control header (CamelHttpMethod,
CamelHttpQuery...), which can make an http
* producer call with GET and leave the body out. A match anywhere in the
steps only turns the http check off for
@@ -379,9 +428,9 @@ final class TextBodyFlow {
return null;
}
if (out.payload()) {
- // the file as read has the format of the files it reads
+ // the file as read has the format of the files it reads, and the
data of the files in its directory
return out.file()
- ? new Text(consumer, fileFormat(from), true)
+ ? new Text(consumer, fileFormat(from), true, false,
directoryKey(uri, from))
: new Text(consumer + " as " + out.inWords(),
out.dataFormat());
}
return out.data() ? new Parsed(consumer + " as " + out.inWords(),
out.dataFormat()) : null;
@@ -451,6 +500,9 @@ final class TextBodyFlow {
String name = step.fieldNames().next();
JsonNode value = step.get(name);
NodePath here = at.append(j).append(name);
+ if (w.errors() != null) {
+ recordSend(name, value, body, w);
+ }
boolean splitWalked = false;
if (w.errors() != null && parsesData(name, value, body, here, w)) {
// the unmarshal fails at runtime, so what comes after it is
not run: one report
@@ -482,6 +534,12 @@ final class TextBodyFlow {
body = null;
continue;
} else if (body instanceof Parsed parsed) {
+ String unknownField = parsed.list() || parsed.fields() ==
null
+ ? null : unknownField(name, value,
parsed.fields());
+ if (unknownField != null) {
+ // a field that is not there is null, and the route
goes on
+ w.errors().add(fieldError(here, w.routeId(),
unknownField, parsed.fields()));
+ }
Need need = needsText(name, value, w);
Parsed element = "split".equals(name) ?
splitElement(value, parsed, w.bodyTypes()) : null;
if (need != null) {
@@ -550,6 +608,11 @@ final class TextBodyFlow {
if (read != null) {
w.errors().add(listError(at.append(key), w.routeId(),
read, parsed));
}
+ } else if (body instanceof Parsed parsed && parsed.fields() !=
null) {
+ String read = unknownField(key, child, parsed.fields());
+ if (read != null) {
+ w.errors().add(fieldError(at.append(key), w.routeId(),
read, parsed.fields()));
+ }
}
}
}
@@ -573,12 +636,16 @@ final class TextBodyFlow {
}
case "unmarshal" -> {
Parsed parsed = unmarshalled(value, w.bodyTypes());
+ ProjectFacts.JsonFields fields = parsed != null &&
"json".equals(parsed.format())
+ && body instanceof Text text && text.resource() != null
+ ? w.jsonFields().get(text.resource()) : null;
if (parsed != null && "json".equals(parsed.format()) && body
instanceof Text text
&& text.resource() != null &&
w.jsonArrays().contains(text.resource())) {
String file =
text.resource().substring(text.resource().indexOf(':') + 1);
- return new Parsed("unmarshal: json turns the JSON array of
" + file + " into a List", "json", true);
+ return new Parsed(
+ "unmarshal: json turns the JSON array of " + file
+ " into a List", "json", true, fields);
}
- return parsed;
+ return fields != null ? new Parsed(parsed.origin(), "json",
fields.list(), fields) : parsed;
}
case "marshal" -> {
return marshalled(value, body);
@@ -1140,7 +1207,7 @@ final class TextBodyFlow {
if (readIn(step.get(name), TEXT_PROPERTIES) != null ||
readsAFieldWithJsonPath(step.get(name))) {
return new Parsed(
parsed.origin() + ", and split " + expression.trim() +
" gives its elements as data",
- parsed.format());
+ parsed.format(), false, "simple".equals(language) ?
elementFields(parsed, expression) : null);
}
if (!bodyTypes.keepsTheBody(name, step.get(name))) {
return null;
@@ -1695,6 +1762,112 @@ final class TextBodyFlow {
.build();
}
+ /** The key of the JSON data in the directory a file consumer reads, or
null for any other consumer. */
+ private static String directoryKey(String uri, JsonNode from) {
+ if (!"file".equals(scheme(uri))) {
+ return null;
+ }
+ String path = uri.indexOf(':') > 0 ?
normalize(uri).substring(uri.indexOf(':') + 1) : null;
+ if ((path == null || path.isBlank()) && from != null &&
from.isObject() && from.get("parameters") != null
+ && from.get("parameters").get("directoryName") != null) {
+ path = from.get("parameters").get("directoryName").asText();
+ }
+ return path != null && !path.isBlank() ?
ProjectFacts.directoryKey(path) : null;
+ }
+
+ /** Remembers the body a step sends to a direct: or seda: endpoint, for
the route of the file that consumes it. */
+ private static void recordSend(String name, JsonNode value, Body body,
Walk w) {
+ if (!SENDS_THE_BODY.contains(name)) {
+ return;
+ }
+ String uri = sendUri(name, value);
+ if (uri != null && INTERNAL.contains(scheme(uri)) &&
!uri.contains("${")) {
+ w.sent().computeIfAbsent(normalize(uri), k -> new
ArrayList<>()).add(body);
+ }
+ }
+
+ /** ${body.lines}, ${body[lines]} or ${body['lines']}: one field of the
body, and nothing else. */
+ private static final Pattern ONE_FIELD
+ =
Pattern.compile("\\$\\{body(?:\\??\\.([A-Za-z_]\\w*)|\\[\\s*['\"]?([A-Za-z_]\\w*)['\"]?\\s*])}");
+
+ /** The names a field read may be that are not fields of the data: what a
Map or a List has itself. */
+ private static final Set<String> COLLECTION_PROPERTIES = Set.of("size",
"empty", "class", "length", "isEmpty",
+ "keySet", "values", "entrySet");
+
+ /**
+ * The first read of a field of the body that the JSON data does not have,
in the simple expressions and templates
+ * of a step (not of its nested steps), as written; null when the data has
every field read.
+ */
+ private static String unknownField(String name, JsonNode value,
ProjectFacts.JsonFields fields) {
+ List<String> texts = new ArrayList<>();
+ if ("simple".equals(name)) {
+ texts.add(expressionText(value));
+ } else {
+ simpleExpressions(value, texts);
+ templates(name, value).forEach(t -> texts.add(t.text()));
+ }
+ for (String text : texts) {
+ Matcher m = text != null ? FIELD_READ.matcher(text) : null;
+ while (m != null && m.find()) {
+ String field = m.group(1);
+ boolean call = m.end() < text.length() && text.charAt(m.end())
== '(';
+ if (!call && !COLLECTION_PROPERTIES.contains(field) &&
!fields.fields().contains(field)) {
+ int end = text.indexOf('}', m.start());
+ return end > 0 ? text.substring(m.start(), end + 1) :
m.group();
+ }
+ }
+ }
+ return null;
+ }
+
+ /**
+ * The fields of the elements a simple split of a field of the data gives:
${body[lines]} of an order gives its
+ * lines; ${body} of a list its elements. Null when not known.
+ */
+ private static ProjectFacts.JsonFields elementFields(Parsed parsed, String
expression) {
+ ProjectFacts.JsonFields fields = parsed.fields();
+ String e = expression.trim();
+ if (fields == null || !THE_BODY.matcher(e).matches()) {
+ return null;
+ }
+ if ("${body}".equals(e)) {
+ return fields.list()
+ ? new ProjectFacts.JsonFields(
+ "one element of " + fields.where(),
fields.fields(), fields.elementFields(), false, null)
+ : null;
+ }
+ Matcher m = ONE_FIELD.matcher(e);
+ String field = !fields.list() && m.matches() ? m.group(1) != null ?
m.group(1) : m.group(2) : null;
+ Set<String> element = field != null ?
fields.elementFields().get(field) : null;
+ return element != null
+ ? new ProjectFacts.JsonFields(
+ "one of the " + field + " of " + fields.where(),
element, Map.of(), false, fields.fields())
+ : null;
+ }
+
+ private static Error fieldError(NodePath at, String routeId, String read,
ProjectFacts.JsonFields fields) {
+ Matcher m = FIELD_READ.matcher(read);
+ String field = m.find() ? m.group(1) : read;
+ String advice;
+ if (fields.outer().contains(field)) {
+ advice = field + " is a field of the data that was split, not of
its element: read it before the split,"
+ + " into a variable (setVariable), and write ${variable."
+ field + "} in the split";
+ } else {
+ String list =
fields.elementFields().keySet().stream().sorted().findFirst().orElse(null);
+ advice = "read one of those"
+ + (list != null ? " (the number of " + list + " is
${body[" + list + "].size()})" : "");
+ }
+ return Error.builder()
+ .keyword("type")
+ .instanceLocation(at)
+ .messageKey("type")
+ .format(new MessageFormat("{0}"))
+ .arguments((routeId != null ? "route " + routeId + ": " : "")
+ "simple reads " + read + NO_FIELD
+ + field + " (the fields of " + fields.where() + ": "
+ + String.join(", ", new TreeSet<>(fields.fields()))
+ "): " + advice)
+ .build();
+ }
+
/**
* An http call that sends empty text as a POST: the http producer sends a
POST for any body that is not null, and
* empty text is a body (HttpMethodHelper). r24 connect-http-client: a
setBody with constant "" before a GET of the
diff --git
a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java
b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java
index f7be71e5a827..4887f00f528f 100644
---
a/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java
+++
b/dsl/camel-yaml-dsl/camel-yaml-dsl-validator/src/test/java/org/apache/camel/dsl/yaml/validator/TextBodyFlowTest.java
@@ -2298,6 +2298,121 @@ public class TextBodyFlowTest {
.isEmpty();
}
+ /**
+ * The project has order files in orders/, each an order with lines: the
facts the AI tools read beside the route.
+ */
+ private static final ProjectFacts ORDER_FILES = new ProjectFacts(
+ Set.of(), Map.of(), null, Map.of(), Map.of(), Map.of(), Set.of(),
+ Map.of("dir:orders", new ProjectFacts.JsonFields(
+ "the files in orders/", Set.of("orderId", "customer",
"country", "status", "lines"),
+ Map.of("lines", Set.of("sku", "qty", "price")), false,
null)));
+
+ /** A route that reads the order files and unmarshals them, then the steps
given (written from column 0). */
+ private static String orders(String steps) {
+ return """
+ - route:
+ id: orders
+ from:
+ uri: file
+ parameters:
+ directoryName: orders
+ noop: true
+ steps:
+ - unmarshal:
+ json: {}
+ """ + steps.indent(8);
+ }
+
+ @Test
+ public void testAFieldTheJsonDataDoesNotHave() {
+ // r24 route-filter-and-multicast step 2: ${body[lineCount]} on an
order, which has lines
+ assertThat(lines(orders("- log: \"Pick ${body[lineCount]} line(s) for
${body[orderId]}\"\n"), ORDER_FILES))
+ .singleElement().satisfies(m -> {
+ assertThat(m).contains("route orders: simple reads
${body[lineCount]}, but the data here has no field"
+ + " lineCount (the fields of the
files in orders/: country, customer, lines,"
+ + " orderId, status)");
+ assertThat(m).contains("read one of those (the number of
lines is ${body[lines].size()})");
+ });
+ // the fields the data has, a method of the Map, and a predicate
+ assertThat(lines(orders("- log: \"Pick ${body[lines].size()} line(s)
for ${body.orderId}\"\n"), ORDER_FILES))
+ .isEmpty();
+ assertThat(lines(orders("- log: \"${body.size()} fields\"\n"),
ORDER_FILES)).isEmpty();
+ assertThat(lines(orders("""
+ - filter:
+ simple: "${body[state]} == 'paid'"
+ steps:
+ - log: "paid"
+ """), ORDER_FILES)).singleElement().asString().contains("has
no field state");
+ // nothing known without the facts, or after a step that changes the
body
+ assertThat(lines(orders("- log: \"${body[lineCount]}\"\n"))).isEmpty();
+ assertThat(lines(orders("""
+ - setBody:
+ groovy: "[lineCount: body.lines.size()]"
+ - log: "${body[lineCount]}"
+ """), ORDER_FILES)).isEmpty();
+ }
+
+ @Test
+ public void testAFieldOfTheOrderReadInTheSplitOfItsLines() {
+ // r24 connect-http-client step 2: ${body[orderId]} inside the split
of the lines, which have no orderId
+ String yaml = orders("""
+ - split:
+ simple: "${body[lines]}"
+ steps:
+ - setVariable:
+ name: orderId
+ simple: "${body[orderId]}"
+ - log: "${variable.orderId}: ${body[sku]} x ${body[qty]}"
+ """);
+ assertThat(lines(yaml, ORDER_FILES)).singleElement().satisfies(m -> {
+ assertThat(m).contains("simple reads ${body[orderId]}, but the
data here has no field orderId (the fields of"
+ + " one of the lines of the files in
orders/: price, qty, sku)");
+ assertThat(m).contains("orderId is a field of the data that was
split, not of its element: read it before"
+ + " the split, into a variable
(setVariable), and write ${variable.orderId} in the"
+ + " split");
+ });
+ assertThat(lines(yaml.replace("${body[orderId]}", "${body[sku]}"),
ORDER_FILES)).isEmpty();
+ }
+
+ @Test
+ public void testADirectRouteThatEveryCallSendsTheData() {
+ // r24 route-filter-and-multicast step 2: the route the multicast
sends the order to reads ${body[lineCount]}
+ String yaml = orders("""
+ - filter:
+ simple: "${body[status]} == 'paid'"
+ steps:
+ - multicast:
+ steps:
+ - to: direct:warehouse
+ - to: direct:invoicing
+ """) + """
+ - route:
+ id: warehouse
+ from:
+ uri: direct:warehouse
+ steps:
+ - log: "Warehouse: pick ${body[lineCount]} line(s) for
${body[orderId]}"
+ - route:
+ id: invoicing
+ from:
+ uri: direct:invoicing
+ steps:
+ - log: "Invoicing: bill customer ${body[customer]} for
${body[orderId]}"
+ """;
+ assertThat(lines(yaml, ORDER_FILES)).singleElement().satisfies(m -> {
+ assertThat(m).contains("route warehouse: simple reads
${body[lineCount]}, but the data here has no field"
+ + " lineCount");
+ });
+ // a call that sends something else: the route's body is not known
+ assertThat(lines(yaml + """
+ - route:
+ from:
+ uri: timer:tick
+ steps:
+ - to: direct:warehouse
+ """, ORDER_FILES)).isEmpty();
+ }
+
/** The project has orders.json, a JSON array: the facts the AI tools read
beside the route file. */
private static final ProjectFacts ORDERS_ARRAY
= new ProjectFacts(Set.of(), Map.of(), null, Map.of(), Map.of(),
Map.of(), Set.of("file:orders.json"));
