This is an automated email from the ASF dual-hosted git repository. davsclaus pushed a commit to branch CAMEL-27225-4.22.x in repository https://gitbox.apache.org/repos/asf/camel.git
commit 026a22d885d604ffbc8579a67d585ebf87351aab Author: Claus Ibsen <[email protected]> AuthorDate: Thu Nov 19 11:51:42 2026 +0100 CAMEL-27225: camel-redis - RedisAggregationRepository stores exchanges in Java serialization and reads them through a deserialization filter RedisAggregationRepository stores the DefaultExchangeHolder of each exchange in Java serialization, with the correlation key (or exchange id) as a plain string field, and reads it back through an ObjectInputFilter. New deserializationFilter option (jdk.serialFilter syntax; default: the JVM-wide filter, otherwise the Camel default filter), and a new codec option to keep using a Redisson codec for the hashes, such as Kryo5Codec to read the entries stored by older versions. At start, a WARN reports entries in another format among the first 100 of each hash, and recover() skips a completed exchange it cannot read, with a WARN. Backport of #30368 to camel-4.22.x, adapted to the RedisAggregationRepository of this branch. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Claus Ibsen <[email protected]> --- .../catalog/beans/RedisAggregationRepository.json | 2 +- .../org/apache/camel/catalog/docs/redis.adoc | 38 ++ .../RedisAggregationRepositoryConfigurer.java | 9 + .../camel/bean/RedisAggregationRepository.json | 2 +- components/camel-redis/src/main/docs/redis.adoc | 38 ++ .../aggregate/RedisAggregationRepository.java | 248 +++++++++++- ...isAggregationRepositoryDeserializationTest.java | 423 +++++++++++++++++++++ .../RedisAggregationRepositoryCodecIT.java | 108 ++++++ .../RedisAggregationRepositoryOperationsIT.java | 63 ++- 9 files changed, 909 insertions(+), 22 deletions(-) diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/beans/RedisAggregationRepository.json b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/beans/RedisAggregationRepository.json index c9a94101ccb5..7575c8b33495 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/beans/RedisAggregationRepository.json +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/beans/RedisAggregationRepository.json @@ -10,7 +10,7 @@ "groupId": "org.apache.camel", "artifactId": "camel-redis", "version": "4.22.3-SNAPSHOT", - "properties": { "redisson": { "index": 0, "kind": "property", "displayName": "Redisson", "label": "advanced", "required": false, "type": "object", "javaType": "org.redisson.api.RedissonClient", "deprecated": false, "autowired": false, "secret": false, "description": "To use an existing Redis client to connect to Redis server" }, "endpoint": { "index": 1, "kind": "property", "displayName": "Endpoint", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": fa [...] + "properties": { "redisson": { "index": 0, "kind": "property", "displayName": "Redisson", "label": "advanced", "required": false, "type": "object", "javaType": "org.redisson.api.RedissonClient", "deprecated": false, "autowired": false, "secret": false, "description": "To use an existing Redis client to connect to Redis server" }, "endpoint": { "index": 1, "kind": "property", "displayName": "Endpoint", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": fa [...] } } diff --git a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/redis.adoc b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/redis.adoc index 31cd6f779579..5a97da221a38 100644 --- a/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/redis.adoc +++ b/catalog/camel-catalog/src/generated/resources/org/apache/camel/catalog/docs/redis.adoc @@ -10,3 +10,41 @@ *Since Camel {since}* The Redis component provides an `AggregationStrategy` to use Redis as the backend datastore. + +== Storage format + +The `RedisAggregationRepository` stores each exchange in a Redis hash: the field is the correlation key as a string, +and the value is the exchange (its body, headers, exchange properties and variables) in Java serialization. +The codec configured on a `RedissonClient` given with the `redisson` option is not used for these hashes, unless it is +also given with the `codec` option. + +When it reads an exchange back, the repository only accepts what the `deserializationFilter` option allows, a pattern +in the syntax of the `jdk.serialFilter` system property. When the option is not set, the JVM-wide `jdk.serialFilter` +applies if it is set, otherwise the default filter: + +---- +!java.net.**;java.**;javax.**;org.apache.camel.**;maxdepth=20;maxrefs=10000;maxbytes=10485760;!* +---- + +It allows the classes of the `java`, `javax` and `org.apache.camel` packages (except `java.net`) and rejects the other +classes. It also limits the size of a stored exchange: `maxdepth=20` (the depth of the object graph), `maxrefs=10000` +(the number of objects in it) and `maxbytes=10485760` (10 MB). An exchange over a limit cannot be read back, so its +aggregation does not complete. This happens, for example, with a `List` body that collects more than 10000 strings, or +with a body of more than 10 MB. + +Set the `deserializationFilter` option when the message bodies or headers of the aggregated exchanges are instances of +your own classes, or when the exchanges are larger. The option replaces the whole default pattern, so keep its other +parts. For example, to allow the classes of `com.foo.model` and raise the limits: + +---- +!java.net.**;com.foo.model.**;java.**;javax.**;org.apache.camel.**;maxdepth=50;maxrefs=1000000;maxbytes=104857600;!* +---- + +or leave out the `maxdepth`, `maxrefs` and `maxbytes` parts to have no limits. The `codec` option below is the other +way, without the filter. + +The `codec` option sets a Redisson `Codec` for the hashes of the repository, in place of the Java serialization. The +codec then encodes and decodes the exchanges, and the `deserializationFilter` option does not apply: the codec reads +back whatever classes it is given, so only use it when the Redis data is trusted. Camel 4.22.2 and older stored the +exchanges with the default codec of the Redisson client, `org.redisson.codec.Kryo5Codec`: set the `codec` option to an +instance of it to read the exchanges stored by these versions. diff --git a/components/camel-redis/src/generated/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryConfigurer.java b/components/camel-redis/src/generated/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryConfigurer.java index b696d8441c05..4246460d6615 100644 --- a/components/camel-redis/src/generated/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryConfigurer.java +++ b/components/camel-redis/src/generated/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryConfigurer.java @@ -25,8 +25,11 @@ public class RedisAggregationRepositoryConfigurer extends org.apache.camel.suppo switch (ignoreCase ? name.toLowerCase() : name) { case "allowserializedheaders": case "allowSerializedHeaders": target.setAllowSerializedHeaders(property(camelContext, boolean.class, value)); return true; + case "codec": target.setCodec(property(camelContext, org.redisson.client.codec.Codec.class, value)); return true; case "deadletteruri": case "deadLetterUri": target.setDeadLetterUri(property(camelContext, java.lang.String.class, value)); return true; + case "deserializationfilter": + case "deserializationFilter": target.setDeserializationFilter(property(camelContext, java.lang.String.class, value)); return true; case "endpoint": target.setEndpoint(property(camelContext, java.lang.String.class, value)); return true; case "mapname": case "mapName": target.setMapName(property(camelContext, java.lang.String.class, value)); return true; @@ -49,8 +52,11 @@ public class RedisAggregationRepositoryConfigurer extends org.apache.camel.suppo switch (ignoreCase ? name.toLowerCase() : name) { case "allowserializedheaders": case "allowSerializedHeaders": return boolean.class; + case "codec": return org.redisson.client.codec.Codec.class; case "deadletteruri": case "deadLetterUri": return java.lang.String.class; + case "deserializationfilter": + case "deserializationFilter": return java.lang.String.class; case "endpoint": return java.lang.String.class; case "mapname": case "mapName": return java.lang.String.class; @@ -74,8 +80,11 @@ public class RedisAggregationRepositoryConfigurer extends org.apache.camel.suppo switch (ignoreCase ? name.toLowerCase() : name) { case "allowserializedheaders": case "allowSerializedHeaders": return target.isAllowSerializedHeaders(); + case "codec": return target.getCodec(); case "deadletteruri": case "deadLetterUri": return target.getDeadLetterUri(); + case "deserializationfilter": + case "deserializationFilter": return target.getDeserializationFilter(); case "endpoint": return target.getEndpoint(); case "mapname": case "mapName": return target.getMapName(); diff --git a/components/camel-redis/src/generated/resources/META-INF/services/org/apache/camel/bean/RedisAggregationRepository.json b/components/camel-redis/src/generated/resources/META-INF/services/org/apache/camel/bean/RedisAggregationRepository.json index c9a94101ccb5..7575c8b33495 100644 --- a/components/camel-redis/src/generated/resources/META-INF/services/org/apache/camel/bean/RedisAggregationRepository.json +++ b/components/camel-redis/src/generated/resources/META-INF/services/org/apache/camel/bean/RedisAggregationRepository.json @@ -10,7 +10,7 @@ "groupId": "org.apache.camel", "artifactId": "camel-redis", "version": "4.22.3-SNAPSHOT", - "properties": { "redisson": { "index": 0, "kind": "property", "displayName": "Redisson", "label": "advanced", "required": false, "type": "object", "javaType": "org.redisson.api.RedissonClient", "deprecated": false, "autowired": false, "secret": false, "description": "To use an existing Redis client to connect to Redis server" }, "endpoint": { "index": 1, "kind": "property", "displayName": "Endpoint", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": fa [...] + "properties": { "redisson": { "index": 0, "kind": "property", "displayName": "Redisson", "label": "advanced", "required": false, "type": "object", "javaType": "org.redisson.api.RedissonClient", "deprecated": false, "autowired": false, "secret": false, "description": "To use an existing Redis client to connect to Redis server" }, "endpoint": { "index": 1, "kind": "property", "displayName": "Endpoint", "required": true, "type": "string", "javaType": "java.lang.String", "deprecated": fa [...] } } diff --git a/components/camel-redis/src/main/docs/redis.adoc b/components/camel-redis/src/main/docs/redis.adoc index 31cd6f779579..5a97da221a38 100644 --- a/components/camel-redis/src/main/docs/redis.adoc +++ b/components/camel-redis/src/main/docs/redis.adoc @@ -10,3 +10,41 @@ *Since Camel {since}* The Redis component provides an `AggregationStrategy` to use Redis as the backend datastore. + +== Storage format + +The `RedisAggregationRepository` stores each exchange in a Redis hash: the field is the correlation key as a string, +and the value is the exchange (its body, headers, exchange properties and variables) in Java serialization. +The codec configured on a `RedissonClient` given with the `redisson` option is not used for these hashes, unless it is +also given with the `codec` option. + +When it reads an exchange back, the repository only accepts what the `deserializationFilter` option allows, a pattern +in the syntax of the `jdk.serialFilter` system property. When the option is not set, the JVM-wide `jdk.serialFilter` +applies if it is set, otherwise the default filter: + +---- +!java.net.**;java.**;javax.**;org.apache.camel.**;maxdepth=20;maxrefs=10000;maxbytes=10485760;!* +---- + +It allows the classes of the `java`, `javax` and `org.apache.camel` packages (except `java.net`) and rejects the other +classes. It also limits the size of a stored exchange: `maxdepth=20` (the depth of the object graph), `maxrefs=10000` +(the number of objects in it) and `maxbytes=10485760` (10 MB). An exchange over a limit cannot be read back, so its +aggregation does not complete. This happens, for example, with a `List` body that collects more than 10000 strings, or +with a body of more than 10 MB. + +Set the `deserializationFilter` option when the message bodies or headers of the aggregated exchanges are instances of +your own classes, or when the exchanges are larger. The option replaces the whole default pattern, so keep its other +parts. For example, to allow the classes of `com.foo.model` and raise the limits: + +---- +!java.net.**;com.foo.model.**;java.**;javax.**;org.apache.camel.**;maxdepth=50;maxrefs=1000000;maxbytes=104857600;!* +---- + +or leave out the `maxdepth`, `maxrefs` and `maxbytes` parts to have no limits. The `codec` option below is the other +way, without the filter. + +The `codec` option sets a Redisson `Codec` for the hashes of the repository, in place of the Java serialization. The +codec then encodes and decodes the exchanges, and the `deserializationFilter` option does not apply: the codec reads +back whatever classes it is given, so only use it when the Redis data is trusted. Camel 4.22.2 and older stored the +exchanges with the default codec of the Redisson client, `org.redisson.codec.Kryo5Codec`: set the `codec` option to an +instance of it to read the exchanges stored by these versions. diff --git a/components/camel-redis/src/main/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepository.java b/components/camel-redis/src/main/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepository.java index 214fbf4da58f..dc52ee63f093 100644 --- a/components/camel-redis/src/main/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepository.java +++ b/components/camel-redis/src/main/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepository.java @@ -16,9 +16,16 @@ */ package org.apache.camel.component.redis.processor.aggregate; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InvalidClassException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; import java.util.Collections; import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import org.apache.camel.CamelContext; @@ -28,6 +35,7 @@ import org.apache.camel.spi.Configurer; import org.apache.camel.spi.Metadata; import org.apache.camel.spi.OptimisticLockingAggregationRepository; import org.apache.camel.spi.RecoverableAggregationRepository; +import org.apache.camel.support.CamelObjectInputStream; import org.apache.camel.support.DefaultExchange; import org.apache.camel.support.DefaultExchangeHolder; import org.apache.camel.support.service.ServiceSupport; @@ -38,12 +46,22 @@ import org.redisson.api.RMap; import org.redisson.api.RTransaction; import org.redisson.api.RedissonClient; import org.redisson.api.TransactionOptions; +import org.redisson.client.codec.ByteArrayCodec; +import org.redisson.client.codec.Codec; +import org.redisson.client.codec.StringCodec; +import org.redisson.codec.CompositeCodec; import org.redisson.config.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * {@link org.apache.camel.spi.AggregationRepository} using Redis as store. + * <p/> + * The exchanges are stored in Redis hashes: the field is the correlation key (or the exchange id in the persistence + * map) as a string, and the value is the {@link DefaultExchangeHolder} of the exchange in Java serialization. The + * values are read back through an {@link java.io.ObjectInputFilter}, see {@link #setDeserializationFilter(String)}. The + * codec configured on the {@link RedissonClient} is not used for these hashes, unless it is given with + * {@link #setCodec(Codec)}, which replaces the Java serialization and its filter. */ @Metadata(label = "bean", description = "Aggregation repository that uses Redis to store exchanges.", @@ -54,8 +72,24 @@ public class RedisAggregationRepository extends ServiceSupport private static final Logger LOG = LoggerFactory.getLogger(RedisAggregationRepository.class); private static final String COMPLETED_SUFFIX = "-completed"; - private Map<String, DefaultExchangeHolder> cache; - private Map<String, DefaultExchangeHolder> persistedCache; + /** + * The codec of the hashes: the fields are strings and the values are the serialized {@link DefaultExchangeHolder}. + */ + private static final Codec CODEC + = new CompositeCodec(StringCodec.INSTANCE, ByteArrayCodec.INSTANCE, ByteArrayCodec.INSTANCE); + + /** + * The number of entries checked in each hash at startup, which is also the number read per HSCAN call. + */ + private static final int SCAN_COUNT = 100; + + private RMap<String, Object> cache; + private RMap<String, Object> persistedCache; + private Codec mapCodec; + /** + * The ids of the completed exchanges that could not be read, to log the WARN once per exchange. + */ + private final Set<String> unreadableExchangeIds = ConcurrentHashMap.newKeySet(); private boolean shutdownRedisson; @Metadata(label = "advanced", description = "To use an existing Redis client to connect to Redis server") @@ -81,6 +115,21 @@ public class RedisAggregationRepository extends ServiceSupport @Metadata(label = "advanced", security = "insecure:serialization", description = "Whether headers on the Exchange that are Java objects and Serializable should be included and saved to the repository") private boolean allowSerializedHeaders; + @Metadata(label = "advanced,security", + description = "Sets an ObjectInputFilter pattern (jdk.serialFilter syntax) applied when deserializing" + + " the exchanges read back from Redis. When not set, the JVM-wide jdk.serialFilter is" + + " used if present; otherwise a conservative default filter denying java.net.* and" + + " otherwise allowing java.*, javax.* and org.apache.camel.* packages is applied. Widen" + + " this pattern when the message bodies or headers stored in the repository are instances" + + " of your own classes.") + private String deserializationFilter; + @Metadata(label = "advanced,security", security = "insecure:serialization", + description = "The Redisson codec of the Redis hashes of the repository, in place of the Java serialization" + + " read through the deserialization filter. When set, the codec encodes and decodes the" + + " exchanges, and the deserializationFilter option does not apply. For example, a" + + " org.redisson.codec.Kryo5Codec reads the exchanges stored by Camel 4.22.2 or older, which" + + " used the default codec of the Redisson client.") + private Codec codec; public RedisAggregationRepository() { } @@ -119,18 +168,18 @@ public class RedisAggregationRepository extends ServiceSupport } LOG.trace("Adding an Exchange with ID {} for key {} in an optimistic manner.", newExchange.getExchangeId(), key); if (oldExchange == null) { - DefaultExchangeHolder holder = DefaultExchangeHolder.marshal(newExchange, true, allowSerializedHeaders); - final DefaultExchangeHolder misbehaviorHolder = cache.putIfAbsent(key, holder); + Object holder = marshallHolder(newExchange); + final Object misbehaviorHolder = cache.putIfAbsent(key, holder); if (misbehaviorHolder != null) { - Exchange misbehaviorEx = unmarshallExchange(camelContext, misbehaviorHolder); + Exchange misbehaviorEx = unmarshallExchange(camelContext, mapName, key, misbehaviorHolder); LOG.warn( "Optimistic locking failed for exchange with key {}: RMap#putIfAbsent returned Exchange with ID {}, while it's expected no exchanges to be returned", key, misbehaviorEx != null ? misbehaviorEx.getExchangeId() : "<null>"); throw new OptimisticLockingException(); } } else { - DefaultExchangeHolder oldHolder = DefaultExchangeHolder.marshal(oldExchange, true, allowSerializedHeaders); - DefaultExchangeHolder newHolder = DefaultExchangeHolder.marshal(newExchange, true, allowSerializedHeaders); + Object oldHolder = marshallHolder(oldExchange); + Object newHolder = marshallHolder(newExchange); if (!cache.replace(key, oldHolder, newHolder)) { LOG.warn( "Optimistic locking failed for exchange with key {}: RMap#replace returned no Exchanges, while it's expected to replace one", @@ -148,12 +197,12 @@ public class RedisAggregationRepository extends ServiceSupport throw new UnsupportedOperationException(); } LOG.trace("Adding an Exchange with ID {} for key {} in a thread-safe manner.", exchange.getExchangeId(), key); - DefaultExchangeHolder newHolder = DefaultExchangeHolder.marshal(exchange, true, allowSerializedHeaders); + Object newHolder = marshallHolder(exchange); RLock lock = redisson.getLock(mapName + "-lock-" + key); try { lock.lock(); - DefaultExchangeHolder oldHolder = cache.put(key, newHolder); - return unmarshallExchange(camelContext, oldHolder); + Object oldHolder = cache.put(key, newHolder); + return unmarshallExchange(camelContext, mapName, key, oldHolder); } finally { LOG.trace("Added an Exchange with ID {} for key {} in a thread-safe manner.", exchange.getExchangeId(), key); lock.unlock(); @@ -178,7 +227,31 @@ public class RedisAggregationRepository extends ServiceSupport @Override public Exchange recover(CamelContext camelContext, String exchangeId) { LOG.trace("Recovering an Exchange with ID {}.", exchangeId); - return useRecovery ? unmarshallExchange(camelContext, persistedCache.get(exchangeId)) : null; + if (!useRecovery) { + return null; + } + try { + return unmarshallExchange(camelContext, persistenceMapName, exchangeId, persistedCache.get(exchangeId)); + } catch (RuntimeException e) { + // skip this exchange, so the recovery of the other exchanges goes on + String reason; + if (e.getCause() instanceof InvalidClassException) { + reason = "rejected by the deserialization filter, see the deserializationFilter option"; + } else if (e instanceof RuntimeCamelException) { + // the messages of this repository do not include the value + reason = e.getMessage(); + } else { + reason = e.getClass().getName(); + } + if (unreadableExchangeIds.add(exchangeId)) { + LOG.warn("Cannot read the completed exchange {} in the Redis hash {}, so it is not recovered: {}", + exchangeId, persistenceMapName, reason); + } else { + LOG.debug("Cannot read the completed exchange {} in the Redis hash {}, so it is not recovered: {}", + exchangeId, persistenceMapName, reason); + } + return null; + } } public boolean isOptimistic() { @@ -268,7 +341,7 @@ public class RedisAggregationRepository extends ServiceSupport @Override public Exchange get(CamelContext camelContext, String key) { - return unmarshallExchange(camelContext, cache.get(key)); + return unmarshallExchange(camelContext, mapName, key, cache.get(key)); } /** @@ -292,9 +365,38 @@ public class RedisAggregationRepository extends ServiceSupport this.allowSerializedHeaders = allowSerializedHeaders; } + public String getDeserializationFilter() { + return deserializationFilter; + } + + /** + * Sets an {@link java.io.ObjectInputFilter} pattern (same syntax as {@code jdk.serialFilter}) applied when + * deserializing the exchanges read back from Redis. When not set, the JVM-wide {@code jdk.serialFilter} is used if + * present, otherwise a conservative default filter is applied, see + * {@link org.apache.camel.support.DeserializationFilterHelper#DEFAULT_DESERIALIZATION_FILTER}. Widen this pattern + * when the message bodies or headers stored in the repository are instances of your own classes. + */ + public void setDeserializationFilter(String deserializationFilter) { + this.deserializationFilter = deserializationFilter; + } + + public Codec getCodec() { + return codec; + } + + /** + * The Redisson codec of the Redis hashes of the repository, in place of the Java serialization read through the + * deserialization filter. When set, the codec encodes and decodes the exchanges, and + * {@link #setDeserializationFilter(String)} does not apply. For example, a {@code org.redisson.codec.Kryo5Codec} + * reads the exchanges stored by Camel 4.22.2 or older, which used the default codec of the Redisson client. + */ + public void setCodec(Codec codec) { + this.codec = codec; + } + @Override public void remove(CamelContext camelContext, String key, Exchange exchange) { - DefaultExchangeHolder holder = DefaultExchangeHolder.marshal(exchange, true, allowSerializedHeaders); + Object holder = marshallHolder(exchange); if (optimistic) { LOG.trace("Removing an exchange with ID {} for key {} in an optimistic manner.", exchange.getExchangeId(), key); if (!cache.remove(key, holder)) { @@ -318,10 +420,10 @@ public class RedisAggregationRepository extends ServiceSupport RTransaction transaction = redisson.createTransaction(tOpts); try { - RMap<String, DefaultExchangeHolder> tCache = transaction.getMap(mapName); - RMap<String, DefaultExchangeHolder> tPersistentCache = transaction.getMap(persistenceMapName); + RMap<String, Object> tCache = transaction.getMap(mapName, mapCodec); + RMap<String, Object> tPersistentCache = transaction.getMap(persistenceMapName, mapCodec); - DefaultExchangeHolder removedHolder = tCache.remove(key); + Object removedHolder = tCache.remove(key); LOG.trace("Putting an exchange with ID {} for key {} into a recoverable storage in a thread-safe manner.", exchange.getExchangeId(), key); tPersistentCache.put(exchange.getExchangeId(), removedHolder); @@ -351,6 +453,7 @@ public class RedisAggregationRepository extends ServiceSupport LOG.trace("Confirming an exchange with ID {}.", exchangeId); if (useRecovery) { persistedCache.remove(exchangeId); + unreadableExchangeIds.remove(exchangeId); } } @@ -388,12 +491,59 @@ public class RedisAggregationRepository extends ServiceSupport redisson = Redisson.create(config); shutdownRedisson = true; } - cache = redisson.getMap(mapName); + unreadableExchangeIds.clear(); + mapCodec = codec != null ? codec : CODEC; + cache = redisson.getMap(mapName, mapCodec); + if (codec == null) { + warnUnreadableEntries(cache, mapName); + } + if (useRecovery) { - persistedCache = redisson.getMap(persistenceMapName); + persistedCache = redisson.getMap(persistenceMapName, mapCodec); + if (codec == null) { + warnUnreadableEntries(persistedCache, persistenceMapName); + } } } + /** + * Logs a WARN when the hash has entries that are not serialized exchanges, such as the entries stored by an older + * version of this repository, which are not read. Only the first entries of the hash, and the first bytes of their + * values, are checked. + */ + private void warnUnreadableEntries(RMap<String, Object> map, String name) { + try { + // only the first entries are checked, which is enough to spot a hash written by an older version + int checked = 0; + int count = 0; + for (Map.Entry<String, Object> entry : map.entrySet(SCAN_COUNT)) { + if (!isSerializedObject(entry.getValue())) { + count++; + } + if (++checked >= SCAN_COUNT) { + break; + } + } + if (count > 0) { + boolean partial = checked >= SCAN_COUNT && map.size() > checked; + LOG.warn("The Redis hash {} has entries that are not exchanges serialized by this version of" + + " RedisAggregationRepository (count: {}{}). These entries are ignored: entries stored by an" + + " older Camel version are not read. See the camel-redis section of the Camel 4.22 upgrade" + + " guide.", + name, partial ? "at least " : "", count); + } + } catch (Exception e) { + LOG.warn("Cannot check the entries of the Redis hash {}", name, e); + } + } + + /** + * Whether the value starts with the magic number 0xACED of a Java serialization stream. + */ + private static boolean isSerializedObject(Object value) { + return value instanceof byte[] bytes && bytes.length >= 2 && bytes[0] == (byte) 0xAC && bytes[1] == (byte) 0xED; + } + @Override protected void doStop() throws Exception { if (redisson != null && shutdownRedisson) { @@ -410,4 +560,66 @@ public class RedisAggregationRepository extends ServiceSupport } return exchange; } + + /** + * The {@link DefaultExchangeHolder} of the exchange, as stored in the Redis hashes: in Java serialization, or the + * holder itself when a {@link #setCodec(Codec) codec} is set. + */ + protected Object marshallHolder(Exchange exchange) { + DefaultExchangeHolder holder = DefaultExchangeHolder.marshal(exchange, true, allowSerializedHeaders); + if (codec != null) { + return holder; + } + try (ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(holder); + out.flush(); + return bytes.toByteArray(); + } catch (IOException e) { + throw new RuntimeCamelException( + "Cannot serialize the exchange with ID " + exchange.getExchangeId() + " to store it in Redis", e); + } + } + + /** + * Turns a value read from Redis into an exchange. Without a {@link #setCodec(Codec) codec}, the value is + * deserialized through the deserialization filter. + * + * @param camelContext the camel context + * @param name the name of the Redis hash the value was read from + * @param key the field of the value in the hash + * @param value the value, may be null + * @return the exchange, or null when the value is null + */ + protected Exchange unmarshallExchange(CamelContext camelContext, String name, String key, Object value) { + if (value == null) { + return null; + } + if (codec != null) { + if (!(value instanceof DefaultExchangeHolder holder)) { + throw new RuntimeCamelException( + "The value of " + key + " in the Redis hash " + name + " is not an exchange but " + + value.getClass().getName()); + } + return unmarshallExchange(camelContext, holder); + } + if (!isSerializedObject(value)) { + throw new RuntimeCamelException( + "The value of " + key + " in the Redis hash " + name + " is not a serialized exchange"); + } + Object answer; + try (ObjectInputStream in + = new CamelObjectInputStream(new ByteArrayInputStream((byte[]) value), camelContext, deserializationFilter)) { + answer = in.readObject(); + } catch (IOException | ClassNotFoundException e) { + throw new RuntimeCamelException( + "Cannot deserialize the exchange of " + key + " in the Redis hash " + name, e); + } + if (!(answer instanceof DefaultExchangeHolder holder)) { + throw new RuntimeCamelException( + "The value of " + key + " in the Redis hash " + name + " is not a serialized exchange but " + + (answer != null ? answer.getClass().getName() : "null")); + } + return unmarshallExchange(camelContext, holder); + } } diff --git a/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryDeserializationTest.java b/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryDeserializationTest.java new file mode 100644 index 000000000000..46a097e9865c --- /dev/null +++ b/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/RedisAggregationRepositoryDeserializationTest.java @@ -0,0 +1,423 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.redis.processor.aggregate; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InvalidClassException; +import java.io.ObjectOutputStream; +import java.net.URI; +import java.util.AbstractSet; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.camel.Exchange; +import org.apache.camel.RuntimeCamelException; +import org.apache.camel.support.DefaultExchange; +import org.apache.camel.support.DefaultExchangeHolder; +import org.apache.camel.test.junit6.CamelTestSupport; +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.redisson.api.RLock; +import org.redisson.api.RMap; +import org.redisson.api.RedissonClient; +import org.redisson.client.codec.ByteArrayCodec; +import org.redisson.client.codec.Codec; +import org.redisson.client.codec.StringCodec; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +/** + * Checks how the exchanges are stored and read back, with an in-memory map in place of the Redis hashes. + */ +public class RedisAggregationRepositoryDeserializationTest extends CamelTestSupport { + + private final Map<String, Object> active = new HashMap<>(); + private final Map<String, Object> completed = new HashMap<>(); + private final AtomicInteger readEntries = new AtomicInteger(); + private RedissonClient redisson; + private RedisAggregationRepository repository; + + @BeforeEach + void createRepository() { + redisson = mock(RedissonClient.class); + RMap<String, Object> activeMap = inMemoryMap(active, readEntries); + RMap<String, Object> completedMap = inMemoryMap(completed, readEntries); + when(redisson.<String, Object> getMap(eq("orders"), any(Codec.class))).thenReturn(activeMap); + when(redisson.<String, Object> getMap(eq("orders-completed"), any(Codec.class))).thenReturn(completedMap); + when(redisson.getLock(anyString())).thenReturn(mock(RLock.class)); + + repository = new RedisAggregationRepository(); + repository.setRedisson(redisson); + repository.setMapName("orders"); + } + + @AfterEach + void stopRepository() { + repository.stop(); + } + + @Test + void testHashesUseStringFieldsAndByteArrayValues() { + repository.start(); + + ArgumentCaptor<Codec> codec = ArgumentCaptor.forClass(Codec.class); + verify(redisson).getMap(eq("orders"), codec.capture()); + assertSame(StringCodec.INSTANCE.getMapKeyDecoder(), codec.getValue().getMapKeyDecoder()); + assertSame(ByteArrayCodec.INSTANCE.getMapValueDecoder(), codec.getValue().getMapValueDecoder()); + } + + @Test + void testAddAndGet() { + repository.start(); + + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody("Hello"); + exchange.getIn().setHeader("count", 3); + repository.add(context, "key1", exchange); + + // the value is a Java serialization stream + byte[] value = (byte[]) active.get("key1"); + assertEquals((byte) 0xAC, value[0]); + assertEquals((byte) 0xED, value[1]); + + Exchange answer = repository.get(context, "key1"); + assertNotNull(answer); + assertEquals(exchange.getExchangeId(), answer.getExchangeId()); + assertEquals("Hello", answer.getIn().getBody()); + assertEquals(3, answer.getIn().getHeader("count")); + } + + @Test + void testBodyOfClassOutsideTheDefaultFilterIsNotRead() { + repository.start(); + + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(URI.create("http://localhost/orders")); + repository.add(context, "key1", exchange); + + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> repository.get(context, "key1")); + assertInstanceOf(InvalidClassException.class, e.getCause()); + } + + @Test + void testDeserializationFilterAllowsClass() { + repository.setDeserializationFilter("java.net.URI;java.**;org.apache.camel.**;!*"); + repository.start(); + + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(URI.create("http://localhost/orders")); + repository.add(context, "key1", exchange); + + Exchange answer = repository.get(context, "key1"); + assertEquals(URI.create("http://localhost/orders"), answer.getIn().getBody()); + } + + @Test + void testRecoverSkipsTheExchangeThatCannotBeRead() { + repository.start(); + + // three completed exchanges, the middle one with a body of a class outside the default filter + List<String> ids = new ArrayList<>(); + for (Object body : List.of("first", URI.create("http://localhost/orders"), "third")) { + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(body); + completed.put(exchange.getExchangeId(), serialize(DefaultExchangeHolder.marshal(exchange, true, false))); + ids.add(exchange.getExchangeId()); + } + + Logger logger = (Logger) LogManager.getLogger(RedisAggregationRepository.class); + LogEventCaptor captor = new LogEventCaptor(); + logger.addAppender(captor); + List<Object> bodies = new ArrayList<>(); + try { + // as the recovery task of the aggregator, twice + for (int run = 0; run < 2; run++) { + for (String id : ids) { + Exchange recovered = repository.recover(context, id); + if (recovered != null) { + bodies.add(recovered.getIn().getBody()); + } + } + } + } finally { + logger.removeAppender(captor); + } + + assertEquals(List.of("first", "third", "first", "third"), bodies); + List<String> warnings = captor.events.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(event -> event.getMessage().getFormattedMessage()) + .toList(); + // once per exchange, with its id and not its value + assertEquals(1, warnings.size(), warnings.toString()); + assertTrue(warnings.get(0).contains(ids.get(1)), warnings.get(0)); + assertTrue(warnings.get(0).contains("deserialization filter"), warnings.get(0)); + assertFalse(warnings.get(0).contains("localhost"), warnings.get(0)); + } + + @Test + void testValueInAnotherFormatIsNotRead() { + repository.start(); + + // not a Java serialization stream, such as a value stored by an older version + active.put("key1", new byte[] { 1, 0, 'o', 'r', 'g' }); + + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> repository.get(context, "key1")); + assertTrue(e.getMessage().contains("is not a serialized exchange"), e.getMessage()); + } + + @Test + void testValueThatIsNotAnExchangeIsNotUsed() { + repository.start(); + + active.put("key1", serialize("Hello")); + + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> repository.get(context, "key1")); + assertTrue(e.getMessage().contains("java.lang.String"), e.getMessage()); + } + + @Test + void testWarnAtStartForEntriesInAnotherFormat() { + // such as entries stored by an older version, which are not read + active.put("\u0003key1\uFFFD", new byte[] { 1, 0, 'o', 'r', 'g' }); + active.put("\u0003key2\uFFFD", new byte[] { 1, 0, 'o', 'r', 'g' }); + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody("Hello"); + active.put("key3", serialize(DefaultExchangeHolder.marshal(exchange, true, false))); + completed.put("\u0003id1\uFFFD", new byte[] { 1, 0, 'o', 'r', 'g' }); + + Logger logger = (Logger) LogManager.getLogger(RedisAggregationRepository.class); + LogEventCaptor captor = new LogEventCaptor(); + logger.addAppender(captor); + try { + repository.start(); + } finally { + logger.removeAppender(captor); + } + + assertTrue(repository.isStarted()); + List<String> warnings = captor.events.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(event -> event.getMessage().getFormattedMessage()) + .toList(); + assertEquals(2, warnings.size(), warnings.toString()); + assertTrue(warnings.get(0).contains("The Redis hash orders has entries") && warnings.get(0).contains("(count: 2)"), + warnings.get(0)); + assertTrue(warnings.get(0).contains("upgrade guide"), warnings.get(0)); + assertTrue(warnings.get(1).contains("The Redis hash orders-completed has entries") + && warnings.get(1).contains("(count: 1)"), warnings.get(1)); + // the fields and values are not logged + assertTrue(warnings.stream().noneMatch(w -> w.contains("key1") || w.contains("\uFFFD")), warnings.toString()); + } + + @Test + void testWarnAtStartChecksOnlyTheFirstEntries() { + for (int i = 0; i < 250; i++) { + active.put("\u0003key" + i + "\uFFFD", new byte[] { 1, 0, 'o', 'r', 'g' }); + } + + Logger logger = (Logger) LogManager.getLogger(RedisAggregationRepository.class); + LogEventCaptor captor = new LogEventCaptor(); + logger.addAppender(captor); + try { + repository.start(); + } finally { + logger.removeAppender(captor); + } + + assertEquals(100, readEntries.get()); + List<String> warnings = captor.events.stream() + .filter(event -> event.getLevel() == Level.WARN) + .map(event -> event.getMessage().getFormattedMessage()) + .toList(); + assertEquals(1, warnings.size(), warnings.toString()); + assertTrue(warnings.get(0).contains("The Redis hash orders has entries") + && warnings.get(0).contains("(count: at least 100)"), warnings.get(0)); + } + + @Test + void testNoWarnAtStartForExchanges() { + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody("Hello"); + active.put("key1", serialize(DefaultExchangeHolder.marshal(exchange, true, false))); + + Logger logger = (Logger) LogManager.getLogger(RedisAggregationRepository.class); + LogEventCaptor captor = new LogEventCaptor(); + logger.addAppender(captor); + try { + repository.start(); + } finally { + logger.removeAppender(captor); + } + + assertTrue(captor.events.stream().noneMatch(event -> event.getLevel() == Level.WARN)); + } + + @Test + void testCodecIsUsedForTheHashes() { + Codec codec = mock(Codec.class); + repository.setCodec(codec); + repository.start(); + + verify(redisson).getMap("orders", codec); + verify(redisson).getMap("orders-completed", codec); + } + + @Test + void testCodecStoresTheHolderAndTheFilterDoesNotApply() { + repository.setCodec(mock(Codec.class)); + repository.start(); + + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(URI.create("http://localhost/orders")); + exchange.getIn().setHeader("count", 3); + repository.add(context, "key1", exchange); + + // the codec encodes the holder itself + assertInstanceOf(DefaultExchangeHolder.class, active.get("key1")); + + Exchange answer = repository.get(context, "key1"); + assertEquals(exchange.getExchangeId(), answer.getExchangeId()); + assertEquals(URI.create("http://localhost/orders"), answer.getIn().getBody()); + assertEquals(3, answer.getIn().getHeader("count")); + + completed.put(exchange.getExchangeId(), DefaultExchangeHolder.marshal(exchange, true, false)); + assertEquals(URI.create("http://localhost/orders"), + repository.recover(context, exchange.getExchangeId()).getIn().getBody()); + } + + @Test + void testCodecValueThatIsNotAnExchangeIsNotUsed() { + repository.setCodec(mock(Codec.class)); + repository.start(); + + active.put("key1", "Hello"); + + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> repository.get(context, "key1")); + assertTrue(e.getMessage().contains("java.lang.String"), e.getMessage()); + } + + @Test + void testNoWarnAtStartWithCodec() { + // entries of the codec are not Java serialization streams, and are expected + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody("Hello"); + active.put("key1", DefaultExchangeHolder.marshal(exchange, true, false)); + active.put("key2", new byte[] { 1, 0, 'o', 'r', 'g' }); + completed.put("id1", new byte[] { 1, 0, 'o', 'r', 'g' }); + repository.setCodec(mock(Codec.class)); + + Logger logger = (Logger) LogManager.getLogger(RedisAggregationRepository.class); + LogEventCaptor captor = new LogEventCaptor(); + logger.addAppender(captor); + try { + repository.start(); + } finally { + logger.removeAppender(captor); + } + + assertTrue(repository.isStarted()); + assertTrue(captor.events.stream().noneMatch(event -> event.getLevel() == Level.WARN)); + } + + @SuppressWarnings("unchecked") + private static RMap<String, Object> inMemoryMap(Map<String, Object> store, AtomicInteger readEntries) { + RMap<String, Object> map = mock(RMap.class); + when(map.get(any())).thenAnswer(i -> store.get(i.<String> getArgument(0))); + when(map.put(anyString(), any())).thenAnswer(i -> store.put(i.getArgument(0), i.getArgument(1))); + when(map.keySet()).thenAnswer(i -> Set.copyOf(store.keySet())); + when(map.size()).thenAnswer(i -> store.size()); + // counts the entries read, as HSCAN reads them from Redis + when(map.entrySet(anyInt())).thenAnswer(i -> new AbstractSet<Map.Entry<String, Object>>() { + @Override + public Iterator<Map.Entry<String, Object>> iterator() { + Iterator<Map.Entry<String, Object>> it = Set.copyOf(store.entrySet()).iterator(); + return new Iterator<>() { + @Override + public boolean hasNext() { + return it.hasNext(); + } + + @Override + public Map.Entry<String, Object> next() { + readEntries.incrementAndGet(); + return it.next(); + } + }; + } + + @Override + public int size() { + return store.size(); + } + }); + return map; + } + + private static final class LogEventCaptor extends AbstractAppender { + private final List<LogEvent> events = new CopyOnWriteArrayList<>(); + + LogEventCaptor() { + super("RedisAggregationRepositoryLogCaptor", null, null, false, Property.EMPTY_ARRAY); + start(); + } + + @Override + public void append(LogEvent event) { + events.add(event.toImmutable()); + } + } + + private static byte[] serialize(Object value) { + try (ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + ObjectOutputStream out = new ObjectOutputStream(bytes)) { + out.writeObject(value); + out.flush(); + return bytes.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } +} diff --git a/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryCodecIT.java b/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryCodecIT.java new file mode 100644 index 000000000000..ebd3cdb07f31 --- /dev/null +++ b/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryCodecIT.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.camel.component.redis.processor.aggregate.integration; + +import java.net.URI; +import java.util.Set; + +import org.apache.camel.Exchange; +import org.apache.camel.component.redis.processor.aggregate.RedisAggregationRepository; +import org.apache.camel.support.DefaultExchange; +import org.apache.camel.support.DefaultExchangeHolder; +import org.junit.jupiter.api.Test; +import org.redisson.Redisson; +import org.redisson.api.RedissonClient; +import org.redisson.codec.Kryo5Codec; +import org.redisson.config.Config; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * Runs the repository operations with the codec option set to the codec that Camel 4.22.2 and older used by default. + */ +public class RedisAggregationRepositoryCodecIT extends RedisAggregationRepositoryOperationsIT { + + @Override + protected void configure(RedisAggregationRepository repo) { + repo.setCodec(new Kryo5Codec()); + } + + @Override + @Test + public void testBodyOfClassOutsideTheDeserializationFilterIsNotRead() { + // with a codec, the deserialization filter does not apply + RedisAggregationRepository repo = createRepo("deserializationFilterDefault", false); + try { + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(URI.create("http://localhost/orders")); + repo.add(context, "key1", exchange); + + assertEquals(URI.create("http://localhost/orders"), repo.get(context, "key1").getIn().getBody()); + } finally { + repo.stop(); + } + } + + @Test + public void testReadEntriesStoredByOlderVersions() { + // store the entries as Camel 4.22.2 and older did: with the default codec of the Redisson client + Exchange active = createExchange("active"); + active.setProperty(Exchange.AGGREGATED_SIZE, 2); + Exchange completed = createExchange("completed"); + Config config = new Config(); + config.useSingleServer().setAddress(String.format("redis://%s", service.getServiceAddress())); + RedissonClient client = Redisson.create(config); + try { + client.getMap("legacy").put("key1", DefaultExchangeHolder.marshal(active, true, false)); + client.getMap("legacy-completed").put(completed.getExchangeId(), + DefaultExchangeHolder.marshal(completed, true, false)); + } finally { + client.shutdown(); + } + + RedisAggregationRepository repo = createRepo("legacy", false); + try { + assertEquals(Set.of("key1"), repo.getKeys()); + Exchange result = repo.get(context, "key1"); + assertNotNull(result); + assertEquals(active.getExchangeId(), result.getExchangeId()); + assertEquals("active", result.getIn().getBody(String.class)); + assertEquals(2, result.getProperty(Exchange.AGGREGATED_SIZE, Integer.class)); + + assertEquals(Set.of(completed.getExchangeId()), repo.scan(context)); + Exchange recovered = repo.recover(context, completed.getExchangeId()); + assertEquals("completed", recovered.getIn().getBody(String.class)); + + // the aggregation goes on, and completes + Exchange next = createExchange("active and next"); + next.setExchangeId(active.getExchangeId()); + assertNotNull(repo.add(context, "key1", next)); + repo.remove(context, "key1", next); + assertNull(repo.get(context, "key1")); + assertTrue(repo.scan(context).contains(next.getExchangeId())); + + repo.confirm(context, completed.getExchangeId()); + repo.confirm(context, next.getExchangeId()); + assertTrue(repo.scan(context).isEmpty()); + } finally { + repo.stop(); + } + } +} diff --git a/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryOperationsIT.java b/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryOperationsIT.java index 07231bf520d1..96dcc6a746b1 100644 --- a/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryOperationsIT.java +++ b/components/camel-redis/src/test/java/org/apache/camel/component/redis/processor/aggregate/integration/RedisAggregationRepositoryOperationsIT.java @@ -16,14 +16,18 @@ */ package org.apache.camel.component.redis.processor.aggregate.integration; +import java.io.InvalidClassException; +import java.net.URI; import java.util.Set; import org.apache.camel.Exchange; +import org.apache.camel.RuntimeCamelException; import org.apache.camel.component.redis.processor.aggregate.RedisAggregationRepository; import org.apache.camel.support.DefaultExchange; import org.apache.camel.test.infra.redis.services.RedisService; import org.apache.camel.test.infra.redis.services.RedisServiceFactory; import org.apache.camel.test.junit6.CamelTestSupport; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.redisson.Redisson; @@ -32,6 +36,7 @@ import org.redisson.config.Config; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -42,13 +47,33 @@ public class RedisAggregationRepositoryOperationsIT extends CamelTestSupport { @RegisterExtension static RedisService service = RedisServiceFactory.createService(); - private RedisAggregationRepository createRepo(String mapName, boolean optimistic) { + @BeforeEach + void flushRedis() { + // the tests of the subclasses use the same map names + Config config = new Config(); + config.useSingleServer().setAddress(String.format("redis://%s", service.getServiceAddress())); + RedissonClient client = Redisson.create(config); + try { + client.getKeys().flushall(); + } finally { + client.shutdown(); + } + } + + /** + * Configures the repository before it starts. + */ + protected void configure(RedisAggregationRepository repo) { + } + + protected RedisAggregationRepository createRepo(String mapName, boolean optimistic) { RedisAggregationRepository repo = new RedisAggregationRepository(mapName, service.getServiceAddress(), optimistic); + configure(repo); repo.start(); return repo; } - private Exchange createExchange(String body) { + protected Exchange createExchange(String body) { Exchange exchange = new DefaultExchange(context); exchange.getIn().setBody(body); return exchange; @@ -70,6 +95,39 @@ public class RedisAggregationRepositoryOperationsIT extends CamelTestSupport { } } + @Test + public void testBodyOfClassOutsideTheDeserializationFilterIsNotRead() { + RedisAggregationRepository repo = createRepo("deserializationFilterDefault", false); + try { + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(URI.create("http://localhost/orders")); + repo.add(context, "key1", exchange); + + RuntimeCamelException e = assertThrows(RuntimeCamelException.class, () -> repo.get(context, "key1")); + assertInstanceOf(InvalidClassException.class, e.getCause()); + } finally { + repo.stop(); + } + } + + @Test + public void testDeserializationFilterAllowsClass() { + RedisAggregationRepository repo + = new RedisAggregationRepository("deserializationFilterCustom", service.getServiceAddress(), false); + repo.setDeserializationFilter("java.net.URI;java.**;org.apache.camel.**;!*"); + repo.start(); + try { + Exchange exchange = new DefaultExchange(context); + exchange.getIn().setBody(URI.create("http://localhost/orders")); + repo.add(context, "key1", exchange); + + Exchange result = repo.get(context, "key1"); + assertEquals(URI.create("http://localhost/orders"), result.getIn().getBody()); + } finally { + repo.stop(); + } + } + @Test public void testPessimisticReplace() { RedisAggregationRepository repo = createRepo("pessimisticReplace", false); @@ -182,6 +240,7 @@ public class RedisAggregationRepositoryOperationsIT extends CamelTestSupport { RedisAggregationRepository repo = new RedisAggregationRepository("removeNoRecovery", service.getServiceAddress(), true); repo.setUseRecovery(false); + configure(repo); repo.start(); try { Exchange exchange = createExchange("noRecover");
