This is an automated email from the ASF dual-hosted git repository. orpiske pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/camel-examples.git
commit 9674ff8e4686bf862cd483f08a51ae2e44899012 Author: Otavio Rodolfo Piske <[email protected]> AuthorDate: Tue Aug 9 17:42:26 2022 +0200 (chores) camel-resume-api-examples: support checking results for the offset example --- examples/resume-api/resume-api-common/pom.xml | 4 +++ .../kafka/file/LargeFileRouteBuilder.java | 8 +++++ .../src/main/docker/Dockerfile | 2 +- .../resume-api-file-offset/src/main/scripts/run.sh | 37 +++++++++++++++++++--- 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/examples/resume-api/resume-api-common/pom.xml b/examples/resume-api/resume-api-common/pom.xml index deabd257..c7f6f5cf 100644 --- a/examples/resume-api/resume-api-common/pom.xml +++ b/examples/resume-api/resume-api-common/pom.xml @@ -35,6 +35,10 @@ <groupId>org.apache.camel</groupId> <artifactId>camel-kafka</artifactId> </dependency> + <dependency> + <groupId>org.apache.camel</groupId> + <artifactId>camel-direct</artifactId> + </dependency> </dependencies> </project> diff --git a/examples/resume-api/resume-api-common/src/main/java/org/apache/camel/example/resume/strategies/kafka/file/LargeFileRouteBuilder.java b/examples/resume-api/resume-api-common/src/main/java/org/apache/camel/example/resume/strategies/kafka/file/LargeFileRouteBuilder.java index 0c1e6237..33451452 100644 --- a/examples/resume-api/resume-api-common/src/main/java/org/apache/camel/example/resume/strategies/kafka/file/LargeFileRouteBuilder.java +++ b/examples/resume-api/resume-api-common/src/main/java/org/apache/camel/example/resume/strategies/kafka/file/LargeFileRouteBuilder.java @@ -21,6 +21,7 @@ import java.io.File; import java.util.concurrent.CountDownLatch; import org.apache.camel.Exchange; +import org.apache.camel.ProducerTemplate; import org.apache.camel.builder.RouteBuilder; import org.apache.camel.component.file.FileConstants; import org.apache.camel.processor.resume.kafka.KafkaResumeStrategy; @@ -37,6 +38,7 @@ import org.slf4j.LoggerFactory; public class LargeFileRouteBuilder extends RouteBuilder { private static final Logger LOG = LoggerFactory.getLogger(LargeFileRouteBuilder.class); + private ProducerTemplate producerTemplate; private KafkaResumeStrategy testResumeStrategy; private final ResumeCache<File> cache; @@ -70,6 +72,7 @@ public class LargeFileRouteBuilder extends RouteBuilder { exchange.getMessage().setHeader(Exchange.OFFSET, Resumables.of(file, lastOffset)); + producerTemplate.sendBody("direct:summary", String.valueOf(lastOffset)); LOG.info("Read data: {} / offset key: {} / offset value: {}", body, filePath, lastOffset); if (latch.getCount() == 1) { exchange.setRouteStop(true); @@ -82,6 +85,8 @@ public class LargeFileRouteBuilder extends RouteBuilder { * Let's configure the Camel routing rules using Java code... */ public void configure() { + producerTemplate = getContext().createProducerTemplate(); + getCamelContext().getRegistry().bind(ResumeStrategy.DEFAULT_NAME, testResumeStrategy); getCamelContext().getRegistry().bind(ResumeCache.DEFAULT_NAME, cache); @@ -96,6 +101,9 @@ public class LargeFileRouteBuilder extends RouteBuilder { .intermittent(true) .process(this::process); + from("direct:summary") + .to("file:{{output.dir}}?fileName=summary.txt&fileExist=Append&appendChars=\n"); + } } diff --git a/examples/resume-api/resume-api-file-offset/src/main/docker/Dockerfile b/examples/resume-api/resume-api-file-offset/src/main/docker/Dockerfile index 50067719..2a32e74a 100644 --- a/examples/resume-api/resume-api-file-offset/src/main/docker/Dockerfile +++ b/examples/resume-api/resume-api-file-offset/src/main/docker/Dockerfile @@ -15,7 +15,7 @@ FROM fedora:35 as resume-api-file-offset LABEL maintainer="[email protected]" - +ENV OUTPUT_DIR /data/output ENV DATA_DIR /data/source ENV DATA_FILE data.txt ENV DEPLOYMENT_DIR /deployments diff --git a/examples/resume-api/resume-api-file-offset/src/main/scripts/run.sh b/examples/resume-api/resume-api-file-offset/src/main/scripts/run.sh index 2158150f..be547653 100644 --- a/examples/resume-api/resume-api-file-offset/src/main/scripts/run.sh +++ b/examples/resume-api/resume-api-file-offset/src/main/scripts/run.sh @@ -1,3 +1,4 @@ +#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with @@ -14,6 +15,37 @@ # See the License for the specific language governing permissions and # limitations under the License. # +function checkResults() { +# expectedItems=$((ITERATIONS * BATCH_SIZE)) +# processedRecords=$(cat ${OUTPUT_DIR}/summary.txt | wc -l) +# repeated=$(cat ${OUTPUT_DIR}/summary.txt | sort | uniq --count --repeated | wc -l) + + expectedOffset=0 + dataSize=11 + errors=0 + echo "###**************************************************************************###" + for line in $(cat ${OUTPUT_DIR}/summary.txt) ; do + expectedOffset=$((dataSize + expectedOffset)) + + offsetValue=$(echo $line | sed 's/^[ ]*//') + + if [[ "${expectedOffset}" != "${offsetValue}" ]] ; then + errors=$(( errors++ )) + echo "Results: offset value with error = ${offsetValue} | expectedOffset = ${expectedOffset}." + echo "Error count: ${errors}" + fi + done + + echo "Results: number of items with errors: ${errors}" + echo "###**************************************************************************###" + echo "Resume simulation completed" + echo "###**************************************************************************###" + +} + +trap checkResults exit SIGINT SIGABRT SIGHUP + + ITERATIONS=${1:-5} BATCH_SIZE=${2:-50} @@ -29,7 +61,7 @@ for i in $(seq 0 ${ITERATIONS}) ; do echo "Running the iteration ${i} of ${ITERATIONS} with a batch of ${BATCH_SIZE} offsets" echo "********************************************************************************" java -Dinput.dir=${DATA_DIR} \ - -Doutput.dir=/tmp/out \ + -Doutput.dir=${OUTPUT_DIR} \ -Dinput.file=${DATA_FILE} \ -Dresume.type=kafka \ -Dresume.type.kafka.topic=file-offsets \ @@ -42,7 +74,4 @@ for i in $(seq 0 ${ITERATIONS}) ; do sleep 2s done -echo "###**************************************************************************###" -echo "Resume simulation completed" -echo "###**************************************************************************###" exit 0
