Vladsz83 commented on a change in pull request #8159:
URL: https://github.com/apache/ignite/pull/8159#discussion_r474615025
##########
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
##########
@@ -17,30 +17,111 @@
package org.apache.ignite.internal.ducktest.tests;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
/**
*
*/
public class DataGenerationApplication extends IgniteAwareApplication {
+ /** Logger. */
+ protected static final Logger log =
LogManager.getLogger(DataGenerationApplication.class.getName());
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_NANO = 1500 * 1000000L;
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_AMOUNT = 10_000;
+
+
/** {@inheritDoc} */
@Override protected void run(JsonNode jsonNode) {
- log.info("Creating cache...");
+ String cacheName = jsonNode.get("cacheName").asText();
+ boolean infinite = jsonNode.hasNonNull("infinite") &&
jsonNode.get("infinite").asBoolean();
+ int range = jsonNode.get("range").asInt();
+
+ if (infinite) {
+ boolean error = true;
+ AtomicInteger cycle = new AtomicInteger();
+
+ log.info("Generating data in background...");
+
+ try {
+ while (active()) {
+ generateData(cacheName, range, (idx) -> idx + cycle.get(),
true, cycle.get() > 0);
- IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
+ cycle.incrementAndGet();
+ }
- try (IgniteDataStreamer<Integer, Integer> stmr =
ignite.dataStreamer(cache.getName())) {
- for (int i = 0; i < jsonNode.get("range").asInt(); i++) {
- stmr.addData(i, i);
+ log.info("Background data generation finished.");
- if (i % 10_000 == 0)
- log.info("Streamed " + i + " entries");
+ error = false;
+ }
+ catch (Throwable e) {
+ // The data streamer fails with an error on node stoppage
event before the termination.
+ if (X.hasCause(e, NodeStoppingException.class))
+ error = false;
+ else if (e instanceof Exception)
+ log.error("Failed to generate data in background.", e);
+ }
+ finally {
+ if (error)
+ markBroken();
+ else
+ markFinished(false);
Review comment:
That has reasons. Some time was spent to make this code choise. Catching
no exception causes app marked broken what is not actually broken. The
datastreamer seems to have no way to stop correctly on node stoppage. I tried
to cancel it on node stop event, on shutdown-hook. Another exception arises.
Such behaviour is interesting but I decided not to research it within trivial
and raw loading test.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
