Vladsz83 commented on a change in pull request #8159:
URL: https://github.com/apache/ignite/pull/8159#discussion_r474654596
##########
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
##########
@@ -17,30 +17,111 @@
package org.apache.ignite.internal.ducktest.tests;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
/**
*
*/
public class DataGenerationApplication extends IgniteAwareApplication {
+ /** Logger. */
+ protected static final Logger log =
LogManager.getLogger(DataGenerationApplication.class.getName());
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_NANO = 1500 * 1000000L;
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_AMOUNT = 10_000;
+
+
/** {@inheritDoc} */
@Override protected void run(JsonNode jsonNode) {
- log.info("Creating cache...");
+ String cacheName = jsonNode.get("cacheName").asText();
+ boolean infinite = jsonNode.hasNonNull("infinite") &&
jsonNode.get("infinite").asBoolean();
+ int range = jsonNode.get("range").asInt();
+
+ if (infinite) {
+ boolean error = true;
+ AtomicInteger cycle = new AtomicInteger();
+
+ log.info("Generating data in background...");
+
+ try {
+ while (active()) {
+ generateData(cacheName, range, (idx) -> idx + cycle.get(),
true, cycle.get() > 0);
- IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
+ cycle.incrementAndGet();
Review comment:
Smplified.
##########
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
##########
@@ -17,30 +17,111 @@
package org.apache.ignite.internal.ducktest.tests;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
/**
*
*/
public class DataGenerationApplication extends IgniteAwareApplication {
+ /** Logger. */
+ protected static final Logger log =
LogManager.getLogger(DataGenerationApplication.class.getName());
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_NANO = 1500 * 1000000L;
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_AMOUNT = 10_000;
+
+
/** {@inheritDoc} */
@Override protected void run(JsonNode jsonNode) {
- log.info("Creating cache...");
+ String cacheName = jsonNode.get("cacheName").asText();
+ boolean infinite = jsonNode.hasNonNull("infinite") &&
jsonNode.get("infinite").asBoolean();
+ int range = jsonNode.get("range").asInt();
+
+ if (infinite) {
+ boolean error = true;
+ AtomicInteger cycle = new AtomicInteger();
+
+ log.info("Generating data in background...");
+
+ try {
+ while (active()) {
+ generateData(cacheName, range, (idx) -> idx + cycle.get(),
true, cycle.get() > 0);
- IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
+ cycle.incrementAndGet();
+ }
- try (IgniteDataStreamer<Integer, Integer> stmr =
ignite.dataStreamer(cache.getName())) {
- for (int i = 0; i < jsonNode.get("range").asInt(); i++) {
- stmr.addData(i, i);
+ log.info("Background data generation finished.");
- if (i % 10_000 == 0)
- log.info("Streamed " + i + " entries");
+ error = false;
+ }
+ catch (Throwable e) {
+ // The data streamer fails with an error on node stoppage
event before the termination.
+ if (X.hasCause(e, NodeStoppingException.class))
+ error = false;
+ else if (e instanceof Exception)
+ log.error("Failed to generate data in background.", e);
+ }
+ finally {
+ if (error)
+ markBroken();
+ else
+ markFinished(false);
Review comment:
Simplified.
##########
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
##########
@@ -17,30 +17,111 @@
package org.apache.ignite.internal.ducktest.tests;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
/**
*
*/
public class DataGenerationApplication extends IgniteAwareApplication {
+ /** Logger. */
+ protected static final Logger log =
LogManager.getLogger(DataGenerationApplication.class.getName());
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_NANO = 1500 * 1000000L;
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_AMOUNT = 10_000;
+
+
/** {@inheritDoc} */
@Override protected void run(JsonNode jsonNode) {
- log.info("Creating cache...");
+ String cacheName = jsonNode.get("cacheName").asText();
+ boolean infinite = jsonNode.hasNonNull("infinite") &&
jsonNode.get("infinite").asBoolean();
+ int range = jsonNode.get("range").asInt();
+
+ if (infinite) {
+ boolean error = true;
+ AtomicInteger cycle = new AtomicInteger();
+
+ log.info("Generating data in background...");
+
+ try {
+ while (active()) {
+ generateData(cacheName, range, (idx) -> idx + cycle.get(),
true, cycle.get() > 0);
- IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
+ cycle.incrementAndGet();
+ }
- try (IgniteDataStreamer<Integer, Integer> stmr =
ignite.dataStreamer(cache.getName())) {
- for (int i = 0; i < jsonNode.get("range").asInt(); i++) {
- stmr.addData(i, i);
+ log.info("Background data generation finished.");
- if (i % 10_000 == 0)
- log.info("Streamed " + i + " entries");
+ error = false;
+ }
+ catch (Throwable e) {
+ // The data streamer fails with an error on node stoppage
event before the termination.
+ if (X.hasCause(e, NodeStoppingException.class))
+ error = false;
+ else if (e instanceof Exception)
+ log.error("Failed to generate data in background.", e);
+ }
+ finally {
+ if (error)
+ markBroken();
+ else
+ markFinished(false);
}
}
+ else {
+ log.info("Generating data...");
+
+ generateData(cacheName, range, Function.identity(), false, false);
- markSyncExecutionComplete();
+ log.info("Data generation finished. Generated " + range + "
entries.");
+
+ markSyncExecutionComplete();
+ }
+ }
+
+ /** */
+ private void generateData(String cacheName, int range, Function<Integer,
Integer> supplier, boolean markInited,
+ boolean overwrite) {
+ long notifyTime = System.nanoTime();
+ int streamed = 0;
Review comment:
Fixed.
##########
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
##########
@@ -17,30 +17,111 @@
package org.apache.ignite.internal.ducktest.tests;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
/**
*
*/
public class DataGenerationApplication extends IgniteAwareApplication {
+ /** Logger. */
+ protected static final Logger log =
LogManager.getLogger(DataGenerationApplication.class.getName());
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_NANO = 1500 * 1000000L;
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_AMOUNT = 10_000;
+
+
/** {@inheritDoc} */
@Override protected void run(JsonNode jsonNode) {
- log.info("Creating cache...");
+ String cacheName = jsonNode.get("cacheName").asText();
+ boolean infinite = jsonNode.hasNonNull("infinite") &&
jsonNode.get("infinite").asBoolean();
+ int range = jsonNode.get("range").asInt();
+
+ if (infinite) {
+ boolean error = true;
+ AtomicInteger cycle = new AtomicInteger();
+
+ log.info("Generating data in background...");
+
+ try {
+ while (active()) {
+ generateData(cacheName, range, (idx) -> idx + cycle.get(),
true, cycle.get() > 0);
- IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
+ cycle.incrementAndGet();
+ }
- try (IgniteDataStreamer<Integer, Integer> stmr =
ignite.dataStreamer(cache.getName())) {
- for (int i = 0; i < jsonNode.get("range").asInt(); i++) {
- stmr.addData(i, i);
+ log.info("Background data generation finished.");
- if (i % 10_000 == 0)
- log.info("Streamed " + i + " entries");
+ error = false;
+ }
+ catch (Throwable e) {
+ // The data streamer fails with an error on node stoppage
event before the termination.
+ if (X.hasCause(e, NodeStoppingException.class))
+ error = false;
+ else if (e instanceof Exception)
+ log.error("Failed to generate data in background.", e);
+ }
+ finally {
+ if (error)
+ markBroken();
+ else
+ markFinished(false);
}
}
+ else {
+ log.info("Generating data...");
+
+ generateData(cacheName, range, Function.identity(), false, false);
- markSyncExecutionComplete();
+ log.info("Data generation finished. Generated " + range + "
entries.");
+
+ markSyncExecutionComplete();
+ }
+ }
+
+ /** */
+ private void generateData(String cacheName, int range, Function<Integer,
Integer> supplier, boolean markInited,
+ boolean overwrite) {
+ long notifyTime = System.nanoTime();
+ int streamed = 0;
+
+ if (log.isDebugEnabled())
+ log.debug("Creating cache...");
+
+ ignite.getOrCreateCache(cacheName);
+
+ try (IgniteDataStreamer<Integer, Integer> streamer =
ignite.dataStreamer(cacheName)) {
+ streamer.allowOverwrite(overwrite);
+
+ for (int i = 0; i < range && active(); i++) {
+ streamer.addData(i, supplier.apply(i));
Review comment:
Simplified.
##########
File path:
modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/DataGenerationApplication.java
##########
@@ -17,30 +17,111 @@
package org.apache.ignite.internal.ducktest.tests;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Function;
import com.fasterxml.jackson.databind.JsonNode;
-import org.apache.ignite.IgniteCache;
import org.apache.ignite.IgniteDataStreamer;
+import org.apache.ignite.internal.NodeStoppingException;
import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+import org.apache.ignite.internal.util.typedef.X;
+import org.apache.log4j.LogManager;
+import org.apache.log4j.Logger;
/**
*
*/
public class DataGenerationApplication extends IgniteAwareApplication {
+ /** Logger. */
+ protected static final Logger log =
LogManager.getLogger(DataGenerationApplication.class.getName());
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_NANO = 1500 * 1000000L;
+
+ /** */
+ private static final long DATAGEN_NOTIFY_INTERVAL_AMOUNT = 10_000;
+
+
/** {@inheritDoc} */
@Override protected void run(JsonNode jsonNode) {
- log.info("Creating cache...");
+ String cacheName = jsonNode.get("cacheName").asText();
+ boolean infinite = jsonNode.hasNonNull("infinite") &&
jsonNode.get("infinite").asBoolean();
+ int range = jsonNode.get("range").asInt();
+
+ if (infinite) {
+ boolean error = true;
+ AtomicInteger cycle = new AtomicInteger();
+
+ log.info("Generating data in background...");
+
+ try {
+ while (active()) {
+ generateData(cacheName, range, (idx) -> idx + cycle.get(),
true, cycle.get() > 0);
- IgniteCache<Integer, Integer> cache =
ignite.createCache(jsonNode.get("cacheName").asText());
+ cycle.incrementAndGet();
+ }
- try (IgniteDataStreamer<Integer, Integer> stmr =
ignite.dataStreamer(cache.getName())) {
- for (int i = 0; i < jsonNode.get("range").asInt(); i++) {
- stmr.addData(i, i);
+ log.info("Background data generation finished.");
- if (i % 10_000 == 0)
- log.info("Streamed " + i + " entries");
+ error = false;
+ }
+ catch (Throwable e) {
+ // The data streamer fails with an error on node stoppage
event before the termination.
+ if (X.hasCause(e, NodeStoppingException.class))
+ error = false;
+ else if (e instanceof Exception)
+ log.error("Failed to generate data in background.", e);
+ }
+ finally {
+ if (error)
+ markBroken();
+ else
+ markFinished(false);
}
}
+ else {
+ log.info("Generating data...");
+
+ generateData(cacheName, range, Function.identity(), false, false);
- markSyncExecutionComplete();
+ log.info("Data generation finished. Generated " + range + "
entries.");
+
+ markSyncExecutionComplete();
+ }
+ }
+
+ /** */
+ private void generateData(String cacheName, int range, Function<Integer,
Integer> supplier, boolean markInited,
+ boolean overwrite) {
+ long notifyTime = System.nanoTime();
+ int streamed = 0;
+
+ if (log.isDebugEnabled())
+ log.debug("Creating cache...");
+
+ ignite.getOrCreateCache(cacheName);
+
+ try (IgniteDataStreamer<Integer, Integer> streamer =
ignite.dataStreamer(cacheName)) {
+ streamer.allowOverwrite(overwrite);
+
+ for (int i = 0; i < range && active(); i++) {
+ streamer.addData(i, supplier.apply(i));
+
+ if (notifyTime + DATAGEN_NOTIFY_INTERVAL_NANO <
System.nanoTime() ||
+ i - streamed >= DATAGEN_NOTIFY_INTERVAL_AMOUNT) {
+ notifyTime = System.nanoTime();
+
+ if (markInited && !inited())
+ markInitialized();
Review comment:
Simplified.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
