This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 53f4eb42379 Speed up HttpEmitterEventCollectorTest and
EmbeddedMSQRealtimeQueryTest (#18475)
53f4eb42379 is described below
commit 53f4eb423795fccd2035c5f21b7952175ed572f0
Author: Kashif Faraz <[email protected]>
AuthorDate: Wed Sep 3 22:00:43 2025 +0530
Speed up HttpEmitterEventCollectorTest and EmbeddedMSQRealtimeQueryTest
(#18475)
Changes:
- Add config to reduce test run time and avoid unnecessary timeouts
- Reduce runtime of `HttpEmitterEventCollectorTest` from 1 minute to 1
second
- Reuse ingested data to reduce runtime of `EmbeddedMSQRealtimeQueryTest`
from 25s to 9s
---
.../testing/embedded/msq/BaseRealtimeQueryTest.java | 21 ++++++++++++++-------
.../msq/EmbeddedDurableShuffleStorageTest.java | 2 +-
.../embedded/msq/EmbeddedMSQRealtimeQueryTest.java | 11 ++---------
.../msq/EmbeddedMSQRealtimeUnnestQueryTest.java | 9 +++------
.../server/HttpEmitterEventCollectorTest.java | 4 +++-
.../embedded/junit5/EmbeddedClusterTestBase.java | 5 ++++-
6 files changed, 27 insertions(+), 25 deletions(-)
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java
index 1934786afda..ebc3ba723aa 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/BaseRealtimeQueryTest.java
@@ -41,14 +41,13 @@ import
org.apache.druid.testing.embedded.indexing.MoreResources;
import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.joda.time.Period;
-import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.BeforeAll;
import java.io.IOException;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.concurrent.ExecutionException;
/**
* Base test for Kafka related embedded test.
@@ -71,12 +70,20 @@ public class BaseRealtimeQueryTest extends
EmbeddedClusterTestBase
.addResource(kafka);
}
- @BeforeEach
- void setupCreateKafkaTopic()
+ @BeforeAll
+ void setupCreateKafkaTopicAndDatasource()
{
// Create Kafka topic.
topic = EmbeddedClusterApis.createTestDatasourceName();
kafka.createTopicWithPartitions(topic, 2);
+
+ super.refreshDatasourceName();
+ }
+
+ @Override
+ protected void refreshDatasourceName()
+ {
+ // Do not refresh datasource name to allow reuse
}
/**
@@ -123,8 +130,8 @@ public class BaseRealtimeQueryTest extends
EmbeddedClusterTestBase
);
}
- @AfterEach
- void tearDownEach() throws ExecutionException, InterruptedException,
IOException
+ @AfterAll
+ void tearDownAll() throws IOException
{
final Map<String, String> terminateSupervisorResult =
cluster.callApi().onLeaderOverlord(o ->
o.terminateSupervisor(dataSource));
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
index 9813e978709..fd6c819e756 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedDurableShuffleStorageTest.java
@@ -111,7 +111,7 @@ public class EmbeddedDurableShuffleStorageTest extends
EmbeddedClusterTestBase
}
@Override
- protected void beforeEachTest()
+ protected void refreshDatasourceName()
{
// do nothing here, the super version of this method generates a new value
for dataSource field each time, but
// we are setting that in our @BeforeAll where we are just inserting data
once and re-using between runs
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeQueryTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeQueryTest.java
index 502482cce77..e1e83190be3 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeQueryTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeQueryTest.java
@@ -40,7 +40,6 @@ import org.hamcrest.MatcherAssert;
import org.junit.internal.matchers.ThrowableMessageMatcher;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
-import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
@@ -91,11 +90,9 @@ public class EmbeddedMSQRealtimeQueryTest extends
BaseRealtimeQueryTest
coordinator.addProperty("druid.manager.segments.useIncrementalCache",
"always");
- overlord.addProperty("druid.manager.segments.useIncrementalCache",
"always")
- .addProperty("druid.manager.segments.pollDuration", "PT0.1s");
-
broker.addProperty("druid.msq.dart.controller.heapFraction", "0.9")
- .addProperty("druid.query.default.context.maxConcurrentStages", "1");
+ .addProperty("druid.query.default.context.maxConcurrentStages", "1")
+ .addProperty("druid.sql.planner.metadataRefreshPeriod", "PT0.1s");
historical.addProperty("druid.msq.dart.worker.heapFraction", "0.9")
.addProperty("druid.msq.dart.worker.concurrentQueries", "1")
@@ -141,11 +138,7 @@ public class EmbeddedMSQRealtimeQueryTest extends
BaseRealtimeQueryTest
broker.start();
indexer.start();
historical.start();
- }
- @BeforeEach
- void setUpEach()
- {
msqApis = new EmbeddedMSQApis(cluster, overlord);
submitSupervisor();
publishToKafka(TestIndex.getMMappedWikipediaIndex());
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeUnnestQueryTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeUnnestQueryTest.java
index 728e7dc2e1f..2f85d1685e1 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeUnnestQueryTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/EmbeddedMSQRealtimeUnnestQueryTest.java
@@ -33,7 +33,7 @@ import org.apache.druid.testing.embedded.EmbeddedIndexer;
import org.apache.druid.testing.embedded.EmbeddedOverlord;
import org.apache.druid.testing.embedded.EmbeddedRouter;
import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
@@ -58,9 +58,6 @@ public class EmbeddedMSQRealtimeUnnestQueryTest extends
BaseRealtimeQueryTest
coordinator.addProperty("druid.manager.segments.useIncrementalCache",
"always");
- overlord.addProperty("druid.manager.segments.useIncrementalCache",
"always")
- .addProperty("druid.manager.segments.pollDuration", "PT0.1s");
-
broker.addProperty("druid.msq.dart.controller.heapFraction", "0.9")
.addProperty("druid.query.default.context.maxConcurrentStages", "1");
@@ -86,8 +83,8 @@ public class EmbeddedMSQRealtimeUnnestQueryTest extends
BaseRealtimeQueryTest
.addServer(indexer);
}
- @BeforeEach
- void setUpEach()
+ @BeforeAll
+ void setupAll()
{
msqApis = new EmbeddedMSQApis(cluster, overlord);
diff --git
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/HttpEmitterEventCollectorTest.java
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/HttpEmitterEventCollectorTest.java
index 59246d05d3b..d37bf7df44d 100644
---
a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/HttpEmitterEventCollectorTest.java
+++
b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/HttpEmitterEventCollectorTest.java
@@ -42,6 +42,8 @@ public class HttpEmitterEventCollectorTest extends
EmbeddedClusterTestBase
{
private final EmbeddedOverlord overlord = new EmbeddedOverlord();
private final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
+ private final EmbeddedBroker broker = new EmbeddedBroker()
+ .addProperty("druid.sql.planner.metadataRefreshPeriod", "PT0.1s");
private final EmbeddedEventCollector eventCollector = new
EmbeddedEventCollector()
.addProperty("druid.emitter", "latching");
@@ -59,8 +61,8 @@ public class HttpEmitterEventCollectorTest extends
EmbeddedClusterTestBase
.addServer(overlord)
.addServer(eventCollector)
.addServer(coordinator)
+ .addServer(broker)
.addServer(new EmbeddedHistorical())
- .addServer(new EmbeddedBroker())
.addServer(new EmbeddedIndexer());
}
diff --git
a/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java
b/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java
index c00bdcab229..69ed752642e 100644
---
a/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java
+++
b/services/src/test/java/org/apache/druid/testing/embedded/junit5/EmbeddedClusterTestBase.java
@@ -76,8 +76,11 @@ public abstract class EmbeddedClusterTestBase
}
}
+ /**
+ * Assigns a new value to the {@link #dataSource} before each test.
+ */
@BeforeEach
- protected void beforeEachTest()
+ protected void refreshDatasourceName()
{
dataSource = EmbeddedClusterApis.createTestDatasourceName();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
