This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 8c4ab96a2cf Do not emit metrics from integration testing client
(#17867)
8c4ab96a2cf is described below
commit 8c4ab96a2cfceee0c7715893cfd6e087f5b6b241
Author: Kashif Faraz <[email protected]>
AuthorDate: Sat Apr 5 08:38:30 2025 +0530
Do not emit metrics from integration testing client (#17867)
Changes
---------
* Remove unused ZookeeperClient
* Use NoopEmitter on the client side of standard ITs
* Deserialize response of /compaction/status API into AutoCompactionSnapshot
* Do not print complete/incomplete tasks in
ITAutoCompactionLockContentionTest
---
integration-tests-ex/cases/pom.xml | 4 -
.../druid/testsEx/cluster/ZooKeeperClient.java | 85 ----------------------
.../clients/CompactionResourceTestClient.java | 8 +-
.../druid/testing/guice/DruidTestModule.java | 11 ++-
.../apache/druid/testing/utils/ITRetryUtil.java | 2 +
.../duty/ITAutoCompactionLockContentionTest.java | 20 -----
.../coordinator/duty/ITAutoCompactionTest.java | 22 +++---
7 files changed, 23 insertions(+), 129 deletions(-)
diff --git a/integration-tests-ex/cases/pom.xml
b/integration-tests-ex/cases/pom.xml
index d21460b9b40..efeff98676d 100644
--- a/integration-tests-ex/cases/pom.xml
+++ b/integration-tests-ex/cases/pom.xml
@@ -83,10 +83,6 @@
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-framework</artifactId>
- </dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
diff --git
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java
b/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java
deleted file mode 100644
index 32f6cb1fdfc..00000000000
---
a/integration-tests-ex/cases/src/test/java/org/apache/druid/testsEx/cluster/ZooKeeperClient.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.druid.testsEx.cluster;
-
-import org.apache.curator.framework.CuratorFramework;
-import org.apache.druid.curator.CuratorConfig;
-import org.apache.druid.curator.CuratorModule;
-import org.apache.druid.java.util.common.ISE;
-import org.apache.druid.testsEx.config.ResolvedConfig;
-import org.apache.druid.testsEx.config.ResolvedService.ResolvedZk;
-
-import java.util.concurrent.TimeUnit;
-
-/**
- * Test oriented ZooKeeper client.
- * <p>
- * Currently contains just enough functionality to verify that
- * ZK is ready.
- */
-public class ZooKeeperClient
-{
- private final ResolvedConfig clusterConfig;
- private final ResolvedZk config;
- private CuratorFramework curatorFramework;
-
- public ZooKeeperClient(ResolvedConfig config)
- {
- this.clusterConfig = config;
- this.config = config.zk();
- if (this.config == null) {
- throw new ISE("ZooKeeper not configured");
- }
- prepare();
- awaitReady();
- }
-
- private void prepare()
- {
- CuratorConfig curatorConfig = clusterConfig.toCuratorConfig();
- curatorFramework = CuratorModule.createCurator(curatorConfig);
- }
-
- private void awaitReady()
- {
- int timeoutSec = config.startTimeoutSecs();
- if (timeoutSec == 0) {
- timeoutSec = 5;
- }
- try {
- curatorFramework.start();
- curatorFramework.blockUntilConnected(timeoutSec, TimeUnit.SECONDS);
- }
- catch (InterruptedException e) {
- throw new ISE("ZooKeeper timed out waiting for connect");
- }
- }
-
- public CuratorFramework curator()
- {
- return curatorFramework;
- }
-
- public void close()
- {
- curatorFramework.close();
- curatorFramework = null;
- }
-}
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/clients/CompactionResourceTestClient.java
b/integration-tests/src/main/java/org/apache/druid/testing/clients/CompactionResourceTestClient.java
index 9649a2cce70..e05e72b12ff 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/clients/CompactionResourceTestClient.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/clients/CompactionResourceTestClient.java
@@ -31,6 +31,8 @@ import org.apache.druid.java.util.http.client.Request;
import org.apache.druid.java.util.http.client.response.StatusResponseHandler;
import org.apache.druid.java.util.http.client.response.StatusResponseHolder;
import org.apache.druid.server.compaction.CompactionSimulateResult;
+import org.apache.druid.server.compaction.CompactionStatusResponse;
+import org.apache.druid.server.coordinator.AutoCompactionSnapshot;
import org.apache.druid.server.coordinator.ClusterCompactionConfig;
import org.apache.druid.server.coordinator.DataSourceCompactionConfig;
import org.apache.druid.server.coordinator.DruidCompactionConfig;
@@ -291,7 +293,7 @@ public class CompactionResourceTestClient
return jsonMapper.readValue(response.getContent(), new TypeReference<>()
{});
}
- public Map<String, String> getCompactionStatus(String dataSource) throws
Exception
+ public AutoCompactionSnapshot getCompactionStatus(String dataSource) throws
Exception
{
String url = StringUtils.format("%scompaction/status?dataSource=%s",
getCoordinatorURL(), StringUtils.urlEncode(dataSource));
StatusResponseHolder response = httpClient.go(
@@ -306,8 +308,8 @@ public class CompactionResourceTestClient
response.getContent()
);
}
- Map<String, List<Map<String, String>>> latestSnapshots =
jsonMapper.readValue(response.getContent(), new TypeReference<>() {});
- return latestSnapshots.get("latestStatus").get(0);
+ final CompactionStatusResponse latestSnapshots =
jsonMapper.readValue(response.getContent(), new TypeReference<>() {});
+ return latestSnapshots.getLatestStatus().get(0);
}
public CompactionSimulateResult simulateRunOnCoordinator() throws Exception
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java
b/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java
index be80a5ddc58..f71ca38fbf3 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/guice/DruidTestModule.java
@@ -19,8 +19,6 @@
package org.apache.druid.testing.guice;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Supplier;
import com.google.inject.Binder;
import com.google.inject.Module;
import com.google.inject.Provides;
@@ -32,8 +30,7 @@ import org.apache.druid.guice.ManageLifecycle;
import org.apache.druid.guice.annotations.EscalatedClient;
import org.apache.druid.guice.annotations.Self;
import org.apache.druid.java.util.common.lifecycle.Lifecycle;
-import org.apache.druid.java.util.emitter.core.LoggingEmitter;
-import org.apache.druid.java.util.emitter.core.LoggingEmitterConfig;
+import org.apache.druid.java.util.emitter.core.NoopEmitter;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.java.util.http.client.CredentialedHttpClient;
import org.apache.druid.java.util.http.client.HttpClient;
@@ -83,8 +80,10 @@ public class DruidTestModule implements Module
@Provides
@ManageLifecycle
- public ServiceEmitter getServiceEmitter(Supplier<LoggingEmitterConfig>
config, ObjectMapper jsonMapper)
+ public ServiceEmitter getServiceEmitter()
{
- return new ServiceEmitter("", "", new LoggingEmitter(config.get(),
jsonMapper));
+ // Disabling metric emission since no useful metrics are emitted by the
integration testing client
+ // Use a LoggingEmitter here if needed in the future
+ return new ServiceEmitter("", "", new NoopEmitter());
}
}
diff --git
a/integration-tests/src/main/java/org/apache/druid/testing/utils/ITRetryUtil.java
b/integration-tests/src/main/java/org/apache/druid/testing/utils/ITRetryUtil.java
index 02fd960e13e..63dbbea0b42 100644
---
a/integration-tests/src/main/java/org/apache/druid/testing/utils/ITRetryUtil.java
+++
b/integration-tests/src/main/java/org/apache/druid/testing/utils/ITRetryUtil.java
@@ -120,6 +120,8 @@ public class ITRetryUtil
}
}
+ System.out.printf("Retries[%d] exhausted.%n", retryCount);
+
if (lastException != null) {
throw new ISE(
lastException,
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionLockContentionTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionLockContentionTest.java
index 4fbbbdbb92a..11e4ec6cad3 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionLockContentionTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionLockContentionTest.java
@@ -324,32 +324,12 @@ public class ITAutoCompactionLockContentionTest extends
AbstractKafkaIndexingSer
*/
private int getNumberOfCompletedCompactionTasks()
{
- List<TaskResponseObject> incompleteTasks = indexer
- .getUncompletedTasksForDataSource(fullDatasourceName);
List<TaskResponseObject> completeTasks = indexer
.getCompleteTasksForDataSource(fullDatasourceName);
- printTasks(incompleteTasks, "Incomplete");
- printTasks(completeTasks, "Complete");
-
return (int) completeTasks.stream().filter(this::isCompactionTask).count();
}
- private void printTasks(List<TaskResponseObject> tasks, String taskState)
- {
- StringBuilder sb = new StringBuilder();
- tasks.forEach(
- task -> sb.append("{")
- .append(task.getType())
- .append(", ")
- .append(task.getStatus())
- .append(", ")
- .append(task.getCreatedTime())
- .append("}, ")
- );
- LOG.info("%s Tasks: %s", taskState, sb);
- }
-
/**
* Retries until the total row count is as expected.
*/
diff --git
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
index 48ef5994177..cdafb7fa48e 100644
---
a/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
+++
b/integration-tests/src/test/java/org/apache/druid/tests/coordinator/duty/ITAutoCompactionTest.java
@@ -2132,17 +2132,17 @@ public class ITAutoCompactionTest extends
AbstractIndexerTest
long intervalCountSkipped
) throws Exception
{
- Map<String, String> actualStatus =
compactionResource.getCompactionStatus(fullDatasourceName);
+ AutoCompactionSnapshot actualStatus =
compactionResource.getCompactionStatus(fullDatasourceName);
Assert.assertNotNull(actualStatus);
- Assert.assertEquals(actualStatus.get("scheduleStatus"),
scheduleStatus.toString());
-
MatcherAssert.assertThat(Long.parseLong(actualStatus.get("bytesAwaitingCompaction")),
bytesAwaitingCompactionMatcher);
-
MatcherAssert.assertThat(Long.parseLong(actualStatus.get("bytesCompacted")),
bytesCompactedMatcher);
- MatcherAssert.assertThat(Long.parseLong(actualStatus.get("bytesSkipped")),
bytesSkippedMatcher);
-
Assert.assertEquals(Long.parseLong(actualStatus.get("segmentCountAwaitingCompaction")),
segmentCountAwaitingCompaction);
-
Assert.assertEquals(Long.parseLong(actualStatus.get("segmentCountCompacted")),
segmentCountCompacted);
-
Assert.assertEquals(Long.parseLong(actualStatus.get("segmentCountSkipped")),
segmentCountSkipped);
-
Assert.assertEquals(Long.parseLong(actualStatus.get("intervalCountAwaitingCompaction")),
intervalCountAwaitingCompaction);
-
Assert.assertEquals(Long.parseLong(actualStatus.get("intervalCountCompacted")),
intervalCountCompacted);
-
Assert.assertEquals(Long.parseLong(actualStatus.get("intervalCountSkipped")),
intervalCountSkipped);
+ Assert.assertEquals(actualStatus.getScheduleStatus(), scheduleStatus);
+ MatcherAssert.assertThat(actualStatus.getBytesAwaitingCompaction(),
bytesAwaitingCompactionMatcher);
+ MatcherAssert.assertThat(actualStatus.getBytesCompacted(),
bytesCompactedMatcher);
+ MatcherAssert.assertThat(actualStatus.getBytesSkipped(),
bytesSkippedMatcher);
+ Assert.assertEquals(actualStatus.getSegmentCountAwaitingCompaction(),
segmentCountAwaitingCompaction);
+ Assert.assertEquals(actualStatus.getSegmentCountCompacted(),
segmentCountCompacted);
+ Assert.assertEquals(actualStatus.getSegmentCountSkipped(),
segmentCountSkipped);
+ Assert.assertEquals(actualStatus.getIntervalCountAwaitingCompaction(),
intervalCountAwaitingCompaction);
+ Assert.assertEquals(actualStatus.getIntervalCountCompacted(),
intervalCountCompacted);
+ Assert.assertEquals(actualStatus.getIntervalCountSkipped(),
intervalCountSkipped);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
