This is an automated email from the ASF dual-hosted git repository.
chia7712 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new eac5d5716e3 KAFKA-20553 Remove storage dependency on core test output
(#22854)
eac5d5716e3 is described below
commit eac5d5716e3a901d5607d5fbe18d64c098918e7c
Author: YANG-SYUAN CHOU <[email protected]>
AuthorDate: Tue Jul 21 14:06:29 2026 +0800
KAFKA-20553 Remove storage dependency on core test output (#22854)
This PR removes the `storage` module's dependency on the `core` test
output.
The remaining usages of `kafka.utils.TestUtils` are replaced by:
- Using empty `Properties` instances in `RemoteLagCopyTest` and
`RemoteLogManagerTest`, since the required remote log manager properties
are added by `appendRLMConfig`.
- Using `org.apache.kafka.test.TestUtils.tempDirectory()` in
`TieredStorageTestUtils`.
With these usages removed, the following dependency is no longer
required:
```gradle
testImplementation project(':core').sourceSets.test.output
```
The dependency on the `core` main output is retained because the storage
tests still use broker classes from that module.
### Testing
```bash
./gradlew :core:clean :storage:clean :storage:test --rerun-tasks
--no-build-cache
```
All storage tests passed.
Reviewers: Ken Huang <[email protected]>, Chia-Ping Tsai
<[email protected]>
---
build.gradle | 1 -
.../log/remote/storage/RemoteLagCopyTest.java | 39 ++++++++--------------
.../log/remote/storage/RemoteLogManagerTest.java | 34 +++++++------------
.../storage/utils/TieredStorageTestUtils.java | 5 ++-
4 files changed, 27 insertions(+), 52 deletions(-)
diff --git a/build.gradle b/build.gradle
index d982af0a8e9..6ce62a932be 100644
--- a/build.gradle
+++ b/build.gradle
@@ -2487,7 +2487,6 @@ project(':storage') {
testImplementation project(':clients')
testImplementation testFixtures(project(':clients'))
testImplementation project(':core')
- testImplementation project(':core').sourceSets.test.output
testImplementation testFixtures(project(':storage:storage-api'))
testImplementation project(':metadata')
testImplementation project(':test-common:test-common-internal-api')
diff --git
a/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLagCopyTest.java
b/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLagCopyTest.java
index f0941f894d8..e4e24ab96b1 100644
---
a/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLagCopyTest.java
+++
b/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLagCopyTest.java
@@ -44,7 +44,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.Properties;
import java.util.concurrent.atomic.AtomicLong;
import static
org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig.DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX;
@@ -85,7 +84,6 @@ public class RemoteLagCopyTest {
private final UnifiedLog mockLog = mock(UnifiedLog.class);
private final Metrics metrics = new Metrics(time);
- private final Properties brokerConfig =
kafka.utils.TestUtils.createDummyBrokerConfig();
private final TopicIdPartition leaderTopicIdPartition =
new TopicIdPartition(Uuid.randomUuid(), new
TopicPartition("Leader", 0));
private final Optional<Endpoint> endPoint =
@@ -97,11 +95,19 @@ public class RemoteLagCopyTest {
@BeforeEach
void setUp() throws Exception {
- Properties props = brokerConfig;
-
props.setProperty(RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP,
"true");
-
props.setProperty(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
"100");
- appendRLMConfig(props);
- config = new RemoteLogManagerConfig(new
AbstractConfig(RemoteLogManagerConfig.configDef(), props));
+ Map<String, Object> configs = Map.of(
+ RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP, true,
+ RemoteLogManagerConfig.REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
100,
+ RemoteLogManagerConfig.REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP,
NoOpRemoteStorageManager.class.getName(),
+
RemoteLogManagerConfig.REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
NoOpRemoteLogMetadataManager.class.getName(),
+ DEFAULT_REMOTE_STORAGE_MANAGER_CONFIG_PREFIX +
remoteLogStorageTestProp, remoteLogStorageTestVal,
+ DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP,
remoteLogMetadataTopicPartitionsNum,
+ DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataTestProp, remoteLogMetadataTestVal,
+ DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataCommonClientTestProp, remoteLogMetadataCommonClientTestVal,
+ DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataConsumerTestProp, remoteLogMetadataConsumerTestVal,
+ DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataProducerTestProp, remoteLogMetadataProducerTestVal
+ );
+ config = new RemoteLogManagerConfig(new
AbstractConfig(RemoteLogManagerConfig.configDef(), configs));
brokerTopicStats = new
BrokerTopicStats(config.isRemoteStorageSystemEnabled());
remoteLogManager = new RemoteLogManager(config, brokerId, logDir,
clusterId, time,
@@ -804,23 +810,4 @@ public class RemoteLagCopyTest {
List<RemoteLogManager.EnrichedLogSegment> actual =
task.candidateLogSegments(log, 5L, 20L);
assertEquals(expected, actual);
}
-
- private void appendRLMConfig(Properties props) {
-
props.put(RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP, true);
-
props.put(RemoteLogManagerConfig.REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP,
- NoOpRemoteStorageManager.class.getName());
-
props.put(RemoteLogManagerConfig.REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
- NoOpRemoteLogMetadataManager.class.getName());
- props.put(DEFAULT_REMOTE_STORAGE_MANAGER_CONFIG_PREFIX +
remoteLogStorageTestProp, remoteLogStorageTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX
- +
TopicBasedRemoteLogMetadataManagerConfig.REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP,
- remoteLogMetadataTopicPartitionsNum);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataTestProp, remoteLogMetadataTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataCommonClientTestProp,
- remoteLogMetadataCommonClientTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataConsumerTestProp,
- remoteLogMetadataConsumerTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataProducerTestProp,
- remoteLogMetadataProducerTestVal);
- }
}
diff --git
a/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerTest.java
b/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerTest.java
index 983ac727f52..b6d5ae7aefb 100644
---
a/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerTest.java
+++
b/storage/src/test/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerTest.java
@@ -216,7 +216,6 @@ public class RemoteLogManagerTest {
private UnifiedLog mockLog = mock(UnifiedLog.class);
private final MockScheduler scheduler = new MockScheduler(time);
- private final Properties brokerConfig =
kafka.utils.TestUtils.createDummyBrokerConfig();
private final String host = "localhost";
private final int port = 1234;
@@ -228,9 +227,7 @@ public class RemoteLogManagerTest {
checkpoint = new LeaderEpochCheckpointFile(TestUtils.tempFile(), new
LogDirFailureChannel(1));
topicIds.put(leaderTopicIdPartition.topicPartition().topic(),
leaderTopicIdPartition.topicId());
topicIds.put(followerTopicIdPartition.topicPartition().topic(),
followerTopicIdPartition.topicId());
- Properties props = brokerConfig;
-
props.setProperty(RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP,
"true");
-
props.setProperty(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
"100");
+ Properties props = new Properties();
appendRLMConfig(props);
config = configs(props);
brokerTopicStats = new
BrokerTopicStats(config.isRemoteStorageSystemEnabled());
@@ -353,7 +350,6 @@ public class RemoteLogManagerTest {
String key = "key";
String configPrefix = "config.prefix";
Properties props = new Properties();
- props.putAll(brokerConfig);
props.put(RemoteLogManagerConfig.REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX_PROP,
configPrefix);
props.put(configPrefix + key, "world");
props.put("remote.log.metadata.y", "z");
@@ -370,7 +366,6 @@ public class RemoteLogManagerTest {
String key = "key";
String configPrefix = "config.prefix";
Properties props = new Properties();
- props.putAll(brokerConfig);
props.put(RemoteLogManagerConfig.REMOTE_STORAGE_MANAGER_CONFIG_PREFIX_PROP,
configPrefix);
props.put(configPrefix + key, "world");
props.put("remote.storage.manager.y", "z");
@@ -397,7 +392,6 @@ public class RemoteLogManagerTest {
@Test
void testRemoteLogMetadataManagerWithEndpointConfigOverridden() throws
IOException {
Properties props = new Properties();
- props.putAll(brokerConfig);
// override common security.protocol by adding "RLMM prefix" and
"remote log metadata common client prefix"
props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX + "security.protocol", "SSL");
appendRLMConfig(props);
@@ -3789,7 +3783,6 @@ public class RemoteLogManagerTest {
@Test
public void testCopyQuotaManagerConfig() {
Properties defaultProps = new Properties();
- defaultProps.putAll(brokerConfig);
appendRLMConfig(defaultProps);
RemoteLogManagerConfig defaultRlmConfig = configs(defaultProps);
RLMQuotaManagerConfig defaultConfig =
RemoteLogManager.copyQuotaManagerConfig(defaultRlmConfig);
@@ -3798,7 +3791,6 @@ public class RemoteLogManagerTest {
assertEquals(DEFAULT_REMOTE_LOG_MANAGER_COPY_QUOTA_WINDOW_SIZE_SECONDS,
defaultConfig.quotaWindowSizeSeconds());
Properties customProps = new Properties();
- customProps.putAll(brokerConfig);
customProps.put(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_COPY_MAX_BYTES_PER_SECOND_PROP,
100);
customProps.put(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_COPY_QUOTA_WINDOW_NUM_PROP,
31);
customProps.put(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_COPY_QUOTA_WINDOW_SIZE_SECONDS_PROP,
1);
@@ -3814,7 +3806,6 @@ public class RemoteLogManagerTest {
@Test
public void testFetchQuotaManagerConfig() {
Properties defaultProps = new Properties();
- defaultProps.putAll(brokerConfig);
appendRLMConfig(defaultProps);
RemoteLogManagerConfig defaultRlmConfig = configs(defaultProps);
@@ -3824,7 +3815,6 @@ public class RemoteLogManagerTest {
assertEquals(DEFAULT_REMOTE_LOG_MANAGER_FETCH_QUOTA_WINDOW_SIZE_SECONDS,
defaultConfig.quotaWindowSizeSeconds());
Properties customProps = new Properties();
- customProps.putAll(brokerConfig);
customProps.put(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_FETCH_MAX_BYTES_PER_SECOND_PROP,
100);
customProps.put(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_FETCH_QUOTA_WINDOW_NUM_PROP,
31);
customProps.put(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_FETCH_QUOTA_WINDOW_SIZE_SECONDS_PROP,
1);
@@ -4207,7 +4197,7 @@ public class RemoteLogManagerTest {
// 100ms, the test will fail.
remoteLogManager.close();
clearInvocations(remoteLogMetadataManager, remoteStorageManager);
- Properties props = brokerConfig;
+ Properties props = new Properties();
props.setProperty(RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP,
"true");
props.setProperty(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
"30000");
appendRLMConfig(props);
@@ -4281,7 +4271,6 @@ public class RemoteLogManagerTest {
@Test
public void testMonitorableRemoteLogStorageManager() throws IOException {
Properties props = new Properties();
- props.putAll(brokerConfig);
appendRLMConfig(props);
props.put(RemoteLogManagerConfig.REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP,
MonitorableNoOpRemoteStorageManager.class.getName());
props.put(RemoteLogManagerConfig.REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
MonitorableNoOpRemoteLogMetadataManager.class.getName());
@@ -4355,16 +4344,17 @@ public class RemoteLogManagerTest {
}
private void appendRLMConfig(Properties props) {
-
props.put(RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP, true);
-
props.put(RemoteLogManagerConfig.REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP,
NoOpRemoteStorageManager.class.getName());
-
props.put(RemoteLogManagerConfig.REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
NoOpRemoteLogMetadataManager.class.getName());
- props.put(DEFAULT_REMOTE_STORAGE_MANAGER_CONFIG_PREFIX +
remoteLogStorageTestProp, remoteLogStorageTestVal);
+
props.putIfAbsent(RemoteLogManagerConfig.REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP,
true);
+
props.putIfAbsent(RemoteLogManagerConfig.REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP,
NoOpRemoteStorageManager.class.getName());
+
props.putIfAbsent(RemoteLogManagerConfig.REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
NoOpRemoteLogMetadataManager.class.getName());
+ props.putIfAbsent(DEFAULT_REMOTE_STORAGE_MANAGER_CONFIG_PREFIX +
remoteLogStorageTestProp, remoteLogStorageTestVal);
// adding configs with "remote log metadata manager config prefix"
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, remoteLogMetadataTopicPartitionsNum);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataTestProp, remoteLogMetadataTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataCommonClientTestProp, remoteLogMetadataCommonClientTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataConsumerTestProp, remoteLogMetadataConsumerTestVal);
- props.put(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataProducerTestProp, remoteLogMetadataProducerTestVal);
+ props.putIfAbsent(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, remoteLogMetadataTopicPartitionsNum);
+ props.putIfAbsent(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataTestProp, remoteLogMetadataTestVal);
+ props.putIfAbsent(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataCommonClientTestProp, remoteLogMetadataCommonClientTestVal);
+ props.putIfAbsent(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataConsumerTestProp, remoteLogMetadataConsumerTestVal);
+ props.putIfAbsent(DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX +
remoteLogMetadataProducerTestProp, remoteLogMetadataProducerTestVal);
+
props.putIfAbsent(RemoteLogManagerConfig.REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
"100");
}
public static class MonitorableNoOpRemoteStorageManager extends
NoOpRemoteStorageManager implements Monitorable {
diff --git
a/storage/src/test/java/org/apache/kafka/tiered/storage/utils/TieredStorageTestUtils.java
b/storage/src/test/java/org/apache/kafka/tiered/storage/utils/TieredStorageTestUtils.java
index cd0d243da04..f99774496ad 100644
---
a/storage/src/test/java/org/apache/kafka/tiered/storage/utils/TieredStorageTestUtils.java
+++
b/storage/src/test/java/org/apache/kafka/tiered/storage/utils/TieredStorageTestUtils.java
@@ -16,8 +16,6 @@
*/
package org.apache.kafka.tiered.storage.utils;
-import kafka.utils.TestUtils;
-
import org.apache.kafka.clients.admin.TopicDescription;
import org.apache.kafka.common.TopicPartition;
import org.apache.kafka.common.config.TopicConfig;
@@ -29,6 +27,7 @@ import
org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMe
import
org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig;
import org.apache.kafka.server.log.remote.storage.LocalTieredStorage;
import org.apache.kafka.storage.internals.log.CleanerConfig;
+import org.apache.kafka.test.TestUtils;
import org.apache.kafka.tiered.storage.TieredStorageTestContext;
import org.junit.jupiter.api.Assertions;
@@ -158,7 +157,7 @@ public class TieredStorageTestUtils {
// Set a small number of retry interval for retrying
RemoteLogMetadataManager resources initialization to speed up the test
overridingProps.setProperty(metadataConfigPrefix(testClassName,
REMOTE_LOG_METADATA_INITIALIZATION_RETRY_INTERVAL_MS_PROP),
RLMM_INIT_RETRY_INTERVAL_MS.toString());
// Set 2 log dirs to make sure JBOD feature is working correctly
- overridingProps.setProperty(ServerLogConfigs.LOG_DIRS_CONFIG,
TestUtils.tempDir().getAbsolutePath() + "," +
TestUtils.tempDir().getAbsolutePath());
+ overridingProps.setProperty(ServerLogConfigs.LOG_DIRS_CONFIG,
TestUtils.tempDirectory().getAbsolutePath() + "," +
TestUtils.tempDirectory().getAbsolutePath());
// Disable unnecessary log cleaner
overridingProps.setProperty(CleanerConfig.LOG_CLEANER_ENABLE_PROP,
"false");
