This is an automated email from the ASF dual-hosted git repository.
kfaraz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new b7752a909c Enable round-robin segment assignment and batch segment
allocation by default (#13942)
b7752a909c is described below
commit b7752a909c7ac6e51f6b1571f3a07471beccc3c9
Author: Kashif Faraz <[email protected]>
AuthorDate: Wed Mar 22 08:20:01 2023 +0530
Enable round-robin segment assignment and batch segment allocation by
default (#13942)
Changes:
- Set `useRoundRobinSegmentAssignment` in coordinator dynamic config to
`true` by default.
- Set `batchSegmentAllocation` in `TaskLockConfig` (used in Overlord
runtime properties) to `true` by default.
---
.../indexing/overlord/config/TaskLockConfig.java | 2 +-
.../coordinator/CoordinatorDynamicConfig.java | 2 +-
.../coordinator/DruidCoordinatorRuntimeParams.java | 18 +++++++++++++++---
.../coordinator/BalanceSegmentsProfiler.java | 4 +++-
.../CoordinatorRuntimeParamsTestHelpers.java | 7 -------
.../server/coordinator/rules/LoadRuleTest.java | 22 ++++++++++++++++------
6 files changed, 36 insertions(+), 19 deletions(-)
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/TaskLockConfig.java
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/TaskLockConfig.java
index c860e4e1d1..e750da6c13 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/TaskLockConfig.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/overlord/config/TaskLockConfig.java
@@ -31,7 +31,7 @@ public class TaskLockConfig
private boolean forceTimeChunkLock = true;
@JsonProperty
- private boolean batchSegmentAllocation = false;
+ private boolean batchSegmentAllocation = true;
@JsonProperty
private long batchAllocationWaitTime = 500L;
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
index b9f0d490a3..e8d66cdfb3 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java
@@ -530,7 +530,7 @@ public class CoordinatorDynamicConfig
private static final boolean DEFAULT_PAUSE_COORDINATION = false;
private static final boolean DEFAULT_REPLICATE_AFTER_LOAD_TIMEOUT = false;
private static final int DEFAULT_MAX_NON_PRIMARY_REPLICANTS_TO_LOAD =
Integer.MAX_VALUE;
- private static final boolean DEFAULT_USE_ROUND_ROBIN_ASSIGNMENT = false;
+ private static final boolean DEFAULT_USE_ROUND_ROBIN_ASSIGNMENT = true;
private Long leadingTimeMillisBeforeCanMarkAsUnusedOvershadowedSegments;
private Long mergeBytesLimit;
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
index f195a5ebad..e3703144eb 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinatorRuntimeParams.java
@@ -81,7 +81,7 @@ public class DruidCoordinatorRuntimeParams
@Nullable DataSourcesSnapshot dataSourcesSnapshot,
Map<String, LoadQueuePeon> loadManagementPeons,
ReplicationThrottler replicationManager,
- RoundRobinServerSelector roundRobinServerSelector,
+ @Nullable RoundRobinServerSelector roundRobinServerSelector,
ServiceEmitter emitter,
CoordinatorDynamicConfig coordinatorDynamicConfig,
CoordinatorCompactionConfig coordinatorCompactionConfig,
@@ -296,7 +296,7 @@ public class DruidCoordinatorRuntimeParams
@Nullable DataSourcesSnapshot dataSourcesSnapshot,
Map<String, LoadQueuePeon> loadManagementPeons,
ReplicationThrottler replicationManager,
- RoundRobinServerSelector roundRobinServerSelector,
+ @Nullable RoundRobinServerSelector roundRobinServerSelector,
ServiceEmitter emitter,
CoordinatorDynamicConfig coordinatorDynamicConfig,
CoordinatorCompactionConfig coordinatorCompactionConfig,
@@ -334,7 +334,7 @@ public class DruidCoordinatorRuntimeParams
dataSourcesSnapshot,
loadManagementPeons,
replicationManager,
- roundRobinServerSelector,
+ getOrCreateRoundRobinServerSelector(),
emitter,
coordinatorDynamicConfig,
coordinatorCompactionConfig,
@@ -344,6 +344,18 @@ public class DruidCoordinatorRuntimeParams
);
}
+ private RoundRobinServerSelector getOrCreateRoundRobinServerSelector()
+ {
+ if (druidCluster == null || coordinatorDynamicConfig == null
+ || !coordinatorDynamicConfig.isUseRoundRobinSegmentAssignment()) {
+ return null;
+ } else if (roundRobinServerSelector == null) {
+ return new RoundRobinServerSelector(druidCluster);
+ } else {
+ return roundRobinServerSelector;
+ }
+ }
+
public Builder withStartTimeNanos(long startTimeNanos)
{
this.startTimeNanos = startTimeNanos;
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/BalanceSegmentsProfiler.java
b/server/src/test/java/org/apache/druid/server/coordinator/BalanceSegmentsProfiler.java
index 87dca60a29..06c43e9a68 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/BalanceSegmentsProfiler.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/BalanceSegmentsProfiler.java
@@ -135,7 +135,9 @@ public class BalanceSegmentsProfiler
.addTier("normal", serverHolderList.toArray(new ServerHolder[0]))
.build();
DruidCoordinatorRuntimeParams params = CoordinatorRuntimeParamsTestHelpers
- .newBuilder(druidCluster)
+ .newBuilder()
+ .withDruidCluster(druidCluster)
+ .withSegmentReplicantLookup(SegmentReplicantLookup.make(druidCluster,
false))
.withLoadManagementPeons(peonMap)
.withUsedSegmentsInTest(segments)
.withDynamicConfigs(
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/CoordinatorRuntimeParamsTestHelpers.java
b/server/src/test/java/org/apache/druid/server/coordinator/CoordinatorRuntimeParamsTestHelpers.java
index 1dec00d118..6b319c02b5 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/CoordinatorRuntimeParamsTestHelpers.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/CoordinatorRuntimeParamsTestHelpers.java
@@ -28,13 +28,6 @@ public class CoordinatorRuntimeParamsTestHelpers
.withStartTimeNanos(System.nanoTime());
}
- public static DruidCoordinatorRuntimeParams.Builder newBuilder(DruidCluster
druidCluster)
- {
- return newBuilder()
- .withDruidCluster(druidCluster)
- .withSegmentReplicantLookup(SegmentReplicantLookup.make(druidCluster,
false));
- }
-
private CoordinatorRuntimeParamsTestHelpers()
{
}
diff --git
a/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
b/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
index db62444c6e..642823d186 100644
---
a/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
+++
b/server/src/test/java/org/apache/druid/server/coordinator/rules/LoadRuleTest.java
@@ -574,9 +574,11 @@ public class LoadRuleTest
mockPeon.loadSegment(EasyMock.anyObject(), EasyMock.anyObject());
EasyMock.expectLastCall().atLeastOnce();
-
EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(),
EasyMock.anyObject()))
- .andDelegateTo(balancerStrategy)
- .times(1);
+ if (!useRoundRobinAssignment) {
+
EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(),
EasyMock.anyObject()))
+ .andDelegateTo(balancerStrategy)
+ .times(1);
+ }
EasyMock.replay(throttler, mockPeon, mockBalancerStrategy);
@@ -596,6 +598,10 @@ public class LoadRuleTest
final DataSegment segment = createDataSegment("foo");
+ final CoordinatorDynamicConfig dynamicConfig =
+ CoordinatorDynamicConfig.builder()
+
.withUseRoundRobinSegmentAssignment(useRoundRobinAssignment)
+ .build();
CoordinatorStats stats = rule.run(
null,
CoordinatorRuntimeParamsTestHelpers
@@ -604,6 +610,7 @@ public class LoadRuleTest
.withSegmentReplicantLookup(SegmentReplicantLookup.make(new
DruidCluster(), false))
.withReplicationManager(throttler)
.withBalancerStrategy(mockBalancerStrategy)
+ .withDynamicConfigs(dynamicConfig)
.withUsedSegmentsInTest(segment)
.build(),
segment
@@ -654,9 +661,11 @@ public class LoadRuleTest
public void testMaxLoadingQueueSize()
{
final int maxSegmentsInLoadQueue = 2;
-
EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(),
EasyMock.anyObject()))
- .andDelegateTo(balancerStrategy)
- .times(2);
+ if (!useRoundRobinAssignment) {
+
EasyMock.expect(mockBalancerStrategy.findNewSegmentHomeReplicator(EasyMock.anyObject(),
EasyMock.anyObject()))
+ .andDelegateTo(balancerStrategy)
+ .times(2);
+ }
EasyMock.replay(throttler, mockBalancerStrategy);
@@ -692,6 +701,7 @@ public class LoadRuleTest
.withDynamicConfigs(
CoordinatorDynamicConfig.builder()
.withMaxSegmentsInNodeLoadingQueue(maxSegmentsInLoadQueue)
+
.withUseRoundRobinSegmentAssignment(useRoundRobinAssignment)
.build()
).build();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
