This is an automated email from the ASF dual-hosted git repository. suneet pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push: new fe0511b Coordinator Dynamic Config changes to ease upgrading with new config value (#10724) fe0511b is described below commit fe0511b16a2d34446d11e3fe4e71c248481c0fa8 Author: Lucas Capistrant <capistr...@users.noreply.github.com> AuthorDate: Sun Jan 10 22:05:39 2021 -0600 Coordinator Dynamic Config changes to ease upgrading with new config value (#10724) * Coordinator Dynamic Config changes to ease upgrading with new config value * change a log to debug level following review * changes based on review feedback * fix checkstyle --- .../coordinator/CoordinatorDynamicConfig.java | 16 +++++++-- .../server/http/CoordinatorDynamicConfigTest.java | 41 +++++++++++++++++++--- 2 files changed, 50 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java index 4415f6a..320961d 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/CoordinatorDynamicConfig.java @@ -25,6 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableSet; import org.apache.druid.common.config.JacksonConfigManager; import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.server.coordinator.duty.KillUnusedSegments; import javax.annotation.Nonnull; @@ -87,6 +88,8 @@ public class CoordinatorDynamicConfig private final int maxSegmentsInNodeLoadingQueue; private final boolean pauseCoordination; + private static final Logger log = new Logger(CoordinatorDynamicConfig.class); + @JsonCreator public CoordinatorDynamicConfig( // Keeping the legacy 'millisToWaitBeforeDeleting' property name for backward compatibility. When the project is @@ -96,7 +99,7 @@ public class CoordinatorDynamicConfig @JsonProperty("mergeBytesLimit") long mergeBytesLimit, @JsonProperty("mergeSegmentsLimit") int mergeSegmentsLimit, @JsonProperty("maxSegmentsToMove") int maxSegmentsToMove, - @JsonProperty("percentOfSegmentsToConsiderPerMove") double percentOfSegmentsToConsiderPerMove, + @JsonProperty("percentOfSegmentsToConsiderPerMove") @Nullable Double percentOfSegmentsToConsiderPerMove, @JsonProperty("replicantLifetime") int replicantLifetime, @JsonProperty("replicationThrottleLimit") int replicationThrottleLimit, @JsonProperty("balancerComputeThreads") int balancerComputeThreads, @@ -126,6 +129,15 @@ public class CoordinatorDynamicConfig this.mergeSegmentsLimit = mergeSegmentsLimit; this.maxSegmentsToMove = maxSegmentsToMove; + if (percentOfSegmentsToConsiderPerMove == null) { + log.debug("percentOfSegmentsToConsiderPerMove was null! This is likely because your metastore does not " + + "reflect this configuration being added to Druid in a recent release. Druid is defaulting the value " + + "to the Druid default of %f. It is recommended that you re-submit your dynamic config with your " + + "desired value for percentOfSegmentsToConsideredPerMove", + Builder.DEFAULT_PERCENT_OF_SEGMENTS_TO_CONSIDER_PER_MOVE + ); + percentOfSegmentsToConsiderPerMove = Builder.DEFAULT_PERCENT_OF_SEGMENTS_TO_CONSIDER_PER_MOVE; + } Preconditions.checkArgument( percentOfSegmentsToConsiderPerMove > 0 && percentOfSegmentsToConsiderPerMove <= 100, "percentOfSegmentsToConsiderPerMove should be between 1 and 100!" @@ -428,7 +440,7 @@ public class CoordinatorDynamicConfig private static final long DEFAULT_MERGE_BYTES_LIMIT = 524_288_000L; private static final int DEFAULT_MERGE_SEGMENTS_LIMIT = 100; private static final int DEFAULT_MAX_SEGMENTS_TO_MOVE = 5; - private static final int DEFAULT_PERCENT_OF_SEGMENTS_TO_CONSIDER_PER_MOVE = 100; + private static final double DEFAULT_PERCENT_OF_SEGMENTS_TO_CONSIDER_PER_MOVE = 100; private static final int DEFAULT_REPLICANT_LIFETIME = 15; private static final int DEFAULT_REPLICATION_THROTTLE_LIMIT = 10; private static final int DEFAULT_BALANCER_COMPUTE_THREADS = 1; diff --git a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java index 15c49a2..fe350d0 100644 --- a/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java +++ b/server/src/test/java/org/apache/druid/server/http/CoordinatorDynamicConfigTest.java @@ -90,7 +90,6 @@ public class CoordinatorDynamicConfigTest + " \"mergeBytesLimit\": 1,\n" + " \"mergeSegmentsLimit\" : 1,\n" + " \"maxSegmentsToMove\": 1,\n" - + " \"percentOfSegmentsToConsiderPerMove\": 1,\n" + " \"replicantLifetime\": 1,\n" + " \"replicationThrottleLimit\": 1,\n" + " \"balancerComputeThreads\": 2, \n" @@ -110,13 +109,13 @@ public class CoordinatorDynamicConfigTest ); ImmutableSet<String> decommissioning = ImmutableSet.of(); ImmutableSet<String> whitelist = ImmutableSet.of("test1", "test2"); - assertConfig(actual, 1, 1, 1, 1, 1, 1, 1, 2, true, whitelist, false, 1, decommissioning, 0, false); + assertConfig(actual, 1, 1, 1, 1, 100, 1, 1, 2, true, whitelist, false, 1, decommissioning, 0, false); actual = CoordinatorDynamicConfig.builder().withDecommissioningNodes(ImmutableSet.of("host1")).build(actual); - assertConfig(actual, 1, 1, 1, 1, 1, 1, 1, 2, true, whitelist, false, 1, ImmutableSet.of("host1"), 0, false); + assertConfig(actual, 1, 1, 1, 1, 100, 1, 1, 2, true, whitelist, false, 1, ImmutableSet.of("host1"), 0, false); actual = CoordinatorDynamicConfig.builder().withDecommissioningMaxPercentOfMaxSegmentsToMove(5).build(actual); - assertConfig(actual, 1, 1, 1, 1, 1, 1, 1, 2, true, whitelist, false, 1, ImmutableSet.of("host1"), 5, false); + assertConfig(actual, 1, 1, 1, 1, 100, 1, 1, 2, true, whitelist, false, 1, ImmutableSet.of("host1"), 5, false); } @Test @@ -166,7 +165,7 @@ public class CoordinatorDynamicConfigTest } @Test - public void testSerdeCorrectsInvalidBadMaxPercentOfSegmentsToConsiderPerMove() throws Exception + public void testSerdeHandleInvalidPercentOfSegmentsToConsiderPerMove() throws Exception { try { String jsonStr = "{\n" @@ -233,6 +232,38 @@ public class CoordinatorDynamicConfigTest } @Test + public void testHandleMissingPercentOfSegmentsToConsiderPerMove() throws Exception + { + String jsonStr = "{\n" + + " \"millisToWaitBeforeDeleting\": 1,\n" + + " \"mergeBytesLimit\": 1,\n" + + " \"mergeSegmentsLimit\" : 1,\n" + + " \"maxSegmentsToMove\": 1,\n" + + " \"replicantLifetime\": 1,\n" + + " \"replicationThrottleLimit\": 1,\n" + + " \"balancerComputeThreads\": 2, \n" + + " \"emitBalancingStats\": true,\n" + + " \"killDataSourceWhitelist\": [\"test1\",\"test2\"],\n" + + " \"maxSegmentsInNodeLoadingQueue\": 1,\n" + + " \"decommissioningNodes\": [\"host1\", \"host2\"],\n" + + " \"decommissioningMaxPercentOfMaxSegmentsToMove\": 9,\n" + + " \"pauseCoordination\": false\n" + + "}\n"; + CoordinatorDynamicConfig actual = mapper.readValue( + mapper.writeValueAsString( + mapper.readValue( + jsonStr, + CoordinatorDynamicConfig.class + ) + ), + CoordinatorDynamicConfig.class + ); + ImmutableSet<String> decommissioning = ImmutableSet.of("host1", "host2"); + ImmutableSet<String> whitelist = ImmutableSet.of("test1", "test2"); + assertConfig(actual, 1, 1, 1, 1, 100, 1, 1, 2, true, whitelist, false, 1, decommissioning, 9, false); + } + + @Test public void testSerdeWithKillAllDataSources() throws Exception { String jsonStr = "{\n" --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org