This is an automated email from the ASF dual-hosted git repository.
Jackie-Jiang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6bb61cac7d5 remove needReload check in table rebalance (#19295)
6bb61cac7d5 is described below
commit 6bb61cac7d56d79b36b4aa1178963fb10f28d1bf
Author: Jhow <[email protected]>
AuthorDate: Thu Aug 20 01:56:14 2026 +0800
remove needReload check in table rebalance (#19295)
---
.../core/rebalance/DefaultRebalancePreChecker.java | 44 -------
.../TableRebalancerClusterStatelessTest.java | 9 +-
.../tests/TableRebalanceIntegrationTest.java | 131 +++------------------
3 files changed, 19 insertions(+), 165 deletions(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
index e99906b6ebb..905690e1ed8 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/DefaultRebalancePreChecker.java
@@ -17,8 +17,6 @@
* under the License.
*/
package org.apache.pinot.controller.helix.core.rebalance;
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -29,16 +27,13 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import javax.annotation.Nullable;
import org.apache.commons.collections4.CollectionUtils;
-import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager;
import org.apache.pinot.common.assignment.InstanceAssignmentConfigUtils;
-import org.apache.pinot.common.exception.InvalidConfigException;
import org.apache.pinot.common.restlet.resources.DiskUsageInfo;
import org.apache.pinot.common.restlet.resources.RebalanceConfig;
import org.apache.pinot.common.restlet.resources.RebalancePreCheckerResult;
import org.apache.pinot.common.restlet.resources.RebalanceSummaryResult;
import org.apache.pinot.controller.helix.core.PinotHelixResourceManager;
import
org.apache.pinot.controller.helix.core.assignment.segment.SegmentAssignmentUtils;
-import org.apache.pinot.controller.util.TableMetadataReader;
import org.apache.pinot.controller.util.TableSizeReader;
import org.apache.pinot.controller.validation.ResourceUtilizationInfo;
import org.apache.pinot.spi.config.table.TableConfig;
@@ -54,7 +49,6 @@ import org.slf4j.LoggerFactory;
public class DefaultRebalancePreChecker implements RebalancePreChecker {
- public static final String NEEDS_RELOAD_STATUS = "needsReloadStatus";
public static final String IS_MINIMIZE_DATA_MOVEMENT =
"isMinimizeDataMovement";
public static final String DISK_UTILIZATION = "diskUtilization";
public static final String REBALANCE_CONFIG_OPTIONS =
"rebalanceConfigOptions";
@@ -90,8 +84,6 @@ public class DefaultRebalancePreChecker implements
RebalancePreChecker {
// Right now pre-check items are done sequentially. If pre-check items are
to be done in parallel, we should not
// use linked hash map but to sort the result in the end
Map<String, RebalancePreCheckerResult> preCheckResult = new
LinkedHashMap<>();
- // Check for reload status
- preCheckResult.put(NEEDS_RELOAD_STATUS,
checkReloadNeededOnServers(tableNameWithType, tableRebalanceLogger));
// Check whether minimizeDataMovement is set in TableConfig
preCheckResult.put(IS_MINIMIZE_DATA_MOVEMENT,
checkIsMinimizeDataMovement(tableConfig, rebalanceConfig,
tableRebalanceLogger));
@@ -120,42 +112,6 @@ public class DefaultRebalancePreChecker implements
RebalancePreChecker {
return preCheckResult;
}
- /// Checks if the current segments on any servers needs a reload (table
config or schema change that hasn't been
- /// applied yet). This check does not guarantee that the segments in deep
store are up to date.
- /// TODO: Add an API to check for whether segments in deep store are up to
date with the table configs and schema
- /// and add a pre-check here to call that API.
- private RebalancePreCheckerResult checkReloadNeededOnServers(String
tableNameWithType, Logger tableRebalanceLogger) {
- tableRebalanceLogger.info("Fetching whether reload is needed");
- Boolean needsReload = null;
- if (_executorService == null) {
- tableRebalanceLogger.warn("Executor service is null, skipping
needsReload check");
- return RebalancePreCheckerResult.error("Could not determine needReload
status, run needReload API manually");
- }
- try (PoolingHttpClientConnectionManager connectionManager = new
PoolingHttpClientConnectionManager()) {
- TableMetadataReader metadataReader = new
TableMetadataReader(_executorService, connectionManager,
- _pinotHelixResourceManager);
- TableMetadataReader.TableReloadJsonResponse needsReloadMetadataPair =
-
metadataReader.getServerCheckSegmentsReloadMetadata(tableNameWithType, 30_000);
- Map<String, JsonNode> needsReloadMetadata =
needsReloadMetadataPair.getServerReloadJsonResponses();
- int failedResponses = needsReloadMetadataPair.getNumFailedResponses();
- tableRebalanceLogger.info("Received {} needs reload responses and {}
failed responses from servers assigned "
- + "to table", needsReloadMetadata.size(), failedResponses);
- needsReload = needsReloadMetadata.values().stream().anyMatch(value ->
value.get("needReload").booleanValue());
- if (!needsReload && failedResponses > 0) {
- tableRebalanceLogger.warn("Received {} failed responses from servers
and needsReload is false from returned "
- + "responses, check needsReload status manually", failedResponses);
- needsReload = null;
- }
- } catch (InvalidConfigException | IOException e) {
- tableRebalanceLogger.warn("Caught exception while trying to fetch reload
status from servers", e);
- }
-
- return needsReload == null
- ? RebalancePreCheckerResult.error("Could not determine needReload
status, run needReload API manually")
- : !needsReload ? RebalancePreCheckerResult.pass("No need to reload")
- : RebalancePreCheckerResult.warn("Reload needed prior to running
rebalance");
- }
-
/// Checks if minimize data movement is set for the given table in the
TableConfig
private RebalancePreCheckerResult checkIsMinimizeDataMovement(TableConfig
tableConfig,
RebalanceConfig rebalanceConfig, Logger tableRebalanceLogger) {
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
index d010aa7ccb6..e07d8e58b06 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalancerClusterStatelessTest.java
@@ -277,18 +277,11 @@ public class TableRebalancerClusterStatelessTest extends
ControllerTest {
assertEquals(rebalanceResult.getStatus(), RebalanceResult.Status.DONE);
Map<String, RebalancePreCheckerResult> preCheckResult =
rebalanceResult.getPreChecksResult();
assertNotNull(preCheckResult);
- assertEquals(preCheckResult.size(), 5);
-
assertTrue(preCheckResult.containsKey(DefaultRebalancePreChecker.NEEDS_RELOAD_STATUS));
+ assertEquals(preCheckResult.size(), 4);
assertTrue(preCheckResult.containsKey(DefaultRebalancePreChecker.IS_MINIMIZE_DATA_MOVEMENT));
assertTrue(preCheckResult.containsKey(DefaultRebalancePreChecker.DISK_UTILIZATION));
assertTrue(preCheckResult.containsKey(DefaultRebalancePreChecker.REBALANCE_CONFIG_OPTIONS));
assertTrue(preCheckResult.containsKey(DefaultRebalancePreChecker.REPLICA_GROUPS_INFO));
- // Sending request to servers should fail for all, so needsPreprocess
should be set to "error" to indicate that a
- // manual check is needed
-
assertEquals(preCheckResult.get(DefaultRebalancePreChecker.NEEDS_RELOAD_STATUS).getPreCheckStatus(),
- RebalancePreCheckerResult.PreCheckStatus.ERROR);
-
assertEquals(preCheckResult.get(DefaultRebalancePreChecker.NEEDS_RELOAD_STATUS).getMessage(),
- "Could not determine needReload status, run needReload API
manually");
assertEquals(preCheckResult.get(DefaultRebalancePreChecker.IS_MINIMIZE_DATA_MOVEMENT).getPreCheckStatus(),
RebalancePreCheckerResult.PreCheckStatus.PASS);
assertEquals(preCheckResult.get(DefaultRebalancePreChecker.IS_MINIMIZE_DATA_MOVEMENT).getMessage(),
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TableRebalanceIntegrationTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TableRebalanceIntegrationTest.java
index d7e7713aaf2..ed24dcaf23f 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TableRebalanceIntegrationTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/TableRebalanceIntegrationTest.java
@@ -27,7 +27,6 @@ import java.util.Map;
import javax.ws.rs.core.Response;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pinot.client.admin.PinotAdminNotFoundException;
-import
org.apache.pinot.common.restlet.resources.PinotTableReloadStatusResponse;
import org.apache.pinot.common.restlet.resources.RebalanceConfig;
import org.apache.pinot.common.restlet.resources.RebalancePreCheckerResult;
import org.apache.pinot.common.restlet.resources.RebalanceResult;
@@ -35,9 +34,6 @@ import
org.apache.pinot.common.restlet.resources.RebalanceSummaryResult;
import org.apache.pinot.common.restlet.resources.TableRebalanceProgressStats;
import org.apache.pinot.common.tier.TierFactory;
import org.apache.pinot.common.utils.config.TagNameUtils;
-import org.apache.pinot.common.utils.regex.JavaUtilPattern;
-import org.apache.pinot.common.utils.regex.Matcher;
-import org.apache.pinot.common.utils.regex.Pattern;
import org.apache.pinot.controller.ControllerConf;
import org.apache.pinot.controller.helix.core.controllerjob.ControllerJobTypes;
import
org.apache.pinot.controller.helix.core.rebalance.DefaultRebalancePreChecker;
@@ -173,7 +169,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
@@ -191,8 +186,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is not enabled for COMPLETED segments, but
instance assignment is allowed",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -213,7 +207,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
@@ -232,8 +225,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled for COMPLETED segments in table
config but it's overridden with disabled",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -252,8 +244,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is not enabled for CONSUMING segments, but
instance assignment is allowed",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - Replica Groups are "
+ "not enabled, replication: " + tableConfig.getReplication() +
"\nCONSUMING segments - numReplicaGroups: "
@@ -274,7 +265,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - Replica Groups are "
@@ -294,8 +284,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled for CONSUMING segments in table
config but it's overridden with disabled",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - Replica Groups are "
+ "not enabled, replication: " + tableConfig.getReplication() +
"\nCONSUMING segments - numReplicaGroups: "
@@ -316,8 +305,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is not enabled for either or both COMPLETED and
CONSUMING segments, but instance "
+ "assignment is allowed for both",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -341,7 +329,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
@@ -363,7 +350,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
@@ -387,8 +373,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled for both COMPLETED and CONSUMING
segments in table config but it's "
+ "overridden with disabled",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -410,8 +395,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is not enabled for either or both COMPLETED and
CONSUMING segments, but instance "
+ "assignment is allowed for both",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -436,8 +420,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled for CONSUMING segments in table
config but it's overridden with disabled",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - Replica Groups are "
+ "not enabled, replication: " + tableConfig.getReplication() +
"\nCONSUMING segments - numReplicaGroups: "
@@ -468,7 +451,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "includeConsuming is
disabled for a realtime table.",
RebalancePreCheckerResult.PreCheckStatus.WARN,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
@@ -486,7 +468,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
RebalancePreCheckerResult.PreCheckStatus.PASS,
- "No need to reload", RebalancePreCheckerResult.PreCheckStatus.PASS,
"All rebalance parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
@@ -504,8 +485,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
+ "\nCONSUMING segments - Replica Groups are not enabled,
replication: " + tableConfig.getReplication(),
@@ -517,7 +497,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
@@ -525,9 +504,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
RebalancePreCheckerResult.PreCheckStatus.PASS);
// Add a new server (to force change in instance assignment) and enable
reassignInstances
- // Validate that the status for reload is still PASS (i.e. even though an
extra server is tagged which has no
- // segments assigned for this table, we don't try to get needReload status
from that extra server, otherwise
- // ERROR status would be returned)
BaseServerStarter serverStarter0 = startOneServer(NUM_SERVERS);
createServerTenant(getServerTenant(), 0, 1);
rebalanceConfig.setReassignInstances(true);
@@ -536,7 +512,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.DONE,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
@@ -557,8 +532,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
+ "\nCONSUMING segments - Replica Groups are not enabled,
replication: " + tableConfig.getReplication(),
@@ -573,7 +547,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
RebalancePreCheckerResult.PreCheckStatus.PASS,
- "Reload needed prior to running rebalance",
RebalancePreCheckerResult.PreCheckStatus.WARN,
"All rebalance parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - numReplicaGroups: "
@@ -592,8 +565,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nCOMPLETED segments - Replica Groups are "
+ "not enabled, replication: " + tableConfig.getReplication() +
"\nCONSUMING segments - numReplicaGroups: "
@@ -615,8 +587,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.DONE,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN,
+ RebalancePreCheckerResult.PreCheckStatus.PASS,
"bestEfforts is enabled, only enable it if you know what you are
doing\n"
+ "bootstrap is enabled which can cause a large amount of data
movement, double check if this is "
+ "intended", RebalancePreCheckerResult.PreCheckStatus.WARN,
@@ -624,15 +595,11 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
+ "\nCONSUMING segments - Replica Groups are not enabled,
replication: " + tableConfig.getReplication(),
RebalancePreCheckerResult.PreCheckStatus.PASS);
- String response = reloadRealtimeTable(getTableName());
- waitForReloadToComplete(getReloadJobIdFromResponse(response), 30_000);
-
rebalanceConfig.setBestEfforts(false);
rebalanceConfig.setBootstrap(false);
rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.REALTIME);
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.DONE,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"COMPLETED segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
@@ -668,7 +635,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication(),
@@ -685,7 +651,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
RebalancePreCheckerResult.PreCheckStatus.PASS,
- "No need to reload", RebalancePreCheckerResult.PreCheckStatus.PASS,
"All rebalance parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nOFFLINE segments - numReplicaGroups: "
@@ -700,8 +665,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled in table config but it's overridden
with disabled",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nOFFLINE segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -720,8 +684,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is not enabled but instance assignment is
allowed",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "No need to reload",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nOFFLINE segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -738,8 +701,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication(),
RebalancePreCheckerResult.PreCheckStatus.PASS);
@@ -751,7 +713,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication(),
@@ -768,7 +729,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication()
@@ -780,7 +740,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"updateTargetTier should be enabled when tier configs are present",
RebalancePreCheckerResult.PreCheckStatus.WARN,
@@ -793,9 +752,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
updateTableConfig(tableConfig);
// Add a new server (to force change in instance assignment) and enable
reassignInstances
- // Validate that the status for reload is still PASS (i.e. even though an
extra server is tagged which has no
- // segments assigned for this table, we don't try to get needReload status
from that extra server, otherwise
- // ERROR status would be returned)
BaseServerStarter serverStarter0 = startOneServer(NUM_SERVERS);
rebalanceConfig.setReassignInstances(true);
tableConfig.setInstanceAssignmentConfigMap(null);
@@ -803,7 +759,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.DONE,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication(),
@@ -823,8 +778,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication(),
RebalancePreCheckerResult.PreCheckStatus.PASS);
@@ -839,7 +793,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
RebalancePreCheckerResult.PreCheckStatus.PASS,
- "Reload needed prior to running rebalance",
RebalancePreCheckerResult.PreCheckStatus.WARN,
"All rebalance parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nOFFLINE segments - numReplicaGroups: "
@@ -858,8 +811,7 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.NO_OP,
"minimizeDataMovement is enabled",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN, "All rebalance
parameters look good",
+ RebalancePreCheckerResult.PreCheckStatus.PASS, "All rebalance
parameters look good",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"reassignInstances is disabled, replica groups may not be
updated.\nOFFLINE segments - numReplicaGroups: "
+ replicaGroupPartitionConfig.getNumReplicaGroups() + ",
numInstancesPerReplicaGroup: "
@@ -880,25 +832,6 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
assertNotNull(rebalanceResult.getRebalanceSummaryResult());
checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.DONE,
"Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "Reload needed prior to
running rebalance",
- RebalancePreCheckerResult.PreCheckStatus.WARN,
- "bestEfforts is enabled, only enable it if you know what you are
doing\n"
- + "bootstrap is enabled which can cause a large amount of data
movement, double check if this is "
- + "intended", RebalancePreCheckerResult.PreCheckStatus.WARN,
- "OFFLINE segments - Replica Groups are not enabled, replication: " +
tableConfig.getReplication(),
- RebalancePreCheckerResult.PreCheckStatus.PASS);
-
- // reload - needed due to the schema change since that cannot be undone
- String response = reloadOfflineTable(getTableName(), false);
- waitForReloadToComplete(getReloadJobIdFromResponse(response), 30_000);
- // reload realtime table as well for other realtime tests
- response = reloadRealtimeTable(getTableName());
- waitForReloadToComplete(getReloadJobIdFromResponse(response), 30_000);
-
- rebalanceResult = triggerTableRebalance(rebalanceConfig,
TableType.OFFLINE);
- checkRebalancePreCheckStatus(rebalanceResult, RebalanceResult.Status.DONE,
- "Instance assignment not allowed, no need for minimizeDataMovement",
- RebalancePreCheckerResult.PreCheckStatus.PASS, "No need to reload",
RebalancePreCheckerResult.PreCheckStatus.PASS,
"bestEfforts is enabled, only enable it if you know what you are
doing\n"
+ "bootstrap is enabled which can cause a large amount of data
movement, double check if this is "
@@ -925,25 +858,19 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
private void checkRebalancePreCheckStatus(RebalanceResult rebalanceResult,
RebalanceResult.Status expectedStatus,
String expectedMinimizeDataMovement,
RebalancePreCheckerResult.PreCheckStatus expectedMinimizeDataMovementStatus,
- String expectedNeedsReloadMessage,
RebalancePreCheckerResult.PreCheckStatus expectedNeedsReloadStatus,
String expectedRebalanceConfig, RebalancePreCheckerResult.PreCheckStatus
expectedRebalanceConfigStatus,
String expectedReplicaGroupMessage,
RebalancePreCheckerResult.PreCheckStatus expectedReplicaGroupStatus) {
assertEquals(rebalanceResult.getStatus(), expectedStatus);
Map<String, RebalancePreCheckerResult> preChecksResult =
rebalanceResult.getPreChecksResult();
assertNotNull(preChecksResult);
- assertEquals(preChecksResult.size(), 5);
+ assertEquals(preChecksResult.size(), 4);
assertTrue(preChecksResult.containsKey(DefaultRebalancePreChecker.IS_MINIMIZE_DATA_MOVEMENT));
-
assertTrue(preChecksResult.containsKey(DefaultRebalancePreChecker.NEEDS_RELOAD_STATUS));
assertTrue(preChecksResult.containsKey(DefaultRebalancePreChecker.DISK_UTILIZATION));
assertTrue(preChecksResult.containsKey(DefaultRebalancePreChecker.REBALANCE_CONFIG_OPTIONS));
assertEquals(preChecksResult.get(DefaultRebalancePreChecker.IS_MINIMIZE_DATA_MOVEMENT).getPreCheckStatus(),
expectedMinimizeDataMovementStatus);
assertEquals(preChecksResult.get(DefaultRebalancePreChecker.IS_MINIMIZE_DATA_MOVEMENT).getMessage(),
expectedMinimizeDataMovement);
-
assertEquals(preChecksResult.get(DefaultRebalancePreChecker.NEEDS_RELOAD_STATUS).getPreCheckStatus(),
- expectedNeedsReloadStatus);
-
assertEquals(preChecksResult.get(DefaultRebalancePreChecker.NEEDS_RELOAD_STATUS).getMessage(),
- expectedNeedsReloadMessage);
assertEquals(preChecksResult.get(DefaultRebalancePreChecker.REBALANCE_CONFIG_OPTIONS).getPreCheckStatus(),
expectedRebalanceConfigStatus);
assertEquals(preChecksResult.get(DefaultRebalancePreChecker.REBALANCE_CONFIG_OPTIONS).getMessage(),
@@ -1412,26 +1339,4 @@ public class TableRebalanceIntegrationTest extends
BaseHybridClusterIntegrationT
serverStarter3.stop();
}
}
-
- private String getReloadJobIdFromResponse(String response) {
- Pattern pattern = new
JavaUtilPattern("([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})");
- Matcher matcher = pattern.matcher(response);
- String jobId = matcher.find() ? matcher.group(1) : null;
- if (jobId == null) {
- return "";
- }
- return jobId;
- }
-
- private void waitForReloadToComplete(String reloadJobId, long timeoutMs) {
- TestUtils.waitForCondition(aVoid -> {
- try {
- PinotTableReloadStatusResponse reloadResult =
getOrCreateAdminClient().getSegmentClient()
- .getSegmentReloadStatusObject(reloadJobId);
- return reloadResult.getEstimatedTimeRemainingInMinutes() == 0.0;
- } catch (Exception e) {
- return null;
- }
- }, 1000L, timeoutMs, "Failed to reload all segments");
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
