This is an automated email from the ASF dual-hosted git repository. nehapawar pushed a commit to branch split_vm_tasks_2 in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
commit 0ff4723735ed8c8cd7146d69cf10a2c716b3cd89 Author: Neha Pawar <[email protected]> AuthorDate: Mon Jan 7 10:49:35 2019 -0800 Review comments --- .../linkedin/pinot/controller/ControllerConf.java | 31 +++++++++++++++++++++- .../pinot/controller/ControllerStarter.java | 3 +-- .../BrokerResourceValidationManager.java | 1 - .../validation/OfflineSegmentIntervalChecker.java | 5 +--- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java index 1b496d9..9f84691 100644 --- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java +++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerConf.java @@ -57,6 +57,7 @@ public class ControllerConf extends PropertiesConfiguration { public static class ControllerPeriodicTasksConf { private static final String RETENTION_MANAGER_FREQUENCY_IN_SECONDS = "controller.retention.frequencyInSeconds"; + private static final String VALIDATION_MANAGER_FREQUENCY_IN_SECONDS = "controller.validation.frequencyInSeconds"; private static final String OFFLINE_SEGMENT_INTERVAL_CHECKER_FREQUENCY_IN_SECONDS = "controller.offline.segment.interval.checker.frequencyInSeconds"; private static final String REALTIME_SEGMENT_VALIDATION_FREQUENCY_IN_SECONDS = @@ -75,6 +76,7 @@ public class ControllerConf extends PropertiesConfiguration { "controller.segment.level.validation.intervalInSeconds"; private static final int DEFAULT_RETENTION_CONTROLLER_FREQUENCY_IN_SECONDS = 6 * 60 * 60; // 6 Hours. + private static final int DEFAULT_VALIDATION_CONTROLLER_FREQUENCY_IN_SECONDS = 60 * 60; // 1 Hour. private static final int DEFAULT_OFFLINE_SEGMENT_INTERVAL_CHECKER_FREQUENCY_IN_SECONDS = 6 * 60 * 60; // 6 Hours. private static final int DEFAULT_REALTIME_SEGMENT_VALIDATION_FREQUENCY_IN_SECONDS = 60 * 60; // 1 Hour. private static final int DEFAULT_BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS = 60 * 60; // 1 Hour. @@ -351,6 +353,28 @@ public class ControllerConf extends PropertiesConfiguration { Integer.toString(retentionFrequencyInSeconds)); } + /** + * Deprecated. The validation manager has been split into 3 separate tasks, each having their own frequency config + * @return + */ + @Deprecated + public int getValidationControllerFrequencyInSeconds() { + if (containsKey(ControllerPeriodicTasksConf.VALIDATION_MANAGER_FREQUENCY_IN_SECONDS)) { + return Integer.parseInt( + (String) getProperty(ControllerPeriodicTasksConf.VALIDATION_MANAGER_FREQUENCY_IN_SECONDS)); + } + return ControllerPeriodicTasksConf.DEFAULT_VALIDATION_CONTROLLER_FREQUENCY_IN_SECONDS; + } + + /** + * Deprecated. The validation manager has been split into 3 separate tasks, each having their own frequency config + * @return + */ + public void setValidationControllerFrequencyInSeconds(int validationFrequencyInSeconds) { + setProperty(ControllerPeriodicTasksConf.VALIDATION_MANAGER_FREQUENCY_IN_SECONDS, + Integer.toString(validationFrequencyInSeconds)); + } + public int getOfflineSegmentIntervalCheckerFrequencyInSeconds() { if (containsKey(ControllerPeriodicTasksConf.OFFLINE_SEGMENT_INTERVAL_CHECKER_FREQUENCY_IN_SECONDS)) { return Integer.parseInt( @@ -378,12 +402,17 @@ public class ControllerConf extends PropertiesConfiguration { Integer.toString(validationFrequencyInSeconds)); } + /** + * Return broker resource validation frequency if present, else return the validation manager frequency + * This is so that we can rollout with no config changes to the frequency of this task + * @return + */ public int getBrokerResourceValidationFrequencyInSeconds() { if (containsKey(ControllerPeriodicTasksConf.BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS)) { return Integer.parseInt( (String) getProperty(ControllerPeriodicTasksConf.BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS)); } - return ControllerPeriodicTasksConf.DEFAULT_BROKER_RESOURCE_VALIDATION_FREQUENCY_IN_SECONDS; + return getValidationControllerFrequencyInSeconds(); } public void setBrokerResourceValidationFrequencyInSeconds(int validationFrequencyInSeconds) { diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java index 07ca195..2bca8ae 100644 --- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java +++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/ControllerStarter.java @@ -193,8 +193,7 @@ public class ControllerStarter { periodicTasks.add(_taskManager); periodicTasks.add(_retentionManager); _offlineSegmentIntervalChecker = - new OfflineSegmentIntervalChecker(_config, _helixResourceManager, PinotLLCRealtimeSegmentManager.getInstance(), - new ValidationMetrics(_metricsRegistry)); + new OfflineSegmentIntervalChecker(_config, _helixResourceManager, new ValidationMetrics(_metricsRegistry)); _realtimeSegmentValidationManager = new RealtimeSegmentValidationManager(_config, _helixResourceManager, PinotLLCRealtimeSegmentManager.getInstance(), new ValidationMetrics(_metricsRegistry)); diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java index e3264c4..1f09b64 100644 --- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java +++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/BrokerResourceValidationManager.java @@ -41,7 +41,6 @@ public class BrokerResourceValidationManager extends ControllerPeriodicTask { @Override protected void preprocess() { - // Cache instance configs to reduce ZK access _instanceConfigs = _pinotHelixResourceManager.getAllHelixInstanceConfigs(); } diff --git a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java index 592bd0a..3c79205 100644 --- a/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java +++ b/pinot-controller/src/main/java/com/linkedin/pinot/controller/validation/OfflineSegmentIntervalChecker.java @@ -29,7 +29,6 @@ import com.linkedin.pinot.common.utils.time.TimeUtils; import com.linkedin.pinot.controller.ControllerConf; import com.linkedin.pinot.controller.helix.core.PinotHelixResourceManager; import com.linkedin.pinot.controller.helix.core.periodictask.ControllerPeriodicTask; -import com.linkedin.pinot.controller.helix.core.realtime.PinotLLCRealtimeSegmentManager; import java.util.ArrayList; import java.util.List; import org.apache.commons.lang3.StringUtils; @@ -46,14 +45,12 @@ import org.slf4j.LoggerFactory; public class OfflineSegmentIntervalChecker extends ControllerPeriodicTask { private static final Logger LOGGER = LoggerFactory.getLogger(OfflineSegmentIntervalChecker.class); - private final PinotLLCRealtimeSegmentManager _llcRealtimeSegmentManager; private final ValidationMetrics _validationMetrics; public OfflineSegmentIntervalChecker(ControllerConf config, PinotHelixResourceManager pinotHelixResourceManager, - PinotLLCRealtimeSegmentManager llcRealtimeSegmentManager, ValidationMetrics validationMetrics) { + ValidationMetrics validationMetrics) { super("OfflineSegmentIntervalChecker", config.getOfflineSegmentIntervalCheckerFrequencyInSeconds(), pinotHelixResourceManager); - _llcRealtimeSegmentManager = llcRealtimeSegmentManager; _validationMetrics = validationMetrics; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
