AMBARI-13678 Make it configurable as to how frequently ActionScheduler scans DB for tasks (dsen)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e86d1cd7 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e86d1cd7 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e86d1cd7 Branch: refs/heads/branch-dev-patch-upgrade Commit: e86d1cd7c3fa6a03f289cd83ed945b5b9acfc668 Parents: 9b87326 Author: Dmytro Sen <[email protected]> Authored: Tue Nov 3 11:43:59 2015 +0200 Committer: Dmytro Sen <[email protected]> Committed: Tue Nov 3 11:43:59 2015 +0200 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 55 +++++++++++++++----- .../server/controller/ControllerModule.java | 3 +- .../server/configuration/ConfigurationTest.java | 18 +++++++ .../scheduler/ExecutionSchedulerTest.java | 6 +-- 4 files changed, 64 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/e86d1cd7/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 227f2a9..3a282ed 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -270,15 +270,17 @@ public class Configuration { public static final String REPO_SUFFIX_KEY_UBUNTU = "repo.validation.suffixes.ubuntu"; public static final String REPO_SUFFIX_KEY_DEFAULT = "repo.validation.suffixes.default"; - public static final String EXECUTION_SCHEDULER_CLUSTERED = "server.execution.scheduler.isClustered"; - public static final String EXECUTION_SCHEDULER_THREADS = "server.execution.scheduler.maxThreads"; - public static final String EXECUTION_SCHEDULER_CONNECTIONS = "server.execution.scheduler.maxDbConnections"; - public static final String EXECUTION_SCHEDULER_MISFIRE_TOLERATION = "server.execution.scheduler.misfire.toleration.minutes"; - public static final String EXECUTION_SCHEDULER_START_DELAY = "server.execution.scheduler.start.delay.seconds"; + public static final String EXECUTION_SCHEDULER_CLUSTERED_KEY = "server.execution.scheduler.isClustered"; + public static final String EXECUTION_SCHEDULER_THREADS_KEY = "server.execution.scheduler.maxThreads"; + public static final String EXECUTION_SCHEDULER_CONNECTIONS_KEY = "server.execution.scheduler.maxDbConnections"; + public static final String EXECUTION_SCHEDULER_MISFIRE_TOLERATION_KEY = "server.execution.scheduler.misfire.toleration.minutes"; + public static final String EXECUTION_SCHEDULER_START_DELAY_KEY = "server.execution.scheduler.start.delay.seconds"; + public static final String EXECUTION_SCHEDULER_WAIT_KEY = "server.execution.scheduler.wait"; public static final String DEFAULT_SCHEDULER_THREAD_COUNT = "5"; public static final String DEFAULT_SCHEDULER_MAX_CONNECTIONS = "5"; public static final String DEFAULT_EXECUTION_SCHEDULER_MISFIRE_TOLERATION = "480"; public static final String DEFAULT_SCHEDULER_START_DELAY_SECONDS = "120"; + public static final String DEFAULT_EXECUTION_SCHEDULER_WAIT_SECONDS = "1"; public static final String SERVER_TMP_DIR_KEY = "server.tmp.dir"; public static final String SERVER_TMP_DIR_DEFAULT = "/var/lib/ambari-server/tmp"; public static final String EXTERNAL_SCRIPT_TIMEOUT_KEY = "server.script.timeout"; @@ -1251,8 +1253,8 @@ public class Configuration { */ public boolean isAgentApiGzipped() { return "true".equalsIgnoreCase(properties.getProperty( - AGENT_API_GZIP_COMPRESSION_ENABLED_KEY, - API_GZIP_COMPRESSION_ENABLED_DEFAULT)); + AGENT_API_GZIP_COMPRESSION_ENABLED_KEY, + API_GZIP_COMPRESSION_ENABLED_DEFAULT)); } /** @@ -1748,17 +1750,17 @@ public class Configuration { public String isExecutionSchedulerClusterd() { - return properties.getProperty(EXECUTION_SCHEDULER_CLUSTERED, "false"); + return properties.getProperty(EXECUTION_SCHEDULER_CLUSTERED_KEY, "false"); } public String getExecutionSchedulerThreads() { - return properties.getProperty(EXECUTION_SCHEDULER_THREADS, + return properties.getProperty(EXECUTION_SCHEDULER_THREADS_KEY, DEFAULT_SCHEDULER_THREAD_COUNT); } public Integer getRequestReadTimeout() { return Integer.parseInt(properties.getProperty(REQUEST_READ_TIMEOUT, - REQUEST_READ_TIMEOUT_DEFAULT)); + REQUEST_READ_TIMEOUT_DEFAULT)); } public Integer getRequestConnectTimeout() { @@ -1767,23 +1769,48 @@ public class Configuration { } public String getExecutionSchedulerConnections() { - return properties.getProperty(EXECUTION_SCHEDULER_CONNECTIONS, - DEFAULT_SCHEDULER_MAX_CONNECTIONS); + return properties.getProperty(EXECUTION_SCHEDULER_CONNECTIONS_KEY, + DEFAULT_SCHEDULER_MAX_CONNECTIONS); } public Long getExecutionSchedulerMisfireToleration() { String limit = properties.getProperty - (EXECUTION_SCHEDULER_MISFIRE_TOLERATION, + (EXECUTION_SCHEDULER_MISFIRE_TOLERATION_KEY, DEFAULT_EXECUTION_SCHEDULER_MISFIRE_TOLERATION); return Long.parseLong(limit); } public Integer getExecutionSchedulerStartDelay() { - String delay = properties.getProperty(EXECUTION_SCHEDULER_START_DELAY, + String delay = properties.getProperty(EXECUTION_SCHEDULER_START_DELAY_KEY, DEFAULT_SCHEDULER_START_DELAY_SECONDS); return Integer.parseInt(delay); } + public Long getExecutionSchedulerWait() { + + String stringValue = properties.getProperty( + EXECUTION_SCHEDULER_WAIT_KEY, DEFAULT_EXECUTION_SCHEDULER_WAIT_SECONDS); + Long sleepTime = Long.parseLong(DEFAULT_EXECUTION_SCHEDULER_WAIT_SECONDS); + if (stringValue != null) { + try { + sleepTime = Long.valueOf(stringValue); + } catch (NumberFormatException ignored) { + LOG.warn("Value of {} ({}) should be a number, " + + "falling back to default value ({})", EXECUTION_SCHEDULER_WAIT_KEY, + stringValue, DEFAULT_EXECUTION_SCHEDULER_WAIT_SECONDS); + } + + } + + if (sleepTime > 60) { + LOG.warn("Value of {} ({}) should be a number between 1 adn 60, " + + "falling back to maximum value ({})", + EXECUTION_SCHEDULER_WAIT_KEY, sleepTime, 60); + sleepTime = 60L; + } + return sleepTime*1000; + } + public Integer getExternalScriptTimeout() { return Integer.parseInt(properties.getProperty(EXTERNAL_SCRIPT_TIMEOUT_KEY, EXTERNAL_SCRIPT_TIMEOUT_DEFAULT)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/e86d1cd7/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index 60217c0..20f0482 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@ -316,7 +316,8 @@ public class ControllerModule extends AbstractModule { bind(Clusters.class).to(ClustersImpl.class); bind(AmbariCustomCommandExecutionHelper.class); bind(ActionDBAccessor.class).to(ActionDBAccessorImpl.class); - bindConstant().annotatedWith(Names.named("schedulerSleeptime")).to(1000L); + bindConstant().annotatedWith(Names.named("schedulerSleeptime")).to( + configuration.getExecutionSchedulerWait()); // This time is added to summary timeout time of all tasks in stage // So it's an "additional time", given to stage to finish execution before http://git-wip-us.apache.org/repos/asf/ambari/blob/e86d1cd7/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java b/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java index 0f349fa..7af0167 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/configuration/ConfigurationTest.java @@ -487,6 +487,24 @@ public class ConfigurationTest { } @Test + public void testGetExecutionSchedulerWait() throws Exception { + final Properties ambariProperties = new Properties(); + final Configuration configuration = new Configuration(ambariProperties); + + //default + Assert.assertEquals(new Long(1000L), configuration.getExecutionSchedulerWait()); + + ambariProperties.setProperty(Configuration.EXECUTION_SCHEDULER_WAIT_KEY, "5"); + Assert.assertEquals(new Long(5000L), configuration.getExecutionSchedulerWait()); + // > 60 secs + ambariProperties.setProperty(Configuration.EXECUTION_SCHEDULER_WAIT_KEY, "100"); + Assert.assertEquals(new Long(60000L), configuration.getExecutionSchedulerWait()); + //not a number + ambariProperties.setProperty(Configuration.EXECUTION_SCHEDULER_WAIT_KEY, "100m"); + Assert.assertEquals(new Long(1000L), configuration.getExecutionSchedulerWait()); + } + + @Test public void testExperimentalConcurrentStageProcessing() throws Exception { final Properties ambariProperties = new Properties(); final Configuration configuration = new Configuration(ambariProperties); http://git-wip-us.apache.org/repos/asf/ambari/blob/e86d1cd7/ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java index faf44e3..8346c65 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/scheduler/ExecutionSchedulerTest.java @@ -48,9 +48,9 @@ public class ExecutionSchedulerTest { @Before public void setup() throws Exception { Properties properties = new Properties(); - properties.setProperty(Configuration.EXECUTION_SCHEDULER_THREADS, "2"); - properties.setProperty(Configuration.EXECUTION_SCHEDULER_CLUSTERED, "false"); - properties.setProperty(Configuration.EXECUTION_SCHEDULER_CONNECTIONS, "2"); + properties.setProperty(Configuration.EXECUTION_SCHEDULER_THREADS_KEY, "2"); + properties.setProperty(Configuration.EXECUTION_SCHEDULER_CLUSTERED_KEY, "false"); + properties.setProperty(Configuration.EXECUTION_SCHEDULER_CONNECTIONS_KEY, "2"); properties.setProperty(Configuration.SERVER_JDBC_DRIVER_KEY, "db.driver"); properties.setProperty(Configuration.SERVER_JDBC_URL_KEY, "jdbc:postgresql://localhost/"); properties.setProperty(Configuration.SERVER_JDBC_USER_NAME_KEY, "user");
