SLIDER-623 make python checks optional for all but the AM tests that spawn workers
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/bdfdd841 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/bdfdd841 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/bdfdd841 Branch: refs/heads/feature/SLIDER-623-python Commit: bdfdd841126a8e41272b94505e8b2175aebe06e7 Parents: e4df15a Author: Steve Loughran <[email protected]> Authored: Sun Nov 9 19:07:48 2014 +0000 Committer: Steve Loughran <[email protected]> Committed: Mon Nov 10 00:09:17 2014 +0000 ---------------------------------------------------------------------- .../org/apache/slider/common/SliderXmlConfKeys.java | 7 +++++++ .../org/apache/slider/common/tools/SliderUtils.java | 12 ++++++++---- .../apache/slider/server/appmaster/SliderAppMaster.java | 7 ++++--- .../slider/common/tools/TestExecutionEnvironment.groovy | 8 ++++++-- .../apache/slider/providers/agent/AgentTestBase.groovy | 10 ++++++---- .../apache/slider/providers/agent/TestAgentEcho.groovy | 5 +++++ .../apache/slider/test/YarnMiniClusterTestBase.groovy | 6 ++++-- 7 files changed, 40 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java index 1c90e3b..e7f8ce5 100644 --- a/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java +++ b/slider-core/src/main/java/org/apache/slider/common/SliderXmlConfKeys.java @@ -149,4 +149,11 @@ public interface SliderXmlConfKeys { String KEY_AM_KEYTAB_LOCAL_PATH = "slider.am.keytab.local.path"; String KEY_KEYTAB_PRINCIPAL = "slider.keytab.principal.name"; String KEY_SECURITY_ENABLED = "site.global.security_enabled"; + + /** + * Set to disable server-side checks for python, openssl &c. + * This should only be set for testing + */ + String KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED = + "slider.am.dependency.checks.disabled"; } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java index 247c25d..db398a6 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java @@ -1975,15 +1975,19 @@ public final class SliderUtils { * This looks for everything felt to be critical for execution, including * native binaries and other essential dependencies. * @param logger logger to log to on normal execution + * @param dependencyChecks flag to indicate checks for agent dependencies * @throws IOException on IO failures * @throws SliderException on validation failures */ - public static void validateSliderServerEnvironment(Logger logger) throws + public static void validateSliderServerEnvironment(Logger logger, + boolean dependencyChecks) throws IOException, SliderException { - maybeVerifyWinUtilsValid(logger); - validatePythonEnv(logger); - validateOpenSSLEnv(logger); + maybeVerifyWinUtilsValid(); + if (dependencyChecks) { + validatePythonEnv(logger); + validateOpenSSLEnv(logger); + } } public static void validateOpenSSLEnv(Logger logger) throws http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index 8791d19..7000583 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -64,7 +64,6 @@ import org.apache.hadoop.registry.client.api.RegistryOperations; import org.apache.hadoop.registry.client.binding.RegistryPathUtils; import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies; import org.apache.hadoop.registry.client.types.ServiceRecord; -import org.apache.hadoop.registry.client.binding.RegistryTypeUtils; import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes; import org.apache.hadoop.registry.server.integration.RMRegistryOperationsService; import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; @@ -106,7 +105,6 @@ import org.apache.slider.core.main.LauncherExitCodes; import org.apache.slider.core.main.RunService; import org.apache.slider.core.main.ServiceLauncher; import org.apache.slider.core.persist.ConfTreeSerDeser; -import org.apache.slider.core.registry.info.CustomRegistryConstants; import org.apache.slider.providers.ProviderCompleted; import org.apache.slider.providers.ProviderRole; import org.apache.slider.providers.ProviderService; @@ -438,7 +436,10 @@ public class SliderAppMaster extends AbstractSliderLaunchedService checkAndWarnForAuthTokenProblems(); // validate server env - SliderUtils.validateSliderServerEnvironment(log); + boolean dependencyChecks = + !conf.getBoolean(KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED, + false); + SliderUtils.validateSliderServerEnvironment(log, dependencyChecks); executorService = new WorkflowExecutorService<ExecutorService>("AmExecutor", Executors.newFixedThreadPool(2, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy index 97b72d1..3b2f992 100644 --- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestExecutionEnvironment.groovy @@ -35,12 +35,16 @@ class TestExecutionEnvironment extends SliderTestBase { @Test public void testWinutils() throws Throwable { SliderUtils.maybeVerifyWinUtilsValid(); - } @Test public void testServerEnv() throws Throwable { - SliderUtils.validateSliderServerEnvironment(log) + SliderUtils.validateSliderServerEnvironment(log, true) + } + + @Test + public void testServerEnvNoDependencies() throws Throwable { + SliderUtils.validateSliderServerEnvironment(log, false) } @Test http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy index ea10494..5bf1c1f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/AgentTestBase.groovy @@ -30,7 +30,6 @@ import org.apache.slider.common.tools.SliderUtils import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.test.YarnZKMiniClusterTestBase import org.junit.Before -import org.junit.BeforeClass import org.junit.Rule import org.junit.rules.TemporaryFolder @@ -52,9 +51,12 @@ public abstract class AgentTestBase extends YarnZKMiniClusterTestBase { /** * Server side test: validate system env before launch */ - @BeforeClass - public static void checkServerEnv() { -// SliderUtils.validateSliderServerEnvironment(null) + public static void assumeValidServerEnv() { + try { + SliderUtils.validateSliderServerEnvironment(log, true) + } catch (Exception e) { + skip(e.toString()) + } } public String app_def_pkg_path; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy index f40d5a7..2eb39e3 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.api.ResourceKeys import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.core.exceptions.BadClusterStateException import org.apache.slider.core.main.ServiceLauncher import org.junit.Before @@ -68,6 +69,8 @@ class TestAgentEcho extends AgentTestBase { @Test public void testEchoOperation() throws Throwable { + assumeValidServerEnv() + String clustername = createMiniCluster("", configuration, 1, @@ -95,6 +98,8 @@ class TestAgentEcho extends AgentTestBase { ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1", ARG_COMP_OPT, role, SCRIPT_PATH, echo_py, ARG_COMP_OPT, role, SERVICE_NAME, "Agent", + ARG_DEFINE, + SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false" ], true, true, true) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/bdfdd841/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy index 4d2cb3b..73b81ab 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy @@ -88,7 +88,10 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest { SLIDER_CONFIG.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 100) SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, false) SLIDER_CONFIG.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, false) + SLIDER_CONFIG.setBoolean(SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED, + true) SLIDER_CONFIG.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 1) + } @@ -345,9 +348,8 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest { public YarnConfiguration getTestConfiguration() { YarnConfiguration conf = getConfiguration() - conf.addResource(SLIDER_TEST_XML) - return conf + return conf; } protected String getRMAddr() {
