SLIDER-443 remove default keystore location SLIDER-449 Consume both streams for openssl (required for windows)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/732569db Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/732569db Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/732569db Branch: refs/heads/feature/SLIDER-149_Support_a_YARN_service_registry Commit: 732569db9468ed7ced6fe5ae750130fdcd455d04 Parents: 964e403 Author: Jon Maron <[email protected]> Authored: Fri Sep 26 17:07:18 2014 -0400 Committer: Jon Maron <[email protected]> Committed: Fri Sep 26 17:07:18 2014 -0400 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 12 +- .../services/security/CertificateManager.java | 48 ++++++- .../providers/slideram/instance/appconf.json | 3 +- .../agent/TestAgentAMManagementWS.groovy | 131 ++++++++++++------- .../apache/slider/test/SliderTestUtils.groovy | 3 +- 5 files changed, 141 insertions(+), 56 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index afbb4a8..d57f7dc 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -782,7 +782,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe builder.init(providerName, instanceDefinition); builder.propagateFilename(); builder.propagatePrincipals(); - builder.setImageDetailsIfAvailable(buildInfo.getImage(), buildInfo.getAppHomeDir()); + builder.setImageDetailsIfAvailable(buildInfo.getImage(), + buildInfo.getAppHomeDir()); builder.setQueue(buildInfo.queue); String quorum = buildInfo.getZKhosts(); @@ -839,7 +840,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe throw e; } try { - builder.persist(appconfdir, overwrite); + persistInstanceDefinition(overwrite, appconfdir, builder); } catch (LockAcquireFailedException e) { log.warn("Failed to get a Lock on {} : {}", builder, e); throw new BadClusterStateException("Failed to save " + clustername @@ -847,6 +848,13 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe } } + protected void persistInstanceDefinition(boolean overwrite, + Path appconfdir, + InstanceBuilder builder) + throws IOException, SliderException, LockAcquireFailedException { + builder.persist(appconfdir, overwrite); + } + @VisibleForTesting public static void replaceTokens(ConfTree conf, String userName, String clusterName) throws IOException { http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java index 8f679e1..d200033 100644 --- a/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java +++ b/slider-core/src/main/java/org/apache/slider/server/services/security/CertificateManager.java @@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory; import java.io.BufferedReader; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.Charset; import java.text.MessageFormat; @@ -86,6 +87,42 @@ public class CertificateManager { return certFile.exists(); } + class StreamConsumer extends Thread + { + InputStream is; + boolean logOutput; + + StreamConsumer(InputStream is, boolean logOutput) + { + this.is = is; + this.logOutput = logOutput; + } + + StreamConsumer(InputStream is) + { + this(is, false); + } + + public void run() + { + try + { + InputStreamReader isr = new InputStreamReader(is, + Charset.forName("UTF8")); + BufferedReader br = new BufferedReader(isr); + String line; + while ( (line = br.readLine()) != null) + if (logOutput) { + LOG.info(line); + } + } catch (IOException e) + { + LOG.error("Error during processing of process stream", e); + } + } + } + + /** * Runs os command * @@ -98,12 +135,13 @@ public class CertificateManager { BufferedReader br= null; try { process = Runtime.getRuntime().exec(command); - br = new BufferedReader(new InputStreamReader( - process.getInputStream(), Charset.forName("UTF8"))); + StreamConsumer outputConsumer = + new StreamConsumer(process.getInputStream(), true); + StreamConsumer errorConsumer = + new StreamConsumer(process.getErrorStream()); - while ((line = br.readLine()) != null) { - LOG.info(line); - } + outputConsumer.start(); + errorConsumer.start(); try { process.waitFor(); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json ---------------------------------------------------------------------- diff --git a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json index 89095b1..81239a2 100644 --- a/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json +++ b/slider-core/src/main/resources/org/apache/slider/providers/slideram/instance/appconf.json @@ -12,8 +12,7 @@ "components": { "slider-appmaster" : { - "jvm.heapsize": "256M", - "ssl.server.keystore.location": "/tmp/work/security/keystore.p12" + "jvm.heapsize": "256M" } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy index 6720bda..7a39035 100644 --- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentAMManagementWS.groovy @@ -22,11 +22,19 @@ import com.sun.jersey.api.client.Client import com.sun.jersey.api.client.WebResource import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import org.apache.hadoop.fs.Path +import org.apache.hadoop.yarn.exceptions.YarnException import org.apache.slider.api.StatusKeys import org.apache.slider.client.SliderClient import org.apache.slider.common.SliderKeys +import org.apache.slider.common.params.AbstractClusterBuildingActionArgs +import org.apache.slider.core.build.InstanceBuilder +import org.apache.slider.core.conf.AggregateConf import org.apache.slider.core.conf.MapOperations +import org.apache.slider.core.exceptions.SliderException +import org.apache.slider.core.launch.LaunchedApplication import org.apache.slider.core.main.ServiceLauncher +import org.apache.slider.core.persist.LockAcquireFailedException import org.apache.slider.server.appmaster.web.rest.agent.RegistrationResponse import org.apache.slider.server.appmaster.web.rest.agent.RegistrationStatus import org.apache.slider.server.services.security.CertificateManager @@ -113,55 +121,86 @@ class TestAgentAMManagementWS extends AgentTestBase { assert app_def_path.exists() assert agt_ver_path.exists() assert agt_conf_path.exists() - ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, - roles, - [ - ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath, - ARG_OPTION, APP_DEF, toURIArg(app_def_path), - ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path), - ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path), - ], - true, true, - true) - SliderClient sliderClient = launcher.service - def report = waitForClusterLive(sliderClient) - def trackingUrl = report.trackingUrl - log.info("tracking URL is $trackingUrl") - def agent_url = trackingUrl + AGENT_URI - - - def status = dumpClusterStatus(sliderClient, "agent AM") - def liveURL = status.getInfo(StatusKeys.INFO_AM_AGENT_OPS_URL) - if (liveURL) { - agent_url = liveURL + AGENT_URI + try { + setSliderClientClassName(TestSliderClient.name) + ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername, + roles, + [ + ARG_OPTION, PACKAGE_PATH, slider_core.absolutePath, + ARG_OPTION, APP_DEF, toURIArg(app_def_path), + ARG_OPTION, AGENT_CONF, toURIArg(agt_conf_path), + ARG_OPTION, AGENT_VERSION, toURIArg(agt_ver_path), + ], + true, true, + true) + SliderClient sliderClient = launcher.service + def report = waitForClusterLive(sliderClient) + def trackingUrl = report.trackingUrl + log.info("tracking URL is $trackingUrl") + def agent_url = trackingUrl + AGENT_URI + + + def status = dumpClusterStatus(sliderClient, "agent AM") + def liveURL = status.getInfo(StatusKeys.INFO_AM_AGENT_OPS_URL) + if (liveURL) { + agent_url = liveURL + AGENT_URI + } + + log.info("Agent is $agent_url") + log.info("stacks is ${liveURL}stacks") + log.info("conf is ${liveURL}conf") + + + def sleeptime = 10 + log.info "sleeping for $sleeptime seconds" + Thread.sleep(sleeptime * 1000) + + + String page = fetchWebPageWithoutError(agent_url); + log.info(page); + + //WS get + Client client = createTestClient(); + + + WebResource webResource = client.resource(agent_url + "test/register"); + RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON) + .post( + RegistrationResponse.class, + createDummyJSONRegister()); + + //TODO: assert failure as actual agent is not started. This test only starts the AM. + assert RegistrationStatus.FAILED == response.getResponseStatus(); + } finally { + setSliderClientClassName(SliderClient.name) } - log.info("Agent is $agent_url") - log.info("stacks is ${liveURL}stacks") - log.info("conf is ${liveURL}conf") - - - def sleeptime = 10 - log.info "sleeping for $sleeptime seconds" - Thread.sleep(sleeptime * 1000) - - - String page = fetchWebPageWithoutError(agent_url); - log.info(page); - - //WS get - Client client = createTestClient(); - - - WebResource webResource = client.resource(agent_url + "test/register"); - RegistrationResponse response = webResource.type(MediaType.APPLICATION_JSON) - .post( - RegistrationResponse.class, - createDummyJSONRegister()); + } - //TODO: assert failure as actual agent is not started. This test only starts the AM. - assert RegistrationStatus.FAILED == response.getResponseStatus(); - + static class TestSliderClient extends SliderClient { + @Override + protected void persistInstanceDefinition(boolean overwrite, + Path appconfdir, + InstanceBuilder builder) + throws IOException, SliderException, LockAcquireFailedException { + AggregateConf conf = builder.getInstanceDescription() + conf.getAppConfOperations().getComponent("slider-appmaster").put( + "ssl.server.keystore.location", + "/tmp/work/security/keystore.p12") + super.persistInstanceDefinition(overwrite, appconfdir, builder) + } + + @Override + LaunchedApplication launchApplication(String clustername, + Path clusterDirectory, + AggregateConf instanceDefinition, + boolean debugAM) + throws YarnException, IOException { + instanceDefinition.getAppConfOperations().getComponent("slider-appmaster").put( + "ssl.server.keystore.location", + "/tmp/work/security/keystore.p12") + return super.launchApplication(clustername, clusterDirectory, instanceDefinition, debugAM) + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/732569db/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 9266634..b6d3e17 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -66,6 +66,7 @@ import static Arguments.ARG_OPTION @Slf4j @CompileStatic class SliderTestUtils extends Assert { + static String sliderClientClassName = SliderClient.class.name public static void describe(String s) { log.info(""); @@ -480,7 +481,7 @@ class SliderTestUtils extends Assert { Configuration conf, List args) { ServiceLauncher<SliderClient> serviceLauncher = - new ServiceLauncher<SliderClient>(SliderClient.name); + new ServiceLauncher<SliderClient>(sliderClientClassName); log.debug("slider ${SliderUtils.join(args, " ", false)}") serviceLauncher.launchService(conf,
