SLIDER-710 WS/ backdoor is open everywhere; funtests verifying ping operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d2e8f88a Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d2e8f88a Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d2e8f88a Branch: refs/heads/develop Commit: d2e8f88a3fcb15aaf56801e3dce00e481c8ae0c1 Parents: 63dc3bb Author: Steve Loughran <[email protected]> Authored: Fri Jan 9 15:09:33 2015 +0000 Committer: Steve Loughran <[email protected]> Committed: Fri Jan 9 15:09:33 2015 +0000 ---------------------------------------------------------------------- .../org/apache/slider/client/SliderClient.java | 2 + .../slider/common/tools/ConfigHelper.java | 40 ++++++++++++++++++++ .../restclient/UrlConnectionOperations.java | 14 ++++--- .../slider/providers/SliderProviderFactory.java | 5 +++ .../server/appmaster/SliderAppMaster.java | 7 +++- .../appmaster/web/rest/InsecureAmFilter.java | 6 ++- .../web/rest/InsecureAmFilterInitializer.java | 5 +-- .../apache/slider/test/SliderTestUtils.groovy | 19 ++++++---- .../funtest/framework/CommandTestBase.groovy | 1 + .../funtest/lifecycle/AgentWebPagesIT.groovy | 5 ++- 10 files changed, 84 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java index 2d1af35..59d317e 100644 --- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java +++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java @@ -245,6 +245,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe config = super.bindArgs(config, args); serviceArgs = new ClientArgs(args); serviceArgs.parse(); + // add the slider XML config + ConfigHelper.injectSliderXMLResource(); // yarn-ify YarnConfiguration yarnConfiguration = new YarnConfiguration(config); return SliderUtils.patchConfiguration(yarnConfiguration); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java index c7b79f0..9db241d 100644 --- a/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java +++ b/slider-core/src/main/java/org/apache/slider/common/tools/ConfigHelper.java @@ -51,6 +51,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.atomic.AtomicBoolean; /** * Methods to aid in config, both in the Configuration class and @@ -62,6 +63,11 @@ import java.util.TreeSet; public class ConfigHelper { private static final Logger log = LoggerFactory.getLogger(ConfigHelper.class); + private static AtomicBoolean sliderResourceInjected = + new AtomicBoolean(false); + private static AtomicBoolean sliderResourceInjectionAttempted = + new AtomicBoolean(false); + /** * Dump the (sorted) configuration * @param conf config @@ -615,4 +621,38 @@ public class ConfigHelper { RegistryConstants.KEY_REGISTRY_ZK_ROOT); } + + /** + * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource + * included + * @return a configuration instance + */ + public static Configuration loadSliderConfiguration() { + Configuration conf = new Configuration(); + conf.addResource(SliderKeys.SLIDER_XML); + return conf; + } + + /** + * Inject the {@link SliderKeys#SLIDER_XML} resource + * into the configuration resources <i>of all configurations</i>. + * <p> + * This operation is idempotent. + * <p> + * If the resource is not on the classpath, downgrades, rather than + * fails. + * @return true if the resource was found and loaded. + */ + public static synchronized boolean injectSliderXMLResource() { + if (sliderResourceInjectionAttempted.getAndSet(true)) { + return sliderResourceInjected.get(); + } + URL resourceUrl = getResourceUrl(SliderKeys.SLIDER_XML); + if (resourceUrl != null) { + Configuration.addDefaultResource(SliderKeys.SLIDER_XML); + sliderResourceInjected.set(true); + } + return sliderResourceInjected.get(); + } + } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java index 4633414..fa0da5a 100644 --- a/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java +++ b/slider-core/src/main/java/org/apache/slider/core/restclient/UrlConnectionOperations.java @@ -30,7 +30,6 @@ import org.apache.hadoop.yarn.webapp.NotFoundException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.ws.rs.core.MediaType; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -149,7 +148,7 @@ public class UrlConnectionOperations extends Configured { conn.disconnect(); } } - uprateFaults(url.toString(), resultCode, body); + uprateFaults(HttpVerb.GET, url, resultCode, body); outcome.responseCode = resultCode; outcome.data = body; return outcome; @@ -159,12 +158,14 @@ public class UrlConnectionOperations extends Configured { * Uprate error codes 400 and up into faults; * 404 is converted to a {@link NotFoundException}, * 401 to {@link ForbiddenException} + * + * @param verb HTTP Verb used * @param url URL as string * @param resultCode response from the request * @param body optional body of the request * @throws IOException if the result was considered a failure */ - public static void uprateFaults(String url, + public static void uprateFaults(HttpVerb verb, URL url, int resultCode, byte[] body) throws IOException { @@ -172,11 +173,12 @@ public class UrlConnectionOperations extends Configured { //success return; } + String msg = verb.toString() +" "+ url.toString(); if (resultCode == 404) { - throw new NotFoundException(url); + throw new NotFoundException(msg); } if (resultCode == 401) { - throw new ForbiddenException(url); + throw new ForbiddenException(msg); } // all other error codes String bodyAsString; @@ -185,7 +187,7 @@ public class UrlConnectionOperations extends Configured { } else { bodyAsString = ""; } - String message = "Request to " + url + + String message = msg + " failed with exit code " + resultCode + ", body length " + bodyAsString.length() + ":\n" + bodyAsString; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java index f3bf0b1..5dd4a32 100644 --- a/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java +++ b/slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java @@ -97,6 +97,11 @@ public abstract class SliderProviderFactory extends Configured { ex); } + /** + * Load a configuration with the {@link SliderKeys#SLIDER_XML} resource + * included + * @return a configuration instance + */ public static Configuration loadSliderConfiguration() { Configuration conf = new Configuration(); conf.addResource(SliderKeys.SLIDER_XML); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java index cc9bf82..a6974b9 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java @@ -503,6 +503,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService Exception { // let the superclass process it Configuration superConf = super.bindArgs(config, args); + // add the slider XML config + ConfigHelper.injectSliderXMLResource(); + //yarn-ify YarnConfiguration yarnConfiguration = new YarnConfiguration( superConf); @@ -574,7 +577,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService /** * Create and run the cluster. - * @param clustername + * @param clustername cluster name * @return exit code * @throws Throwable on a failure */ @@ -2171,7 +2174,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService @Override // NMClientAsync.CallbackHandler public void onStartContainerError(ContainerId containerId, Throwable t) { - LOG_YARN.error("Failed to start Container " + containerId, t); + LOG_YARN.error("Failed to start Container {}", containerId, t); appState.onNodeManagerContainerStartFailed(containerId, t); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java index 07b19e7..6ee3d91 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilter.java @@ -51,6 +51,9 @@ public class InsecureAmFilter extends AmIpFilter { public void init(FilterConfig conf) throws ServletException { super.init(conf); wsContextRoot = conf.getInitParameter(WS_CONTEXT_ROOT); + if (wsContextRoot == null) { + throw new ServletException("No value set for " + WS_CONTEXT_ROOT); + } } private void rejectNonHttpRequests(ServletRequest req) throws @@ -69,7 +72,8 @@ public class InsecureAmFilter extends AmIpFilter { HttpServletResponse httpResp = (HttpServletResponse) resp; - if (!httpReq.getRequestURI().startsWith(wsContextRoot)) { + String requestURI = httpReq.getRequestURI(); + if (requestURI == null || !requestURI.startsWith(wsContextRoot)) { // hand off to the AM filter if it is not the context root super.doFilter(req, resp, chain); return; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java index 111d715..42a5bdd 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/InsecureAmFilterInitializer.java @@ -46,7 +46,7 @@ public class InsecureAmFilterInitializer extends FilterInitializer { private Configuration configuration; public static final String NAME = - "org.apache.slider.server.appmaster.web.InsecureAmFilterInitializer"; + "org.apache.slider.server.appmaster.web.rest.InsecureAmFilterInitializer"; @Override public void initFilter(FilterContainer container, Configuration conf) { @@ -59,8 +59,7 @@ public class InsecureAmFilterInitializer extends FilterInitializer { params.put(InsecureAmFilter.PROXY_URI_BASE, getHttpSchemePrefix() + proxy + getApplicationWebProxyBase()); - params.put(InsecureAmFilter.WS_CONTEXT_ROOT, - conf.get(InsecureAmFilter.WS_CONTEXT_ROOT)); + params.put(InsecureAmFilter.WS_CONTEXT_ROOT, RestPaths.WS_CONTEXT_ROOT); container.addFilter(FILTER_NAME, FILTER_CLASS, params); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index 5034ba6..fca1128 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -505,17 +505,22 @@ class SliderTestUtils extends Assert { def body = get.responseBodyAsString - uprateFaults(url, resultCode, body) + uprateFaults("GET", url, resultCode, body) return body; } /** * uprate some faults - * @param url - * @param resultCode - * @param body + * @param verb HTTP verb + * @param url URL + * @param resultCode result code + * @param body any body */ - public static void uprateFaults(String url, int resultCode, String body) { + public static void uprateFaults( + String verb, + String url, + int resultCode, + String body) { if (resultCode == 404) { throw new NotFoundException(url); @@ -524,10 +529,10 @@ class SliderTestUtils extends Assert { throw new ForbiddenException(url); } if (!(resultCode >= 200 && resultCode < 400)) { - String message = "Request to " + url + + String message = "$verb to $url " + " failed with exit code " + resultCode + ", body length " + - body?.length() + ":\n" + "body;" + body?.length() + ":\n" + body log.error(message); throw new IOException(message); } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy index 00b4062..4871ef8 100644 --- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy +++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy @@ -103,6 +103,7 @@ abstract class CommandTestBase extends SliderTestUtils { */ static { new HdfsConfiguration() + ConfigHelper.injectSliderXMLResource() ConfigHelper.registerDeprecatedConfigItems(); SLIDER_CONFIG = ConfLoader.loadSliderConf(SLIDER_CONF_XML, true); THAW_WAIT_TIME = getTimeOptionMillis(SLIDER_CONFIG, http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d2e8f88a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy ---------------------------------------------------------------------- diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy index dcfc01b..c71df8a 100644 --- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy +++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentWebPagesIT.groovy @@ -22,6 +22,7 @@ import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import org.apache.slider.agent.rest.RestTestDelegates import org.apache.slider.common.SliderExitCodes +import org.apache.slider.common.SliderXmlConfKeys import org.apache.slider.common.params.Arguments import org.apache.slider.common.params.SliderActions import org.apache.slider.funtest.framework.AgentCommandTestBase @@ -42,7 +43,6 @@ public class AgentWebPagesIT extends AgentCommandTestBase static String APP_RESOURCE2 = "../slider-core/src/test/app_packages/test_command_log/resources_no_role.json" - @Before public void prepareCluster() { setupCluster(CLUSTER) @@ -56,6 +56,9 @@ public class AgentWebPagesIT extends AgentCommandTestBase @Test public void testAgentWeb() throws Throwable { describe("Create a 0-role cluster and make web queries against it") + + // verify the ws/ path is open for all HTTP verbs + assert SLIDER_CONFIG.getBoolean(SliderXmlConfKeys.X_DEV_INSECURE_WS, false) def clusterpath = buildClusterPath(CLUSTER) File launchReportFile = createTempJsonFile(); SliderShell shell = createTemplatedSliderApplication(CLUSTER,
