SLIDER-711 verify that cache headers propagate through minimr cluster connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/515ccba5 Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/515ccba5 Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/515ccba5 Branch: refs/heads/develop Commit: 515ccba5524106e86dadb6dbab2fad71a2acbff0 Parents: ca4686d Author: Steve Loughran <[email protected]> Authored: Wed Dec 17 17:59:01 2014 +0000 Committer: Steve Loughran <[email protected]> Committed: Wed Dec 17 17:59:01 2014 +0000 ---------------------------------------------------------------------- .../server/appmaster/web/HttpCacheHeaders.java | 35 ++++++++++++ .../standalone/TestStandaloneAgentWeb.groovy | 24 ++++++--- .../org/apache/slider/test/KeysForTests.groovy | 1 + .../apache/slider/test/SliderTestUtils.groovy | 56 +++++++++++++++----- .../test/YarnZKMiniClusterTestBase.groovy | 1 + 5 files changed, 96 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java new file mode 100644 index 0000000..be8960d --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/HttpCacheHeaders.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web; + +/* + + + , ); + long now = System.currentTimeMillis(); + httpRes.addDateHeader ( "Expires", now ); + httpRes.addDateHeader ( "Date", now ); + httpRes.addHeader ( "Pragma", "no-cache" ); + */ +public interface HttpCacheHeaders { + String HTTP_HEADER_CACHE_CONTROL = "Cache-Control"; + String HTTP_HEADER_CACHE_CONTROL_NONE = "no-cache"; + String HTTP_HEADER_PRAGMA = "Pragma"; + +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy index d524698..7f8ddf5 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy @@ -23,6 +23,7 @@ import groovy.util.logging.Slf4j import org.apache.hadoop.yarn.api.records.ApplicationReport import org.apache.hadoop.yarn.conf.YarnConfiguration import org.apache.slider.agent.AgentMiniClusterTestBase + import static org.apache.slider.api.ResourceKeys.* import static org.apache.slider.api.StatusKeys.* import org.apache.slider.client.SliderClient @@ -58,6 +59,11 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { ApplicationReport report = waitForClusterLive(client) def realappmaster = report.originalTrackingUrl + + // set up url config to match + initConnectionFactory(launcher.configuration) + + execHttpRequest(30000) { GET(realappmaster) } @@ -78,14 +84,19 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { describe "Codahale operations" // now switch to the Hadoop URL connection, with SPNEGO escalation - getWebPage(conf, appmaster) - getWebPage(conf, appmaster, SYSTEM_THREADS) - getWebPage(conf, appmaster, SYSTEM_HEALTHCHECK) - getWebPage(conf, appmaster, SYSTEM_METRICS_JSON) + getWebPage(appmaster) + getWebPage(appmaster, SYSTEM_THREADS) + getWebPage(appmaster, SYSTEM_HEALTHCHECK) + getWebPage(appmaster, SYSTEM_METRICS_JSON) - log.info getWebPage(conf, realappmaster, SYSTEM_METRICS_JSON) + log.info getWebPage(realappmaster, SYSTEM_METRICS_JSON) + + // get the root page, including some checks for connectivity + getWebPage(appmaster, { + HttpURLConnection conn -> + assertConnectionNotCaching(conn) + }) - // now some REST gets describe "Application REST GETs" @@ -111,7 +122,6 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { ConfTreeSerDeser serDeser = new ConfTreeSerDeser() def json = getWebPage( - conf, appmaster, SLIDER_PATH_APPLICATION + subpath) def ctree = serDeser.fromJson(json) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy index ddd52eb..4fae2ae 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/KeysForTests.groovy @@ -34,4 +34,5 @@ public interface KeysForTests extends SliderKeys, SliderXMLConfKeysForTesting { String WAIT_TIME_ARG = WAIT_TIME.toString() String SLIDER_TEST_XML = "slider-test.xml" + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy index d3cbae0..39dfe39 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/SliderTestUtils.groovy @@ -52,10 +52,12 @@ import org.apache.slider.core.exceptions.WaitTimeoutException import org.apache.slider.core.main.ServiceLaunchException import org.apache.slider.core.main.ServiceLauncher import org.apache.slider.core.registry.docstore.PublishedConfigSet +import org.apache.slider.server.appmaster.web.HttpCacheHeaders import org.apache.slider.server.services.workflow.ForkedProcessService import org.junit.Assert import org.junit.Assume +import javax.ws.rs.core.HttpHeaders import java.util.concurrent.TimeoutException import static Arguments.ARG_OPTION @@ -531,11 +533,9 @@ class SliderTestUtils extends Assert { * @param page * @return body of response */ - public static String getWebPage(Configuration conf, - String base, - String path) { + public static String getWebPage(String base, String path) { String s = appendToURL(base, path) - return getWebPage(conf, s) + return getWebPage(s) } /** @@ -561,21 +561,33 @@ class SliderTestUtils extends Assert { throw ex; } + static URLConnectionFactory connectionFactory + + public static def initConnectionFactory(Configuration conf) { + connectionFactory = URLConnectionFactory + .newDefaultURLConnectionFactory(conf); + } + + /** * Fetches a web page asserting that the response code is between 200 and 400. * Will error on 400 and 500 series response codes and let 200 and 300 through. * * if security is enabled, this uses SPNEGO to auth - * @param page + * <p> + * Relies on {@link #initConnectionFactory(org.apache.hadoop.conf.Configuration)} + * to have been called. + * + * @param path path to page + * @param connectionChecks optional closure to run against an open connection * @return body of response */ - public static String getWebPage(Configuration conf, String page) { - assert null != page + public static String getWebPage(String path, Closure connectionChecks = null) { + assert path + assert null != connectionFactory - log.info("Fetching HTTP content at " + page); - URLConnectionFactory connectionFactory = URLConnectionFactory - .newDefaultURLConnectionFactory(conf); - URL url = new URL(page) + log.info("Fetching HTTP content at " + path); + URL url = new URL(path) assert url.port != 0 HttpURLConnection conn = null; int resultCode = 0 @@ -584,8 +596,14 @@ class SliderTestUtils extends Assert { conn = (HttpURLConnection) connectionFactory.openConnection(url); conn.instanceFollowRedirects = true; conn.connect() + resultCode = conn.responseCode + + if (connectionChecks) { + connectionChecks(conn) + } + InputStream stream = conn.errorStream; if (stream == null) { stream = conn.inputStream; @@ -597,11 +615,21 @@ class SliderTestUtils extends Assert { } finally { conn?.disconnect() } - uprateFaults(page, resultCode, body) + uprateFaults(path, resultCode, body) return body; } - - + + /** + * Assert that a connection is not caching by looking at the headers + * @param conn connection to examine + */ + public static void assertConnectionNotCaching(HttpURLConnection conn) { + assert conn.expiration <= conn.date + assert conn.getHeaderField(HttpHeaders.CACHE_CONTROL) == + HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE + assert conn.getHeaderField(HttpCacheHeaders.HTTP_HEADER_PRAGMA) == + HttpCacheHeaders.HTTP_HEADER_CACHE_CONTROL_NONE + } /** * Assert that a service operation succeeded http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/515ccba5/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy index f6b13a4..e907209 100644 --- a/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnZKMiniClusterTestBase.groovy @@ -103,6 +103,7 @@ public abstract class YarnZKMiniClusterTestBase extends YarnMiniClusterTestBase createMicroZKCluster("-${name?:methodName.methodName}", conf) conf.setBoolean(RegistryConstants.KEY_REGISTRY_ENABLED, true) conf.set(RegistryConstants.KEY_REGISTRY_ZK_QUORUM, ZKBinding) + //now create the cluster name = super.createMiniCluster(name, conf, noOfNodeManagers, numLocalDirs, numLogDirs, startHDFS)
