SLIDER-711 container serialization/refresh testing
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8ad2bfbf Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8ad2bfbf Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8ad2bfbf Branch: refs/heads/develop Commit: 8ad2bfbfda1aff91efd630671acfdff6accfa46f Parents: 7454e0b Author: Steve Loughran <[email protected]> Authored: Thu Dec 18 12:13:47 2014 +0000 Committer: Steve Loughran <[email protected]> Committed: Thu Dec 18 12:13:47 2014 +0000 ---------------------------------------------------------------------- .../types/SerializedContainerInformation.java | 56 ++++++++ .../server/appmaster/state/RoleInstance.java | 12 +- .../server/appmaster/web/rest/RestPaths.java | 1 + .../rest/application/ApplicationResource.java | 10 ++ .../application/resources/CachedContent.java | 45 +++++-- .../resources/ContainerListRefresher.java | 49 +++++++ .../application/resources/ContentCache.java | 7 +- .../standalone/TestStandaloneAgentWeb.groovy | 2 +- ...estMockAppStateAppResourceIntegration.groovy | 72 ---------- .../TestMockAppStateAppRestIntegration.groovy | 132 +++++++++++++++++++ 10 files changed, 298 insertions(+), 88 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java new file mode 100644 index 0000000..dfb70f5 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/api/types/SerializedContainerInformation.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.api.types; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; +import org.codehaus.jackson.map.annotate.JsonSerialize; + + +/** + * Serializable version of role instance data + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +public class SerializedContainerInformation { + + public String containerId; + public String component; + public Boolean released; + public int state; + public Integer exitCode; + public String diagnostics; + public long createTime; + public long startTime; + + /** + * What is the tail output from the executed process (or [] if not started + * or the log cannot be picked up + */ + public String[] output; + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("SerializedContainerInformation{"); + sb.append("containerId='").append(containerId).append('\''); + sb.append(", component='").append(component).append('\''); + sb.append('}'); + return sb.toString(); + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java index c8ddc6f..1488fb7 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java @@ -27,6 +27,7 @@ import org.apache.hadoop.registry.client.types.Endpoint; import org.apache.hadoop.registry.client.types.ProtocolTypes; import org.apache.slider.api.ClusterDescription; import org.apache.slider.api.proto.Messages; +import org.apache.slider.api.types.SerializedContainerInformation; import org.apache.slider.common.tools.SliderUtils; import java.util.ArrayList; @@ -40,8 +41,7 @@ public final class RoleInstance implements Cloneable { public Container container; /** - * UUID of container used in Slider RPC to refer to instances. - * The string value of the container ID is used here. + * Container ID */ public final String id; public long createTime; @@ -233,5 +233,11 @@ public final class RoleInstance implements Cloneable { ProtocolTypes.PROTOCOL_TCP, host, port); addEndpoint(epr); } - + + public SerializedContainerInformation serialize() { + SerializedContainerInformation info = new SerializedContainerInformation(); + info.containerId = id; + + return info; + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java index 5dbc090..28c0fab 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/RestPaths.java @@ -131,6 +131,7 @@ public class RestPaths { public static final String APPLICATION_WADL = "/application.wadl"; public static final String LIVE_RESOURCES = "/live/resources"; + public static final String LIVE_CONTAINERS = "/live/containers"; public static final String MODEL_DESIRED = "/model/desired"; public static final String MODEL_DESIRED_APPCONF = MODEL_DESIRED +"/appconf"; public static final String MODEL_DESIRED_RESOURCES = MODEL_DESIRED +"/resources"; http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java index b84717d..374e8de 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/ApplicationResource.java @@ -96,5 +96,15 @@ public class ApplicationResource extends AbstractSliderResource { throw buildException(RestPaths.LIVE_RESOURCES, e); } } + @GET + @Path(RestPaths.LIVE_CONTAINERS) + @Produces({MediaType.APPLICATION_JSON}) + public Object getLiveContainers() { + try { + return cache.get(RestPaths.LIVE_RESOURCES).get(); + } catch (Exception e) { + throw buildException(RestPaths.LIVE_RESOURCES, e); + } + } } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java index c11a3ee..78e65e8 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/CachedContent.java @@ -20,18 +20,24 @@ package org.apache.slider.server.appmaster.web.rest.application.resources; import com.google.common.base.Preconditions; import org.apache.hadoop.util.Time; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * A cached resource is one that can be stored and served up, with a refresh - * only taking place when the expiry happens - * @param <T> + * only taking place when the expiry happens. + * + * The refresh check/refresh is synchronized. + * @param <T> type to return */ public class CachedContent<T> { - + private static final Logger log = + LoggerFactory.getLogger(CachedContent.class); private T cachedValue; private long expires; private final long lifespan; private final ResourceRefresher<T> refresh; + private int refreshCounter; public CachedContent(long lifespan, ResourceRefresher<T> refresh) { @@ -45,17 +51,21 @@ public class CachedContent<T> { */ public T get() { maybeRefresh(); - return cachedValue; + return getCachedValue(); } /** * Get the cached value without any expiry check * @return the last value set. May be null. */ - public T getCachedValue() { + public synchronized T getCachedValue() { return cachedValue; } + public synchronized int getRefreshCounter() { + return refreshCounter; + } + /** * Get the lifespan in millis of the cached value * @return the lifespan @@ -68,10 +78,12 @@ public class CachedContent<T> { * Maybe refresh the content * @return true if a refresh took place. */ - public boolean maybeRefresh() { + public synchronized boolean maybeRefresh() { long now = now(); - if (cachedValue == null || now > expires) { + if (cachedValue == null || now >= expires) { + log.debug("Refreshing at time {}", now); forceRefresh(); + log.debug("Refreshed value now {}", cachedValue); return true; } return false; @@ -85,12 +97,25 @@ public class CachedContent<T> { * Force a refresh and reset the expiry counter * @return the new value */ - public T forceRefresh() { + protected synchronized T forceRefresh() { + refreshCounter ++; T updated = refresh.refresh(); Preconditions.checkNotNull(updated); cachedValue = updated; expires = now() + lifespan; - return updated; + return cachedValue; + } + + @Override + public String toString() { + final StringBuilder sb = + new StringBuilder("CachedContent{"); + sb.append(" expires=").append(expires); + sb.append(", lifespan=").append(lifespan); + sb.append(", refresh=").append(refresh); + sb.append(", refreshCounter=").append(refreshCounter); + sb.append(", cached=").append(cachedValue); + sb.append('}'); + return sb.toString(); } - } http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java new file mode 100644 index 0000000..7a48509 --- /dev/null +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContainerListRefresher.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.web.rest.application.resources; + +import org.apache.slider.api.types.SerializedContainerInformation; +import org.apache.slider.core.conf.ConfTree; +import org.apache.slider.server.appmaster.state.RoleInstance; +import org.apache.slider.server.appmaster.state.StateAccessForProviders; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class ContainerListRefresher implements ResourceRefresher<Map<String, SerializedContainerInformation>> { + + private final StateAccessForProviders state; + + public ContainerListRefresher(StateAccessForProviders state) { + this.state = state; + } + + @Override + public Map<String, SerializedContainerInformation> refresh() { + List<RoleInstance> containerList = state.cloneOwnedContainerList(); + + Map<String, SerializedContainerInformation> map = new HashMap<String, SerializedContainerInformation>(); + for (RoleInstance instance : containerList) { + SerializedContainerInformation serialized = instance.serialize(); + map.put(serialized.containerId, serialized); + } + return map; + } +} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java ---------------------------------------------------------------------- diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java index 169eaa3..f309570 100644 --- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java +++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/application/resources/ContentCache.java @@ -18,9 +18,12 @@ package org.apache.slider.server.appmaster.web.rest.application.resources; -import java.util.HashMap; +import java.util.concurrent.ConcurrentHashMap; -public class ContentCache extends HashMap<String, CachedContent> { +/** + * Cache of content + */ +public class ContentCache extends ConcurrentHashMap<String, CachedContent> { public ContentCache(int initialCapacity) { super(initialCapacity); http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy index 7f8ddf5..cbb340f 100644 --- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy +++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestStandaloneAgentWeb.groovy @@ -91,7 +91,7 @@ class TestStandaloneAgentWeb extends AgentMiniClusterTestBase { log.info getWebPage(realappmaster, SYSTEM_METRICS_JSON) - // get the root page, including some checks for connectivity + // get the root page, including some checks for cache disabled getWebPage(appmaster, { HttpURLConnection conn -> assertConnectionNotCaching(conn) http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy deleted file mode 100644 index 2d0b8bf..0000000 --- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppResourceIntegration.groovy +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.slider.server.appmaster.model.appstate - -import groovy.util.logging.Slf4j -import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest -import org.apache.slider.server.appmaster.model.mock.MockRoles -import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent -import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache -import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher -import org.junit.Test - -@Slf4j -class TestMockAppStateAppResourceIntegration extends BaseMockAppStateTest implements MockRoles { - - @Override - String getTestName() { - return "TestMockAppStateAppResourceIntegration" - } - - @Test - public void testCachedDocument() throws Throwable { - ContentCache cache = new ContentCache() - - def content = new CachedContentManagedTimer(new IntRefresher()) - cache.put("/int", content) - def content1 = cache.get("/int") - assert content.equals(content1) - - assert 0 == content.get() - assert 0 == content.getCachedValue() - - } - - class IntRefresher implements ResourceRefresher<Integer> { - int count ; - @Override - Integer refresh() { - return count++; - } - } - - class CachedContentManagedTimer extends CachedContent { - int time = 0; - - @Override - protected long now() { - return time++; - } - - CachedContentManagedTimer(ResourceRefresher refresh) { - super(1, refresh) - } - - } -} http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8ad2bfbf/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy ---------------------------------------------------------------------- diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy new file mode 100644 index 0000000..aefeecf --- /dev/null +++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAppRestIntegration.groovy @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.slider.server.appmaster.model.appstate + +import groovy.util.logging.Slf4j +import org.apache.slider.api.types.SerializedContainerInformation +import org.apache.slider.core.persist.JsonSerDeser +import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest +import org.apache.slider.server.appmaster.model.mock.MockRoles +import org.apache.slider.server.appmaster.state.ProviderAppState +import org.apache.slider.server.appmaster.state.RoleInstance +import org.apache.slider.server.appmaster.state.StateAccessForProviders +import org.apache.slider.server.appmaster.web.rest.application.resources.CachedContent +import org.apache.slider.server.appmaster.web.rest.application.resources.ContainerListRefresher +import org.apache.slider.server.appmaster.web.rest.application.resources.ContentCache +import org.apache.slider.server.appmaster.web.rest.application.resources.ResourceRefresher +import org.junit.Test + +@Slf4j +class TestMockAppStateAppRestIntegration extends BaseMockAppStateTest implements MockRoles { + + @Override + String getTestName() { + return "TestMockAppStateAppRestIntegration" + } + + @Test + public void testCachedIntDocument() throws Throwable { + ContentCache cache = new ContentCache() + + + def refresher = new IntRefresher() + assert 0 == refresher.count + def entry = new CachedContentManagedTimer(refresher) + cache.put("/int", entry) + def content1 = cache.get("/int") + assert entry.equals(content1) + + assert 0 == entry.get() + assert 1 == refresher.count + assert 0 == entry.cachedValue + assert entry.refreshCounter == 1 + + def got = entry.get() + assert entry.refreshCounter == 2 + assert 1 == got; + } + + @Test + public void testContainerListRefresher() throws Throwable { + int r0 = 1 + int r1 = 2 + int r2 = 3 + role0Status.desired = r0 + role1Status.desired = r1 + role2Status.desired = r2 + ContainerListRefresher clr = new ContainerListRefresher(stateAccess) + def map = clr.refresh() + assert map.size() == 0 + List<RoleInstance> instances = createAndStartNodes() + map = clr.refresh() + assert instances.size() == r0 + r1 + r2 + assert map.size() == instances.size() + log.info("$map") + JsonSerDeser<SerializedContainerInformation> serDeser = + new JsonSerDeser<>(SerializedContainerInformation) + map.each { key, value -> + log.info("$key -> ${serDeser.toJson(value)}") + } + } + + public ProviderAppState getStateAccess() { + StateAccessForProviders state = new ProviderAppState("name", appState) + return state + } + + /** + * Little class to do integer refreshing & so test refresh logic + */ + class IntRefresher implements ResourceRefresher<Integer> { + int count ; + @Override + Integer refresh() { + log.info("Refresh at $count") + def result = count + count += 1; + return result; + } + + @Override + String toString() { + return "IntRefresher at " + count; + } + + } + + class CachedContentManagedTimer extends CachedContent { + int time = 0; + + @Override + protected long now() { + return time++; + } + + CachedContentManagedTimer(ResourceRefresher refresh) { + super(1, refresh) + } + + @Override + String toString() { + return "CachedContentManagedTimer at " + time + "; " + super.toString(); + } + } + + +}
