Repository: ambari Updated Branches: refs/heads/trunk 3dd9ae471 -> 98405ef9b
AMBARI-11850. Blueprint export process is very slow in multi-node clusters. (rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/98405ef9 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/98405ef9 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/98405ef9 Branch: refs/heads/trunk Commit: 98405ef9bee07b974baeaa13cb143aeea6aeadfd Parents: 3dd9ae4 Author: Bob Nettleton <[email protected]> Authored: Thu Jun 11 09:19:52 2015 -0400 Committer: Bob Nettleton <[email protected]> Committed: Thu Jun 11 09:20:30 2015 -0400 ---------------------------------------------------------------------- .../ambari/server/api/query/QueryImpl.java | 11 +- .../server/api/query/render/BaseRenderer.java | 6 + .../query/render/ClusterBlueprintRenderer.java | 9 + .../server/api/query/render/Renderer.java | 10 ++ .../ambari/server/api/query/QueryImplTest.java | 163 +++++++++++++++++++ .../render/ClusterBlueprintRendererTest.java | 10 ++ .../api/query/render/DefaultRendererTest.java | 9 + 7 files changed, 216 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java index 2319683..10d9bef 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java @@ -403,7 +403,10 @@ public class QueryImpl implements Query, ResourceInstance { queryResults.put(null, new QueryResult( request, queryPredicate, userPredicate, getKeyValueMap(), queryResponse)); - clusterController.populateResources(resourceType, providerResourceSet, request, queryPredicate); + if (renderer.requiresPropertyProviderInput()) { + clusterController.populateResources(resourceType, providerResourceSet, request, queryPredicate); + } + queryForSubResources(); } @@ -445,7 +448,11 @@ public class QueryImpl implements Query, ResourceInstance { new QueryResult(request, queryPredicate, subResourcePredicate, map, new QueryResponseImpl(resourceSet))); } } - clusterController.populateResources(resourceType, providerResourceSet, request, subResourcePredicate); + + if (renderer.requiresPropertyProviderInput()) { + clusterController.populateResources(resourceType, providerResourceSet, request, subResourcePredicate); + } + subResource.queryForSubResources(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java index 7866aa4..03bce14 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/BaseRenderer.java @@ -46,6 +46,12 @@ public abstract class BaseRenderer implements Renderer { m_schemaFactory = schemaFactory; } + @Override + public boolean requiresPropertyProviderInput() { + // most renderers require the property provider input support + return true; + } + /** * Obtain a schema instance based on resource type. * http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java index cfc9bc0..29c3040 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRenderer.java @@ -134,6 +134,15 @@ public class ClusterBlueprintRenderer extends BaseRenderer implements Renderer { return new BlueprintPostProcessor(request); } + @Override + public boolean requiresPropertyProviderInput() { + // the Blueprint-based renderer does not require property provider input + // this method will help to filter out the un-necessary calls to the AMS + // and Alerts Property providers, since they are not included in the + // exported Blueprint + return false; + } + // ----- private instance methods ------------------------------------------ /** http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java index f353d53..2ba245c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/api/query/render/Renderer.java @@ -80,4 +80,14 @@ public interface Renderer { * @return associated post processor */ public ResultPostProcessor getResultPostProcessor(Request request); + + + /** + * Obtains the property provider requirements of the given + * renderer implementation. + * + * @return true if property provider support is required + * false if property provider support is not required + */ + public boolean requiresPropertyProviderInput(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java index 01361d2..3bdbb87 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java @@ -27,6 +27,7 @@ import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.createNiceMock; import static org.easymock.EasyMock.eq; import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.isA; import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertFalse; @@ -60,13 +61,16 @@ import org.apache.ambari.server.controller.spi.NoSuchResourceException; import org.apache.ambari.server.controller.spi.PageRequest; import org.apache.ambari.server.controller.spi.Predicate; import org.apache.ambari.server.controller.spi.QueryResponse; +import org.apache.ambari.server.controller.spi.Request; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.spi.Schema; +import org.apache.ambari.server.controller.spi.SchemaFactory; import org.apache.ambari.server.controller.spi.SortRequest; import org.apache.ambari.server.controller.spi.SystemException; import org.apache.ambari.server.controller.spi.UnsupportedPropertyException; import org.apache.ambari.server.controller.utilities.PredicateBuilder; import org.easymock.Capture; +import org.easymock.EasyMockSupport; import org.junit.Assert; import org.junit.Test; @@ -842,6 +846,165 @@ public class QueryImplTest { Assert.assertNotNull(hostNode.getObject().getPropertyValue("c1/p3")); } + @Test + public void testExecute_RendererDoesNotRequirePropertyProviderInput() throws Exception { + EasyMockSupport mockSupport = new EasyMockSupport(); + + ResourceDefinition mockResourceDefinition = + mockSupport.createMock(ResourceDefinition.class); + + SubResourceDefinition mockSubResourceDefinition = + mockSupport.createMock(SubResourceDefinition.class); + + // need to use a default mock here, to verify + // that certain methods are called or not called + ClusterController mockClusterController = + mockSupport.createMock(ClusterController.class); + + Renderer mockRenderer = + mockSupport.createMock(Renderer.class); + + QueryResponse mockQueryResponse = + mockSupport.createMock(QueryResponse.class); + + QueryResponse mockSubQueryResponse = + mockSupport.createMock(QueryResponse.class); + + Resource mockResource = + mockSupport.createMock(Resource.class); + + Schema mockSchema = + mockSupport.createMock(Schema.class); + + expect(mockResourceDefinition.getType()).andReturn(Resource.Type.Host).atLeastOnce(); + expect(mockResourceDefinition.getSubResourceDefinitions()).andReturn(Collections.singleton(mockSubResourceDefinition)).atLeastOnce(); + + expect(mockSubResourceDefinition.getType()).andReturn(Resource.Type.Configuration).atLeastOnce(); + expect(mockSubResourceDefinition.isCollection()).andReturn(false).atLeastOnce(); + + expect(mockSchema.getKeyPropertyId(isA(Resource.Type.class))).andReturn("test-value").anyTimes(); + expect(mockSchema.getKeyTypes()).andReturn(Collections.<Resource.Type>emptySet()).anyTimes(); + + mockRenderer.init(isA(SchemaFactory.class)); + // the mock renderer should return false for requiresPropertyProviderInput, to + // simulate the case of a renderer that does not need the property providers to execute + // this method should be called twice: once each resource (1 resource, 1 sub-resource in this test) + expect(mockRenderer.requiresPropertyProviderInput()).andReturn(false).times(2); + expect(mockRenderer.finalizeProperties(isA(TreeNode.class), eq(true))).andReturn(new TreeNodeImpl<Set<String>>(null, Collections.<String>emptySet(), "test-node")); + expect(mockRenderer.finalizeResult(isA(Result.class))).andReturn(null); + + // note: the expectations for the ClusterController mock are significant to this test. + // the ClusterController.populateResources() method should not be called, and so + // is not expected here. + expect(mockClusterController.getSchema(Resource.Type.Host)).andReturn(mockSchema).anyTimes(); + expect(mockClusterController.getSchema(Resource.Type.Configuration)).andReturn(mockSchema).anyTimes(); + expect(mockClusterController.getResources(eq(Resource.Type.Host), isA(Request.class), (Predicate)eq(null))).andReturn(mockQueryResponse).atLeastOnce(); + expect(mockClusterController.getResources(eq(Resource.Type.Configuration), isA(Request.class), (Predicate)eq(null))).andReturn(mockSubQueryResponse).atLeastOnce(); + expect(mockClusterController.getIterable(eq(Resource.Type.Host), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + expect(mockClusterController.getIterable(eq(Resource.Type.Configuration), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + + expect(mockQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + expect(mockSubQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + + expect(mockResource.getType()).andReturn(Resource.Type.Host).atLeastOnce(); + + Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>(); + + mockSupport.replayAll(); + + QueryImpl instance = new QueryImpl(mapIds, mockResourceDefinition, mockClusterController); + instance.setRenderer(mockRenderer); + + // call these methods to setup sub resources + instance.ensureSubResources(); + instance.addProperty("*", null); + + instance.execute(); + + mockSupport.verifyAll(); + + } + + @Test + public void testExecute_RendererRequiresPropertyProviderInput() throws Exception { + EasyMockSupport mockSupport = new EasyMockSupport(); + + ResourceDefinition mockResourceDefinition = + mockSupport.createMock(ResourceDefinition.class); + + SubResourceDefinition mockSubResourceDefinition = + mockSupport.createMock(SubResourceDefinition.class); + + // need to use a default mock here, to verify + // that certain methods are called or not called + ClusterController mockClusterController = + mockSupport.createMock(ClusterController.class); + + Renderer mockRenderer = + mockSupport.createMock(Renderer.class); + + QueryResponse mockQueryResponse = + mockSupport.createMock(QueryResponse.class); + + QueryResponse mockSubQueryResponse = + mockSupport.createMock(QueryResponse.class); + + Resource mockResource = + mockSupport.createMock(Resource.class); + + Schema mockSchema = + mockSupport.createMock(Schema.class); + + expect(mockResourceDefinition.getType()).andReturn(Resource.Type.Host).atLeastOnce(); + expect(mockResourceDefinition.getSubResourceDefinitions()).andReturn(Collections.singleton(mockSubResourceDefinition)).atLeastOnce(); + + expect(mockSubResourceDefinition.getType()).andReturn(Resource.Type.Configuration).atLeastOnce(); + expect(mockSubResourceDefinition.isCollection()).andReturn(false).atLeastOnce(); + + expect(mockSchema.getKeyPropertyId(isA(Resource.Type.class))).andReturn("test-value").anyTimes(); + expect(mockSchema.getKeyTypes()).andReturn(Collections.<Resource.Type>emptySet()).anyTimes(); + + mockRenderer.init(isA(SchemaFactory.class)); + // simulate the case of a renderer that requires the property providers to execute + // this method should be called twice: once for each resource (1 resource, 1 sub-resource in this test) + expect(mockRenderer.requiresPropertyProviderInput()).andReturn(true).times(2); + expect(mockRenderer.finalizeProperties(isA(TreeNode.class), eq(true))).andReturn(new TreeNodeImpl<Set<String>>(null, Collections.<String>emptySet(), "test-node")); + expect(mockRenderer.finalizeResult(isA(Result.class))).andReturn(null); + + // note: the expectations for the ClusterController mock are significant to this test. + expect(mockClusterController.getSchema(Resource.Type.Host)).andReturn(mockSchema).anyTimes(); + expect(mockClusterController.getSchema(Resource.Type.Configuration)).andReturn(mockSchema).anyTimes(); + expect(mockClusterController.getResources(eq(Resource.Type.Host), isA(Request.class), (Predicate)eq(null))).andReturn(mockQueryResponse).atLeastOnce(); + expect(mockClusterController.getResources(eq(Resource.Type.Configuration), isA(Request.class), (Predicate)eq(null))).andReturn(mockSubQueryResponse).atLeastOnce(); + expect(mockClusterController.getIterable(eq(Resource.Type.Host), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + expect(mockClusterController.getIterable(eq(Resource.Type.Configuration), isA(QueryResponse.class), isA(Request.class),(Predicate)eq(null), (PageRequest)eq(null), (SortRequest)eq(null))).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + // expect call to activate property providers for Host resource + expect(mockClusterController.populateResources(eq(Resource.Type.Host), eq(Collections.singleton(mockResource)), isA(Request.class), (Predicate)eq(null))).andReturn(Collections.<Resource>emptySet()).times(1); + // expect call to activate property providers for Configuration sub-resource + expect(mockClusterController.populateResources(eq(Resource.Type.Configuration), eq(Collections.singleton(mockResource)), isA(Request.class), (Predicate)eq(null))).andReturn(Collections.<Resource>emptySet()).times(1); + + expect(mockQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + expect(mockSubQueryResponse.getResources()).andReturn(Collections.singleton(mockResource)).atLeastOnce(); + + expect(mockResource.getType()).andReturn(Resource.Type.Host).atLeastOnce(); + + Map<Resource.Type, String> mapIds = new HashMap<Resource.Type, String>(); + + mockSupport.replayAll(); + + QueryImpl instance = new QueryImpl(mapIds, mockResourceDefinition, mockClusterController); + instance.setRenderer(mockRenderer); + + // call these methods to setup sub resources + instance.ensureSubResources(); + instance.addProperty("*", null); + + instance.execute(); + + mockSupport.verifyAll(); + + } + public static class TestQuery extends QueryImpl { public TestQuery(Map<Resource.Type, String> mapIds, ResourceDefinition resourceDefinition) { super(mapIds, resourceDefinition, new ClusterControllerImpl(new ClusterControllerImplTest.TestProviderModule())); http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java index 96abb8c..c0fe07e 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/ClusterBlueprintRendererTest.java @@ -58,6 +58,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; /** * ClusterBlueprintRenderer unit tests. @@ -375,6 +376,15 @@ public class ClusterBlueprintRendererTest { } + @Test + public void testClusterRendererDefaults() throws Exception { + Renderer clusterBlueprintRenderer = + new ClusterBlueprintRenderer(); + + assertFalse("ClusterBlueprintRenderer should not require property provider input", + clusterBlueprintRenderer.requiresPropertyProviderInput()); + } + //todo: collection resource private void createClusterResultTree(TreeNode<Resource> resultTree) throws Exception { http://git-wip-us.apache.org/repos/asf/ambari/blob/98405ef9/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java index 4981a67..78efa1c 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/api/query/render/DefaultRendererTest.java @@ -341,4 +341,13 @@ public class DefaultRendererTest { assertSame(result, renderer.finalizeResult(result)); } + + @Test + public void testRequiresInputDefault() throws Exception { + Renderer defaultRenderer = + new DefaultRenderer(); + + assertTrue("Default renderer for cluster resources must require property provider input", + defaultRenderer.requiresPropertyProviderInput()); + } }
