Repository: ambari Updated Branches: refs/heads/trunk bf6147931 -> 200778b42
AMBARI-17758. LogSearch Integration NullPointerException when LogSearch connection not available. (rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/200778b4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/200778b4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/200778b4 Branch: refs/heads/trunk Commit: 200778b425563de6e4b5a88e7fb2614dbae8178d Parents: bf61479 Author: Bob Nettleton <[email protected]> Authored: Sat Jul 16 20:52:28 2016 -0400 Committer: Bob Nettleton <[email protected]> Committed: Sat Jul 16 20:52:28 2016 -0400 ---------------------------------------------------------------------- .../server/controller/ControllerModule.java | 5 + .../logging/LogSearchDataRetrievalService.java | 30 +++- .../logging/LoggingSearchPropertyProvider.java | 6 +- .../LogSearchDataRetrievalServiceTest.java | 105 +++++++++++++ .../LoggingSearchPropertyProviderTest.java | 153 +++++++++++++++++++ 5 files changed, 289 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/200778b4/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java index b4237a91..2bd7eff 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/ControllerModule.java @@ -55,6 +55,8 @@ import org.apache.ambari.server.controller.internal.MemberResourceProvider; import org.apache.ambari.server.controller.internal.RepositoryVersionResourceProvider; import org.apache.ambari.server.controller.internal.ServiceResourceProvider; import org.apache.ambari.server.controller.internal.UpgradeResourceProvider; +import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactory; +import org.apache.ambari.server.controller.logging.LoggingRequestHelperFactoryImpl; import org.apache.ambari.server.controller.metrics.MetricPropertyProviderFactory; import org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricCacheEntryFactory; import org.apache.ambari.server.controller.metrics.timeline.cache.TimelineMetricCacheProvider; @@ -367,6 +369,9 @@ public class ControllerModule extends AbstractModule { bind(AuthenticationEntryPoint.class).to(AmbariEntryPoint.class).in(Scopes.SINGLETON); + // factory to create LoggingRequestHelper instances for LogSearch integration + bind(LoggingRequestHelperFactory.class).to(LoggingRequestHelperFactoryImpl.class); + requestStaticInjection(DatabaseConsistencyCheckHelper.class); requestStaticInjection(KerberosChecker.class); requestStaticInjection(AuthorizationHelper.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/200778b4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java index 877f4e3..4929747 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java @@ -65,6 +65,9 @@ public class LogSearchDataRetrievalService extends AbstractService { @Inject private Configuration configuration; + @Inject + private LoggingRequestHelperFactory loggingRequestHelperFactory; + /** * A Cache of host+component names to a set of log files associated with * that Host/Component combination. This data is retrieved from the @@ -154,20 +157,31 @@ public class LogSearchDataRetrievalService extends AbstractService { return result; } else { // create URI and add to cache before returning - LoggingRequestHelper helper = - new LoggingRequestHelperFactoryImpl().getHelper(getController(), cluster); - String tailFileURI = - helper.createLogFileTailURI(baseURI, component, host); - - if (tailFileURI != null) { - logFileTailURICache.put(key, tailFileURI); - return tailFileURI; + if (loggingRequestHelperFactory != null) { + LoggingRequestHelper helper = + loggingRequestHelperFactory.getHelper(getController(), cluster); + + if (helper != null) { + String tailFileURI = + helper.createLogFileTailURI(baseURI, component, host); + + if (tailFileURI != null) { + logFileTailURICache.put(key, tailFileURI); + return tailFileURI; + } + } + } else { + LOG.debug("LoggingRequestHelperFactory not set on the retrieval service, this probably indicates an error in setup of this service."); } } return null; } + protected void setLoggingRequestHelperFactory(LoggingRequestHelperFactory loggingRequestHelperFactory) { + this.loggingRequestHelperFactory = loggingRequestHelperFactory; + } + private void startLogSearchFileNameRequest(String host, String component, String cluster) { executor.execute(new LogSearchFileNameRequestRunnable(host, component, cluster)); } http://git-wip-us.apache.org/repos/asf/ambari/blob/200778b4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java index 32690e8..f496d32 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProvider.java @@ -92,8 +92,10 @@ public class LoggingSearchPropertyProvider implements PropertyProvider { // generate the URIs that can be used by clients to obtain search results/tail log results/etc final String searchEngineURI = ambariManagementController.getAmbariServerURI(getFullPathToSearchEngine(clusterName)); final String logFileTailURI = logSearchDataRetrievalService.getLogFileTailURI(searchEngineURI, mappedComponentNameForLogSearch, hostName, clusterName); - // all log files are assumed to be service types for now - listOfFileDefinitions.add(new LogFileDefinitionInfo(fileName, LogFileType.SERVICE, searchEngineURI, logFileTailURI)); + if (logFileTailURI != null) { + // all log files are assumed to be service types for now + listOfFileDefinitions.add(new LogFileDefinitionInfo(fileName, LogFileType.SERVICE, searchEngineURI, logFileTailURI)); + } } loggingInfo.setListOfLogFileDefinitions(listOfFileDefinitions); http://git-wip-us.apache.org/repos/asf/ambari/blob/200778b4/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java new file mode 100644 index 0000000..97c59f8 --- /dev/null +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.ambari.server.controller.logging; + +import org.easymock.EasyMockSupport; +import org.junit.Test; + +import static org.easymock.EasyMock.expect; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertEquals; + +/** + * This test verifies the basic behavior of the + * LogSearchDataRetrievalServiceTest, and should + * verify the interaction with its dependencies, as + * well as the interaction with the LogSearch + * server. + * + */ +public class LogSearchDataRetrievalServiceTest { + + @Test + public void testGetTailFileWhenHelperIsAvailable() throws Exception { + final String expectedHostName = "c6401.ambari.apache.org"; + final String expectedComponentName = "DATANODE"; + final String expectedClusterName = "clusterone"; + final String expectedResultURI = "http://localhost/test/result";; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + LoggingRequestHelperFactory helperFactoryMock = + mockSupport.createMock(LoggingRequestHelperFactory.class); + + LoggingRequestHelper helperMock = + mockSupport.createMock(LoggingRequestHelper.class); + + expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(helperMock); + expect(helperMock.createLogFileTailURI("http://localhost";, expectedComponentName, expectedHostName)).andReturn(expectedResultURI); + + mockSupport.replayAll(); + + LogSearchDataRetrievalService retrievalService = + new LogSearchDataRetrievalService(); + retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + // call the initialization routine called by the Google framework + retrievalService.doStart(); + + String resultTailFileURI = + retrievalService.getLogFileTailURI("http://localhost";, expectedComponentName, expectedHostName, expectedClusterName); + + assertEquals("TailFileURI was not returned as expected", + expectedResultURI, resultTailFileURI); + + mockSupport.verifyAll(); + } + + @Test + public void testGetTailFileWhenRequestHelperIsNull() throws Exception { + final String expectedHostName = "c6401.ambari.apache.org"; + final String expectedComponentName = "DATANODE"; + final String expectedClusterName = "clusterone"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + LoggingRequestHelperFactory helperFactoryMock = + mockSupport.createMock(LoggingRequestHelperFactory.class); + + // return null, to simulate the case where LogSearch Server is + // not available for some reason + expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(null); + + mockSupport.replayAll(); + + LogSearchDataRetrievalService retrievalService = + new LogSearchDataRetrievalService(); + retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + // call the initialization routine called by the Google framework + retrievalService.doStart(); + + String resultTailFileURI = + retrievalService.getLogFileTailURI("http://localhost";, expectedComponentName, expectedHostName, expectedClusterName); + + assertNull("TailFileURI should be null in this case", + resultTailFileURI); + + mockSupport.verifyAll(); + + } + +} http://git-wip-us.apache.org/repos/asf/ambari/blob/200778b4/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProviderTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProviderTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProviderTest.java index 30b32d8..af0c7df 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProviderTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LoggingSearchPropertyProviderTest.java @@ -218,6 +218,159 @@ public class LoggingSearchPropertyProviderTest { } /** + * Verifies the following: + * + * 1. That this PropertyProvider implementation uses + * the expected interfaces to make queries to the LogSearch + * service. + * 2. That the PropertyProvider queries the current HostComponent + * resource in order to obtain the correct information to send to + * LogSearch. + * 3. That the output of the LogSearch query is properly set on the + * HostComponent resource in the expected structure. + * 4. That the proper error-handling is in place in the event that a null + * tail log URI is returned by the retrieval service. + * + * + * @throws Exception + */ + @Test + public void testBasicCallWithNullTailLogURIReturned() throws Exception { + final String expectedLogFilePath = + "/var/log/hdfs/hdfs_namenode.log"; + + final String expectedSearchEnginePath = "/api/v1/clusters/clusterone/logging/searchEngine"; + + final String expectedAmbariURL = "http://c6401.ambari.apache.org:8080";; + + final String expectedStackName = "HDP"; + final String expectedStackVersion = "2.4"; + final String expectedComponentName = "NAMENODE"; + final String expectedServiceName = "HDFS"; + final String expectedLogSearchComponentName = "hdfs_namenode"; + + EasyMockSupport mockSupport = new EasyMockSupport(); + + Resource resourceMock = + mockSupport.createMock(Resource.class); + expect(resourceMock.getPropertyValue(PropertyHelper.getPropertyId("HostRoles", "component_name"))).andReturn(expectedComponentName).atLeastOnce(); + expect(resourceMock.getPropertyValue(PropertyHelper.getPropertyId("HostRoles", "host_name"))).andReturn("c6401.ambari.apache.org").atLeastOnce(); + expect(resourceMock.getPropertyValue(PropertyHelper.getPropertyId("HostRoles", "cluster_name"))).andReturn("clusterone").atLeastOnce(); + + Capture<HostComponentLoggingInfo> captureLogInfo = Capture.newInstance(); + // expect set method to be called + resourceMock.setProperty(eq("logging"), capture(captureLogInfo)); + + LogLevelQueryResponse levelQueryResponse = + new LogLevelQueryResponse(); + + levelQueryResponse.setTotalCount("3"); + // setup test data for log levels + List<NameValuePair> testListOfLogLevels = + new LinkedList<NameValuePair>(); + testListOfLogLevels.add(new NameValuePair("ERROR", "150")); + testListOfLogLevels.add(new NameValuePair("WARN", "500")); + testListOfLogLevels.add(new NameValuePair("INFO", "2200")); + + levelQueryResponse.setNameValueList(testListOfLogLevels); + + Request requestMock = + mockSupport.createMock(Request.class); + + Predicate predicateMock = + mockSupport.createMock(Predicate.class); + + AmbariManagementController controllerMock = + mockSupport.createMock(AmbariManagementController.class); + + AmbariMetaInfo metaInfoMock = + mockSupport.createMock(AmbariMetaInfo.class); + + Clusters clustersMock = + mockSupport.createMock(Clusters.class); + + Cluster clusterMock = + mockSupport.createMock(Cluster.class); + + StackId stackIdMock = + mockSupport.createMock(StackId.class); + + ComponentInfo componentInfoMock = + mockSupport.createMock(ComponentInfo.class); + + LogDefinition logDefinitionMock = + mockSupport.createMock(LogDefinition.class); + + LogSearchDataRetrievalService dataRetrievalServiceMock = + mockSupport.createMock(LogSearchDataRetrievalService.class); + + LoggingRequestHelperFactory loggingRequestHelperFactoryMock = + mockSupport.createMock(LoggingRequestHelperFactory.class); + + LoggingRequestHelper loggingRequestHelperMock = + mockSupport.createMock(LoggingRequestHelper.class); + + expect(dataRetrievalServiceMock.getLogFileNames(expectedLogSearchComponentName, "c6401.ambari.apache.org", "clusterone")).andReturn(Collections.singleton(expectedLogFilePath)).atLeastOnce(); + // return null, to simulate the case when the LogSearch service goes down, and the helper object + // is not available to continue servicing the request. + expect(dataRetrievalServiceMock.getLogFileTailURI(expectedAmbariURL + expectedSearchEnginePath, expectedLogSearchComponentName, "c6401.ambari.apache.org", "clusterone")).andReturn(null).atLeastOnce(); + + + expect(controllerMock.getAmbariServerURI(expectedSearchEnginePath)). + andReturn(expectedAmbariURL + expectedSearchEnginePath).atLeastOnce(); + expect(controllerMock.getAmbariMetaInfo()).andReturn(metaInfoMock).atLeastOnce(); + expect(controllerMock.getClusters()).andReturn(clustersMock).atLeastOnce(); + expect(clustersMock.getCluster("clusterone")).andReturn(clusterMock).atLeastOnce(); + expect(stackIdMock.getStackName()).andReturn(expectedStackName).atLeastOnce(); + expect(stackIdMock.getStackVersion()).andReturn(expectedStackVersion).atLeastOnce(); + expect(clusterMock.getCurrentStackVersion()).andReturn(stackIdMock).atLeastOnce(); + expect(loggingRequestHelperFactoryMock.getHelper(anyObject(AmbariManagementController.class), anyObject(String.class))) + .andReturn(loggingRequestHelperMock).atLeastOnce(); + + expect(metaInfoMock.getComponentToService(expectedStackName, expectedStackVersion, expectedComponentName)).andReturn(expectedServiceName).atLeastOnce(); + expect(metaInfoMock.getComponent(expectedStackName, expectedStackVersion, expectedServiceName, expectedComponentName)).andReturn(componentInfoMock).atLeastOnce(); + + expect(componentInfoMock.getLogs()).andReturn(Collections.singletonList(logDefinitionMock)).atLeastOnce(); + expect(logDefinitionMock.getLogId()).andReturn(expectedLogSearchComponentName).atLeastOnce(); + + mockSupport.replayAll(); + + LoggingSearchPropertyProvider propertyProvider = + new LoggingSearchPropertyProvider(); + + propertyProvider.setAmbariManagementController(controllerMock); + propertyProvider.setLogSearchDataRetrievalService(dataRetrievalServiceMock); + propertyProvider.setLoggingRequestHelperFactory(loggingRequestHelperFactoryMock); + + Set<Resource> returnedResources = + propertyProvider.populateResources(Collections.singleton(resourceMock), requestMock, predicateMock); + + // verify that the property provider attached + // the expected logging structure to the associated resource + + assertEquals("Returned resource set was of an incorrect size", + 1, returnedResources.size()); + + HostComponentLoggingInfo returnedLogInfo = + captureLogInfo.getValue(); + + assertNotNull("Returned log info should not be null", + returnedLogInfo); + + assertEquals("Returned component was not the correct name", + "hdfs_namenode", returnedLogInfo.getComponentName()); + + assertEquals("Returned list of log file names for this component was incorrect", + 0, returnedLogInfo.getListOfLogFileDefinitions().size()); + + // verify that the log level count information + // was not added to the HostComponent resource + assertNull(returnedLogInfo.getListOfLogLevels()); + + mockSupport.verifyAll(); + } + + /** * Verifies that this property provider implementation will * properly handle the case of LogSearch not being deployed in * the cluster or available.
