Repository: ambari Updated Branches: refs/heads/branch-2.5 9f8401a4a -> adc63c05a
AMBARI-19658. LogSearch Integration Cache Timeout should be configurable. (rnettleton) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/adc63c05 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/adc63c05 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/adc63c05 Branch: refs/heads/branch-2.5 Commit: adc63c05aa69610a36fcc4aa67ac37aa3cfe522e Parents: 9f8401a Author: Bob Nettleton <[email protected]> Authored: Tue Jan 24 17:53:03 2017 -0500 Committer: Bob Nettleton <[email protected]> Committed: Tue Jan 24 17:53:03 2017 -0500 ---------------------------------------------------------------------- .../server/configuration/Configuration.java | 26 +++++++++++++++++- .../logging/LogSearchDataRetrievalService.java | 28 +++++++++++++++++--- .../LogSearchDataRetrievalServiceTest.java | 27 ++++++++++++++++++- 3 files changed, 76 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/adc63c05/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java index 83e479a..83fe01d 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java @@ -2519,7 +2519,6 @@ public class Configuration { public static final ConfigurationProperty<Integer> LOGSEARCH_PORTAL_READ_TIMEOUT = new ConfigurationProperty<>( "logsearch.portal.read.timeout", 5000); - /** * Global disable flag for AmbariServer Metrics. */ @@ -2527,6 +2526,18 @@ public class Configuration { public static final ConfigurationProperty<Boolean> AMBARISERVER_METRICS_DISABLE = new ConfigurationProperty<>( "ambariserver.metrics.disable", false); + /** + * The time, in hours, that the Ambari Server will hold Log File metadata in its internal cache before making + * a request to the LogSearch Portal to get the latest metadata. + * + * The logging metadata (in this case, log file names) is generally quite static, so the default should + * generally be quite long. + * + */ + @Markdown(description = "The time, in hours, that the Ambari Server will hold Log File metadata in its internal cache before making a request to the LogSearch Portal to get the latest metadata.") + public static final ConfigurationProperty<Integer> LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT = new ConfigurationProperty<>( + "logsearch.metadata.cache.expire.timeout", 24); + private static final Logger LOG = LoggerFactory.getLogger( Configuration.class); @@ -5204,6 +5215,19 @@ public class Configuration { return NumberUtils.toInt(getProperty(LOGSEARCH_PORTAL_READ_TIMEOUT)); } + + /** + * + * Get the max time, in hours, to hold data in the LogSearch + * metadata cache prior to expiring the cache and re-loading + * the data from the LogSearch Portal service. + * + * @return max number of hours that the LogSearch metadata is cached + */ + public int getLogSearchMetadataCacheExpireTimeout() { + return NumberUtils.toInt(getProperty(LOGSEARCH_METADATA_CACHE_EXPIRE_TIMEOUT)); + } + /** * Generates a markdown table which includes: * <ul> http://git-wip-us.apache.org/repos/asf/ambari/blob/adc63c05/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java index 5ae47cb..b227aac 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalService.java @@ -23,6 +23,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import org.apache.ambari.server.AmbariService; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.apache.ambari.server.controller.AmbariServer; import org.apache.commons.collections.CollectionUtils; @@ -76,6 +77,11 @@ public class LogSearchDataRetrievalService extends AbstractService { @Inject private Injector injector; + @Inject + private Configuration ambariServerConfiguration; + + + /** * A Cache of host+component names to a set of log files associated with * that Host/Component combination. This data is retrieved from the @@ -113,10 +119,17 @@ public class LogSearchDataRetrievalService extends AbstractService { protected void doStart() { LOG.debug("Initializing caches"); + + // obtain the max cache expire time from the ambari configuration + final int maxTimeoutForCacheInHours = + ambariServerConfiguration.getLogSearchMetadataCacheExpireTimeout(); + + LOG.debug("Caches configured with a max expire timeout of " + maxTimeoutForCacheInHours + " hours."); + // initialize the log file name cache - logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build(); + logFileNameCache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours, TimeUnit.HOURS).build(); // initialize the log file tail URI cache - logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(1, TimeUnit.HOURS).build(); + logFileTailURICache = CacheBuilder.newBuilder().expireAfterWrite(maxTimeoutForCacheInHours, TimeUnit.HOURS).build(); // initialize the Executor executor = Executors.newSingleThreadExecutor(); @@ -125,7 +138,7 @@ public class LogSearchDataRetrievalService extends AbstractService { @Override protected void doStop() { LOG.debug("Invalidating LogSearch caches"); - // invalidate the cache + // invalidate the caches logFileNameCache.invalidateAll(); logFileTailURICache.invalidateAll(); @@ -229,6 +242,15 @@ public class LogSearchDataRetrievalService extends AbstractService { } /** + * Package-level setter to facilitate simpler unit testing + * + * @param ambariServerConfiguration + */ + void setConfiguration(Configuration ambariServerConfiguration) { + this.ambariServerConfiguration = ambariServerConfiguration; + } + + /** * This protected method allows for simpler unit tests. * * @return the Set of current Requests that are not yet completed http://git-wip-us.apache.org/repos/asf/ambari/blob/adc63c05/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java index 7d4f049..2adaea3 100644 --- a/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java +++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/logging/LogSearchDataRetrievalServiceTest.java @@ -29,6 +29,7 @@ import java.util.Set; import java.util.concurrent.Executor; +import org.apache.ambari.server.configuration.Configuration; import org.apache.ambari.server.controller.AmbariManagementController; import org.easymock.EasyMockSupport; import org.junit.Test; @@ -69,13 +70,18 @@ public class LogSearchDataRetrievalServiceTest { LoggingRequestHelper helperMock = mockSupport.createMock(LoggingRequestHelper.class); + Configuration configurationMock = + mockSupport.createMock(Configuration.class); + expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(helperMock); expect(helperMock.createLogFileTailURI("http://localhost";, expectedComponentName, expectedHostName)).andReturn(expectedResultURI); + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); @@ -95,16 +101,23 @@ public class LogSearchDataRetrievalServiceTest { EasyMockSupport mockSupport = new EasyMockSupport(); - LoggingRequestHelperFactory helperFactoryMock = mockSupport.createMock(LoggingRequestHelperFactory.class); + LoggingRequestHelperFactory helperFactoryMock = + mockSupport.createMock(LoggingRequestHelperFactory.class); + + Configuration configurationMock = + mockSupport.createMock(Configuration.class); // return null, to simulate the case where LogSearch Server is // not available for some reason expect(helperFactoryMock.getHelper(null, expectedClusterName)).andReturn(null); + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); + mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); @@ -131,6 +144,9 @@ public class LogSearchDataRetrievalServiceTest { Injector injectorMock = mockSupport.createMock(Injector.class); + Configuration configurationMock = + mockSupport.createMock(Configuration.class); + // expect the executor to be called to execute the LogSearch request executorMock.execute(isA(LogSearchDataRetrievalService.LogSearchFileNameRequestRunnable.class)); // executor should only be called once @@ -138,11 +154,14 @@ public class LogSearchDataRetrievalServiceTest { expect(injectorMock.getInstance(LoggingRequestHelperFactory.class)).andReturn(helperFactoryMock); + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); + mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); retrievalService.setInjector(injectorMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); retrievalService.setExecutor(executorMock); @@ -173,10 +192,16 @@ public class LogSearchDataRetrievalServiceTest { Executor executorMock = mockSupport.createMock(Executor.class); + Configuration configurationMock = + mockSupport.createMock(Configuration.class); + + expect(configurationMock.getLogSearchMetadataCacheExpireTimeout()).andReturn(1).atLeastOnce(); + mockSupport.replayAll(); LogSearchDataRetrievalService retrievalService = new LogSearchDataRetrievalService(); retrievalService.setLoggingRequestHelperFactory(helperFactoryMock); + retrievalService.setConfiguration(configurationMock); // call the initialization routine called by the Google framework retrievalService.doStart(); // there should be no expectations set on this mock
