This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch branch-2.7 in repository https://gitbox.apache.org/repos/asf/ambari.git
commit c04daafbcbbaac4a5a2fe5796e3aa35cfdfe7585 Author: Oliver Szabo <[email protected]> AuthorDate: Sun Jul 22 14:50:02 2018 +0200 AMBARI-24329. Log Feeder define default log levels per component. --- .../api/model/inputconfig/InputDescriptor.java | 3 +++ .../model/inputconfig/impl/InputDescriptorImpl.java | 20 ++++++++++++++++++++ .../loglevelfilter/LogLevelFilterHandler.java | 20 ++++++++++---------- .../ambari/logfeeder/output/OutputManagerImpl.java | 17 +++++++++++++++-- .../ambari/logfeeder/output/OutputManagerTest.java | 6 ++++-- .../ambari/logsearch/model/common/LSServerInput.java | 9 +++++++++ .../shipper-conf/input.config-logsearch-docker.json | 5 ++++- 7 files changed, 65 insertions(+), 15 deletions(-) diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java index 71f3cd1..7c00a6b 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java @@ -19,6 +19,7 @@ package org.apache.ambari.logsearch.config.api.model.inputconfig; +import java.util.List; import java.util.Map; public interface InputDescriptor { @@ -53,4 +54,6 @@ public interface InputDescriptor { String getGroup(); Boolean isInitDefaultFields(); + + List<String> getDefaultLogLevels(); } diff --git a/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java index 3a04ebc..d175c58 100644 --- a/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-json/src/main/java/org/apache/ambari/logsearch/config/json/model/inputconfig/impl/InputDescriptorImpl.java @@ -19,6 +19,7 @@ package org.apache.ambari.logsearch.config.json.model.inputconfig.impl; +import java.util.List; import java.util.Map; import org.apache.ambari.logsearch.config.api.ShipperConfigElementDescription; @@ -200,6 +201,16 @@ public abstract class InputDescriptorImpl implements InputDescriptor { @SerializedName("init_default_fields") private Boolean initDefaultFields; + @ShipperConfigElementDescription( + path = "/input/[]/default_log_levels", + type = "list of strings", + description = "Use these as default log levels for the input - overrides the global default log levels.", + examples = {"default_log_levels: [\"INFO\", \"WARN\"]"} + ) + @Expose + @SerializedName("default_log_levels") + private List<String> defaultLogLevels; + public String getType() { return type; } @@ -329,4 +340,13 @@ public abstract class InputDescriptorImpl implements InputDescriptor { public void setInitDefaultFields(Boolean initDefaultFields) { this.initDefaultFields = initDefaultFields; } + + @Override + public List<String> getDefaultLogLevels() { + return defaultLogLevels; + } + + public void setDefaultLogLevels(List<String> defaultLogLevels) { + this.defaultLogLevels = defaultLogLevels; + } } diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java index 977e537..8c35d56 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java @@ -98,29 +98,29 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor { return filters; } - public boolean isAllowed(String hostName, String logId, String level) { + public boolean isAllowed(String hostName, String logId, String level, List<String> defaultLogLevels) { if (!logFeederProps.isLogLevelFilterEnabled()) { return true; } - LogLevelFilter logFilter = findLogFilter(logId); + LogLevelFilter logFilter = findLogFilter(logId, defaultLogLevels); List<String> allowedLevels = getAllowedLevels(hostName, logFilter); return allowedLevels.isEmpty() || allowedLevels.contains(level); } - public boolean isAllowed(String jsonBlock, InputMarker inputMarker) { + public boolean isAllowed(String jsonBlock, InputMarker inputMarker, List<String> defaultLogLevels) { if (org.apache.commons.lang3.StringUtils.isEmpty(jsonBlock)) { return DEFAULT_VALUE; } Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock); - return isAllowed(jsonObj, inputMarker); + return isAllowed(jsonObj, inputMarker, defaultLogLevels); } - public boolean isAllowed(Map<String, Object> jsonObj, InputMarker inputMarker) { + public boolean isAllowed(Map<String, Object> jsonObj, InputMarker inputMarker, List<String> defaultLogLevels) { if ("audit".equals(inputMarker.getInput().getInputDescriptor().getRowtype())) return true; - boolean isAllowed = applyFilter(jsonObj); + boolean isAllowed = applyFilter(jsonObj, defaultLogLevels); if (!isAllowed) { LOG.trace("Filter block the content :" + LogFeederUtil.getGson().toJson(jsonObj)); } @@ -128,7 +128,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor { } - public boolean applyFilter(Map<String, Object> jsonObj) { + public boolean applyFilter(Map<String, Object> jsonObj, List<String> defaultLogLevels) { if (MapUtils.isEmpty(jsonObj)) { LOG.warn("Output jsonobj is empty"); return DEFAULT_VALUE; @@ -138,13 +138,13 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor { String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT); String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL); if (org.apache.commons.lang3.StringUtils.isNotBlank(hostName) && org.apache.commons.lang3.StringUtils.isNotBlank(logId) && org.apache.commons.lang3.StringUtils.isNotBlank(level)) { - return isAllowed(hostName, logId, level); + return isAllowed(hostName, logId, level, defaultLogLevels); } else { return DEFAULT_VALUE; } } - private synchronized LogLevelFilter findLogFilter(String logId) { + private synchronized LogLevelFilter findLogFilter(String logId, List<String> defaultLogLevels) { LogLevelFilter logFilter = filters.get(logId); if (logFilter != null) { return logFilter; @@ -153,7 +153,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor { LOG.info("Filter is not present for log " + logId + ", creating default filter"); LogLevelFilter defaultFilter = new LogLevelFilter(); defaultFilter.setLabel(logId); - defaultFilter.setDefaultLevels(logFeederProps.getIncludeDefaultLogLevels()); + defaultFilter.setDefaultLevels(defaultLogLevels); try { config.getLogLevelFilterManager().createLogLevelFilter(logFeederProps.getClusterName(), logId, defaultFilter); diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java index 612573b..09951cc 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java @@ -31,6 +31,7 @@ import org.apache.ambari.logfeeder.plugin.manager.OutputManager; import org.apache.ambari.logfeeder.plugin.output.Output; import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.ambari.logsearch.config.api.OutputConfigMonitor; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -147,7 +148,8 @@ public class OutputManagerImpl extends OutputManager { jsonObj.put("message_md5", "" + Hashing.md5().hashBytes(logMessage.getBytes()).asLong()); } } - if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker) + List<String> defaultLogLevels = getDefaultLogLevels(input); + if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker, defaultLogLevels) && !outputLineFilter.apply(jsonObj, inputMarker.getInput())) { List<? extends Output> outputList = input.getOutputList(); for (Output output : outputList) { @@ -160,6 +162,16 @@ public class OutputManagerImpl extends OutputManager { } } + private List<String> getDefaultLogLevels(Input input) { + List<String> defaultLogLevels = logFeederProps.getIncludeDefaultLogLevels(); + List<String> overrideDefaultLogLevels = input.getInputDescriptor().getDefaultLogLevels(); + if (CollectionUtils.isNotEmpty(overrideDefaultLogLevels)) { + return overrideDefaultLogLevels; + } else { + return defaultLogLevels; + } + } + @SuppressWarnings("unchecked") private String truncateLongLogMessage(Map<String, Object> jsonObj, Input input, String logMessage) { if (logMessage != null && logMessage.getBytes().length > MAX_OUTPUT_SIZE) { @@ -181,7 +193,8 @@ public class OutputManagerImpl extends OutputManager { } public void write(String jsonBlock, InputMarker inputMarker) { - if (logLevelFilterHandler.isAllowed(jsonBlock, inputMarker)) { + List<String> defaultLogLevels = getDefaultLogLevels(inputMarker.getInput()); + if (logLevelFilterHandler.isAllowed(jsonBlock, inputMarker, defaultLogLevels)) { List<? extends Output> outputList = inputMarker.getInput().getOutputList(); for (Output output : outputList) { try { diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java index 1a651ab..9536cf8 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java @@ -110,7 +110,7 @@ public class OutputManagerTest { expect(mockInput.isUseEventMD5()).andReturn(false).anyTimes(); expect(mockInput.isGenEventMD5()).andReturn(false).anyTimes(); expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor).anyTimes(); - expect(mockFilter.isAllowed(jsonObj, inputMarker)).andReturn(true).anyTimes(); + expect(mockFilter.isAllowed(jsonObj, inputMarker, null)).andReturn(true).anyTimes(); expect(mockInput.getCache()).andReturn(null); expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3)); @@ -121,6 +121,7 @@ public class OutputManagerTest { replay(output1, output2, output3, mockFilter, mockInput); OutputManagerImpl manager = new OutputManagerImpl(); + manager.setLogFeederProps(new LogFeederProps()); manager.setLogLevelFilterHandler(mockFilter); manager.add(output1); manager.add(output2); @@ -146,7 +147,7 @@ public class OutputManagerTest { LogLevelFilterHandler mockFilter = strictMock(LogLevelFilterHandler.class); expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor).anyTimes(); - expect(mockFilter.isAllowed(jsonString, inputMarker)).andReturn(true).anyTimes(); + expect(mockFilter.isAllowed(jsonString, inputMarker, null)).andReturn(true).anyTimes(); expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3)); output1.write(jsonString, inputMarker); expectLastCall(); @@ -157,6 +158,7 @@ public class OutputManagerTest { OutputManagerImpl manager = new OutputManagerImpl(); manager.setLogLevelFilterHandler(mockFilter); + manager.setLogFeederProps(new LogFeederProps()); manager.add(output1); manager.add(output2); manager.add(output3); diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java index af28f17..272f443 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java @@ -19,6 +19,7 @@ package org.apache.ambari.logsearch.model.common; +import java.util.List; import java.util.Map; import javax.validation.constraints.NotNull; @@ -73,6 +74,9 @@ public abstract class LSServerInput { @JsonProperty("init_default_fields") private Boolean initDefaultFields; + + @JsonProperty("default_log_levels") + private List<String> defaultLogLevels; public LSServerInput() {} @@ -92,6 +96,7 @@ public abstract class LSServerInput { this.cacheDedupInterval = inputDescriptor.getCacheDedupInterval(); this.isEnabled = inputDescriptor.isEnabled(); this.initDefaultFields = inputDescriptor.isInitDefaultFields(); + this.defaultLogLevels = inputDescriptor.getDefaultLogLevels(); } public String getType() { @@ -153,4 +158,8 @@ public abstract class LSServerInput { public Boolean getInitDefaultFields() { return initDefaultFields; } + + public List<String> getDefaultLogLevels() { + return defaultLogLevels; + } } diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json index a420960..04a3adc 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-logsearch-docker.json @@ -3,7 +3,10 @@ { "type": "logsearch_server", "rowtype": "service", - "docker": "true" + "docker": "true", + "default_log_levels" : [ + "FATAL", "ERROR", "WARN", "INFO", "DEBUG" + ] } ], "filter": [
