This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push: new f355523 AMBARI-24943. Log Feeder: log level filter does not have any affect if level field is null (#38) f355523 is described below commit f3555232e931b3a073d448fcd3874f76300830e2 Author: Olivér Szabó <oleew...@gmail.com> AuthorDate: Thu Nov 22 14:53:01 2018 +0100 AMBARI-24943. Log Feeder: log level filter does not have any affect if level field is null (#38) * AMBARI-24943. Log Feeder: log level filter does not have any affect if level field is null * Remove unused import --- .../loglevelfilter/LogLevelFilterHandler.java | 8 +-- .../logfeeder/output/OutputLineEnricher.java | 8 ++- .../ambari/logfeeder/output/OutputManagerImpl.java | 4 +- .../output/cloud/CloudStorageOutputManager.java | 2 +- .../shipper-conf/input.config-grafana.json | 67 ++++++++++++++++++++++ docker/test-logs/grafana/grafana.log | 8 +++ 6 files changed, 89 insertions(+), 8 deletions(-) diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java index a1780cf..a00e99b 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/loglevelfilter/LogLevelFilterHandler.java @@ -30,7 +30,7 @@ import org.apache.ambari.logsearch.config.zookeeper.LogLevelFilterManagerZK; import org.apache.ambari.logsearch.config.zookeeper.LogSearchConfigZKHelper; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.TreeCache; import org.apache.curator.framework.recipes.cache.TreeCacheListener; @@ -133,7 +133,7 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor { } public boolean isAllowed(String jsonBlock, InputMarker inputMarker, List<String> defaultLogLevels) { - if (org.apache.commons.lang3.StringUtils.isEmpty(jsonBlock)) { + if (StringUtils.isEmpty(jsonBlock)) { return DEFAULT_VALUE; } Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock); @@ -160,8 +160,8 @@ public class LogLevelFilterHandler implements LogLevelFilterMonitor { String hostName = (String) jsonObj.get(LogFeederConstants.SOLR_HOST); String logId = (String) jsonObj.get(LogFeederConstants.SOLR_COMPONENT); - String level = (String) jsonObj.get(LogFeederConstants.SOLR_LEVEL); - if (org.apache.commons.lang3.StringUtils.isNotBlank(hostName) && org.apache.commons.lang3.StringUtils.isNotBlank(logId) && org.apache.commons.lang3.StringUtils.isNotBlank(level)) { + String level = (String) jsonObj.getOrDefault(LogFeederConstants.SOLR_LEVEL, LogFeederConstants.LOG_LEVEL_UNKNOWN); + if (StringUtils.isNotBlank(hostName) && StringUtils.isNotBlank(logId) && StringUtils.isNotBlank(level)) { return isAllowed(hostName, logId, level, defaultLogLevels); } else { return DEFAULT_VALUE; diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java index ff0805d..0b1c5a2 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputLineEnricher.java @@ -19,6 +19,7 @@ package org.apache.ambari.logfeeder.output; import com.google.common.hash.Hashing; +import org.apache.ambari.logfeeder.common.LogFeederConstants; import org.apache.ambari.logfeeder.plugin.common.MetricData; import org.apache.ambari.logfeeder.plugin.input.Input; import org.apache.ambari.logfeeder.plugin.input.InputMarker; @@ -42,7 +43,7 @@ public class OutputLineEnricher { private static final int MAX_OUTPUT_SIZE = 32765; // 32766-1 - public void enrichFields(final Map<String, Object> jsonObj, final InputMarker inputMarker, final MetricData messageTruncateMetric) { + public Map<String, Object> enrichFields(final Map<String, Object> jsonObj, final InputMarker inputMarker, final MetricData messageTruncateMetric) { Input input = inputMarker.getInput(); // Update the block with the context fields for (Map.Entry<String, String> entry : input.getInputDescriptor().getAddFields().entrySet()) { @@ -79,12 +80,17 @@ public class OutputLineEnricher { (Integer) inputMarker.getAllProperties().get("line_number") > 0) { jsonObj.put("logfile_line_number", inputMarker.getAllProperties().get("line_number")); } + if (!jsonObj.containsKey("level")) { + jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN); + } if (jsonObj.containsKey("log_message")) { // TODO: Let's check size only for log_message for now String logMessage = (String) jsonObj.get("log_message"); logMessage = truncateLongLogMessage(messageTruncateMetric, jsonObj, input, logMessage); jsonObj.put("message_md5", "" + Hashing.md5().hashBytes(logMessage.getBytes()).asLong()); } + + return jsonObj; } @SuppressWarnings("unchecked") diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java index b4c862d..9f40b91 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java @@ -82,10 +82,10 @@ public class OutputManagerImpl extends OutputManager { @SuppressWarnings("unchecked") public void write(Map<String, Object> jsonObj, InputMarker inputMarker) { jsonObj.put("seq_num", docCounter++); - if (docCounter == Long.MIN_VALUE) { + if (docCounter == Long.MAX_VALUE) { docCounter = 1; } - outputLineEnricher.enrichFields(jsonObj, inputMarker, messageTruncateMetric); + jsonObj = outputLineEnricher.enrichFields(jsonObj, inputMarker, messageTruncateMetric); Input input = inputMarker.getInput(); List<String> defaultLogLevels = getDefaultLogLevels(input); if (logLevelFilterHandler.isAllowed(jsonObj, inputMarker, defaultLogLevels) diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java index 9be30a0..4aca09d 100644 --- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java +++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageOutputManager.java @@ -60,7 +60,7 @@ public class CloudStorageOutputManager extends OutputManager { @Override public void write(Map<String, Object> jsonObj, InputMarker marker) { if (useFilters.get()) { - outputLineEnricher.enrichFields(jsonObj, marker, messageTruncateMetric); + jsonObj = outputLineEnricher.enrichFields(jsonObj, marker, messageTruncateMetric); if (!outputLineFilter.apply(jsonObj, marker.getInput())) { if (jsonObj.get("id") == null) { jsonObj.put("id", IdGeneratorHelper.generateUUID(jsonObj, storageOutput.getIdFields())); diff --git a/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json b/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json new file mode 100644 index 0000000..863050e --- /dev/null +++ b/docker/test-config/logfeeder/shipper-conf/input.config-grafana.json @@ -0,0 +1,67 @@ +{ + "input": [ + { + "type": "ams_grafana", + "rowtype": "service", + "path": "/root/test-logs/grafana/grafana.log" + } + ], + "filter": [ + { + "filter": "grok", + "conditions": { + "fields": { + "type": [ + "ams_grafana" + ] + } + }, + "skipOnError": false, + "deepExtract": false, + "post_map_values": { + "level": [ + { + "map_field_value": { + "pre_value": "I", + "post_value": "INFO" + } + }, + { + "map_field_value": { + "pre_value": "W", + "post_value": "WARN" + } + }, + { + "map_field_value": { + "pre_value": "D", + "post_value": "DEBUG" + } + }, + { + "map_field_value": { + "pre_value": "E", + "post_value": "ERROR" + } + }, + { + "map_field_value": { + "pre_value": "F", + "post_value": "FATAL" + } + } + ], + "logtime": [ + { + "map_date": { + "target_date_pattern": "yyyy/MM/dd HH:mm:ss" + } + } + ] + }, + "log4j_format": "%d{ISO8601} %-5p [%t] %c{2}: %m%n", + "multiline_pattern": "^(%{DATESTAMP:logtime})", + "message_pattern": "(?m)^%{DATESTAMP:logtime}%{SPACE}(?:%{DATA:method})%{SPACE}\\[%{WORD:level}\\]%{SPACE}%{GREEDYDATA:log_message}" + } + ] +} \ No newline at end of file diff --git a/docker/test-logs/grafana/grafana.log b/docker/test-logs/grafana/grafana.log new file mode 100644 index 0000000..8d561b9 --- /dev/null +++ b/docker/test-logs/grafana/grafana.log @@ -0,0 +1,8 @@ +2018/11/22 10:45:09 [I] Migrator: exec migration id: create index UQE_quota_org_id_user_id_target - v1 +2018/11/22 10:45:09 [I] Created default admin user: admin +2018/11/22 10:45:09 [I] Listen: http://0.0.0.0:3000 +Thu Nov 22 10:45:10 UTC 2018 pid_file has been written to +OK +2018/11/22 10:45:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E] Could not find plugin definition for data source 1: ambari-metrics +2018/11/22 10:46:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E] Could not find plugin definition for data source 2: ambari-metrics +2018/11/22 10:46:20 [frontendsettings.go:47 getFrontendSettingsMap()] [E] Could not find plugin definition for data source 3: ambari-metrics