AMBARI-22300. Log Feeder: allow applying grok filter on default fields (like type, path, host etc.) (oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/da98efc4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/da98efc4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/da98efc4 Branch: refs/heads/branch-2.6 Commit: da98efc4a6fa4e941ab1b997cb73f1e410de936c Parents: 289b251 Author: Oliver Szabo <[email protected]> Authored: Wed Oct 25 00:45:10 2017 +0200 Committer: Oliver Szabo <[email protected]> Committed: Tue Nov 7 22:53:34 2017 +0100 ---------------------------------------------------------------------- .../ambari/logfeeder/filter/FilterGrok.java | 3 +++ .../apache/ambari/logfeeder/input/Input.java | 7 +++++++ .../ambari/logfeeder/output/OutputManager.java | 16 +------------- .../ambari/logfeeder/util/LogFeederUtil.java | 22 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java index deff1b2..49d7e76 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterGrok.java @@ -185,6 +185,7 @@ public class FilterGrok extends Filter { if (strBuff != null) { Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>()); try { + LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, false); applyMessage(strBuff.toString(), jsonObj, currMultilineJsonStr); } finally { strBuff = null; @@ -204,6 +205,7 @@ public class FilterGrok extends Filter { } else { savedInputMarker = inputMarker; Map<String, Object> jsonObj = Collections.synchronizedMap(new HashMap<String, Object>()); + LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, false); applyMessage(inputStr, jsonObj, null); } } @@ -212,6 +214,7 @@ public class FilterGrok extends Filter { public void apply(Map<String, Object> jsonObj, InputMarker inputMarker) throws LogfeederException { if (sourceField != null) { savedInputMarker = inputMarker; + LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, false); applyMessage((String) jsonObj.get(sourceField), jsonObj, null); if (removeSourceField) { jsonObj.remove(sourceField); http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java index 6f6bb6d..96320e9 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/input/Input.java @@ -83,6 +83,8 @@ public abstract class Input extends ConfigBlock implements Runnable, Cloneable { private boolean multiFolder = false; private Map<String, List<File>> folderMap; private Map<String, InputFile> inputChildMap = new HashMap<>(); // TODO: weird it has this relationship + private boolean initDefaultFields = false; + protected MetricData readBytesMetric = new MetricData(getReadBytesMetricName(), false); protected String getReadBytesMetricName() { @@ -96,6 +98,7 @@ public abstract class Input extends ConfigBlock implements Runnable, Cloneable { detachIntervalMin = getIntValue("detach_interval_min", DEFAULT_DETACH_INTERVAL_MIN * 60); detachTimeMin = getIntValue("detach_time_min", DEFAULT_DETACH_TIME_MIN * 60); pathUpdateIntervalMin = getIntValue("path_update_interval_min", DEFAULT_LOG_PATH_UPDATE_INTERVAL_MIN * 60); + initDefaultFields = getBooleanValue("init_default_fields", false); if (typeValue != null) { // Explicitly add type and value to field list contextFields.put("type", typeValue); @@ -440,4 +443,8 @@ public abstract class Input extends ConfigBlock implements Runnable, Cloneable { public void setThread(Thread thread) { this.thread = thread; } + + public boolean isInitDefaultFields() { + return initDefaultFields; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java index 8bf6ae3..c851f04 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java @@ -85,21 +85,7 @@ public class OutputManager { // TODO: Ideally most of the overrides should be configurable - if (jsonObj.get("type") == null) { - jsonObj.put("type", input.getStringValue("type")); - } - if (jsonObj.get("path") == null && input.getFilePath() != null) { - jsonObj.put("path", input.getFilePath()); - } - if (jsonObj.get("path") == null && input.getStringValue("path") != null) { - jsonObj.put("path", input.getStringValue("path")); - } - if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) { - jsonObj.put("host", LogFeederUtil.hostName); - } - if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) { - jsonObj.put("ip", LogFeederUtil.ipAddress); - } + LogFeederUtil.fillMapWithFieldDefaults(jsonObj, inputMarker, true); if (jsonObj.get("level") == null) { jsonObj.put("level", LogFeederConstants.LOG_LEVEL_UNKNOWN); } http://git-wip-us.apache.org/repos/asf/ambari/blob/da98efc4/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java index 73cf449..86287cd 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java @@ -33,6 +33,8 @@ import java.util.Properties; import org.apache.ambari.logfeeder.LogFeeder; import org.apache.ambari.logfeeder.common.LogFeederConstants; +import org.apache.ambari.logfeeder.input.Input; +import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.metrics.MetricData; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; @@ -339,4 +341,24 @@ public class LogFeederUtil { } return logfeederTempDir; } + + public static void fillMapWithFieldDefaults(Map<String, Object> jsonObj, InputMarker inputMarker, boolean force) { + if (inputMarker != null && inputMarker.input != null && (force || inputMarker.input.isInitDefaultFields())) { + if (jsonObj.get("type") == null) { + jsonObj.put("type", inputMarker.input.getStringValue("type")); + } + if (jsonObj.get("path") == null && inputMarker.input.getFilePath() != null) { + jsonObj.put("path", inputMarker.input.getFilePath()); + } + if (jsonObj.get("path") == null && inputMarker.input.getStringValue("path") != null) { + jsonObj.put("path", inputMarker.input.getStringValue("path")); + } + if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) { + jsonObj.put("host", LogFeederUtil.hostName); + } + if (jsonObj.get("ip") == null && LogFeederUtil.ipAddress != null) { + jsonObj.put("ip", LogFeederUtil.ipAddress); + } + } + } }
