[56/58] [abbrv] ambari git commit: AMBARI-18606. Improve Audit Log processing by Logfeeder (Miklos Gergely via oleewere)

[oleewere]

AMBARI-18606. Improve Audit Log processing by Logfeeder (Miklos Gergely via 
oleewere)

Change-Id: I9e357536115a691801013932cf13051908170d93


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a9094ab8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a9094ab8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a9094ab8

Branch: refs/heads/branch-2.5
Commit: a9094ab8ddcc8ac42d2c02de2c66656049d61f7c
Parents: 0e852c1
Author: Miklos Gergely 
Authored: Wed Oct 19 17:24:58 2016 +0200
Committer: oleewere 
Committed: Sat Nov 5 14:46:47 2016 +0100

--
 .../ambari/logfeeder/filter/FilterKeyValue.java |  65 ++-
 .../logfeeder/mapper/MapperFieldCopy.java   |  58 ++
 .../src/main/resources/alias_config.json|   3 +
 .../logfeeder/mapper/MapperFieldCopyTest.java   |  71 +++
 .../configsets/audit_logs/conf/managed-schema   |   5 +
 .../logsearch/common/LogSearchConstants.java|   2 +-
 .../ambari/logsearch/dao/AuditSolrDao.java  |   2 +-
 .../ambari/logsearch/dao/SolrCollectionDao.java |   2 +-
 .../logsearch/dao/SolrSchemaFieldDao.java   | 105 ++--
 .../apache/ambari/logsearch/dao/UserDao.java|   2 +-
 .../logsearch/graph/GraphDataGenerator.java |  10 +-
 .../ambari/logsearch/manager/ManagerBase.java   |   3 +-
 .../logsearch/manager/ServiceLogsManager.java   |   6 +-
 .../logsearch/manager/UserConfigManager.java|   8 +-
 .../logsearch/model/response/CommonLogData.java |  13 +-
 .../logsearch/solr/model/SolrAuditLogData.java  |   1 -
 .../logsearch/solr/model/SolrCommonLogData.java |  42 ++
 .../solr/model/SolrServiceLogData.java  |  34 --
 .../apache/ambari/logsearch/util/SolrUtil.java  |  62 +-
 .../LogsearchKRBAuthenticationFilter.java   |  12 +-
 .../logsearch/web/security/LdapProperties.java  |   2 +-
 .../scripts/views/audit/AuditTabLayoutView.js   |   2 +-
 .../src/main/webapp/static/schema_fields.json   |  19 +-
 .../test-config/logfeeder/logfeeder.properties  |   3 +-
 .../shipper-conf/input.config-ambari.json   | 585 +++
 .../test-logs/ambari-server/ambari-audit.log| 390 +
 .../templates/input.config-ambari.json.j2   | 119 
 .../templates/input.config-falcon.json.j2   |   2 +-
 28 files changed, 1458 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/a9094ab8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
--
diff --git 
a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
 
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
index 5bb15ff..b04a439 100644
--- 
a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
+++ 
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
@@ -19,6 +19,7 @@
 
 package org.apache.ambari.logfeeder.filter;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -68,18 +69,26 @@ public class FilterKeyValue extends Filter {
 if (sourceField == null) {
   return;
 }
-Object valueObj = jsonObj.get(sourceField);
-if (valueObj != null) {
+if (jsonObj.containsKey(sourceField)) {
+  String keyValueString = (String) jsonObj.get(sourceField);
+  Map valueMap = new HashMap<>();
+  if (valueBorders != null) {
+keyValueString = preProcessBorders(keyValueString, valueMap);
+  }
+  
   String splitPattern = Pattern.quote(fieldSplit);
-  String[] tokens = valueObj.toString().split(splitPattern);
+  String[] tokens = keyValueString.split(splitPattern);
   for (String nv : tokens) {
 String[] nameValue = getNameValue(nv);
 String name = nameValue != null && nameValue.length == 2 ? 
nameValue[0] : null;
 String value = nameValue != null && nameValue.length == 2 ? 
nameValue[1] : null;
 if (name != null && value != null) {
-jsonObj.put(name, value);
- } else {
-   logParseError("name=" + name + ", pair=" + nv + ", field=" + 
sourceField + ", field_value=" + valueObj);
+  if (valueMap.containsKey(value)) {
+value = valueMap.get(value);
+  }
+  jsonObj.put(name, value);
+} else {
+ logParseError("name=" + name + ", pair=" + nv + ", field=" + 
sourceField + ", field_value=" + keyValueString);
 }
   }
 }
@@ -87,19 +96,41 @@ public class FilterKeyValue extends Filter {
 

[56/58] [abbrv] ambari git commit: AMBARI-18606. Improve Audit Log processing by Logfeeder (Miklos Gergely via oleewere)

[oleewere]

AMBARI-18606. Improve Audit Log processing by Logfeeder (Miklos Gergely via 
oleewere)

Change-Id: I9e357536115a691801013932cf13051908170d93


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a9094ab8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a9094ab8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a9094ab8

Branch: refs/heads/logsearch-ga
Commit: a9094ab8ddcc8ac42d2c02de2c66656049d61f7c
Parents: 0e852c1
Author: Miklos Gergely 
Authored: Wed Oct 19 17:24:58 2016 +0200
Committer: oleewere 
Committed: Sat Nov 5 14:46:47 2016 +0100

--
 .../ambari/logfeeder/filter/FilterKeyValue.java |  65 ++-
 .../logfeeder/mapper/MapperFieldCopy.java   |  58 ++
 .../src/main/resources/alias_config.json|   3 +
 .../logfeeder/mapper/MapperFieldCopyTest.java   |  71 +++
 .../configsets/audit_logs/conf/managed-schema   |   5 +
 .../logsearch/common/LogSearchConstants.java|   2 +-
 .../ambari/logsearch/dao/AuditSolrDao.java  |   2 +-
 .../ambari/logsearch/dao/SolrCollectionDao.java |   2 +-
 .../logsearch/dao/SolrSchemaFieldDao.java   | 105 ++--
 .../apache/ambari/logsearch/dao/UserDao.java|   2 +-
 .../logsearch/graph/GraphDataGenerator.java |  10 +-
 .../ambari/logsearch/manager/ManagerBase.java   |   3 +-
 .../logsearch/manager/ServiceLogsManager.java   |   6 +-
 .../logsearch/manager/UserConfigManager.java|   8 +-
 .../logsearch/model/response/CommonLogData.java |  13 +-
 .../logsearch/solr/model/SolrAuditLogData.java  |   1 -
 .../logsearch/solr/model/SolrCommonLogData.java |  42 ++
 .../solr/model/SolrServiceLogData.java  |  34 --
 .../apache/ambari/logsearch/util/SolrUtil.java  |  62 +-
 .../LogsearchKRBAuthenticationFilter.java   |  12 +-
 .../logsearch/web/security/LdapProperties.java  |   2 +-
 .../scripts/views/audit/AuditTabLayoutView.js   |   2 +-
 .../src/main/webapp/static/schema_fields.json   |  19 +-
 .../test-config/logfeeder/logfeeder.properties  |   3 +-
 .../shipper-conf/input.config-ambari.json   | 585 +++
 .../test-logs/ambari-server/ambari-audit.log| 390 +
 .../templates/input.config-ambari.json.j2   | 119 
 .../templates/input.config-falcon.json.j2   |   2 +-
 28 files changed, 1458 insertions(+), 170 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/ambari/blob/a9094ab8/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
--
diff --git 
a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
 
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
index 5bb15ff..b04a439 100644
--- 
a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
+++ 
b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/filter/FilterKeyValue.java
@@ -19,6 +19,7 @@
 
 package org.apache.ambari.logfeeder.filter;
 
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -68,18 +69,26 @@ public class FilterKeyValue extends Filter {
 if (sourceField == null) {
   return;
 }
-Object valueObj = jsonObj.get(sourceField);
-if (valueObj != null) {
+if (jsonObj.containsKey(sourceField)) {
+  String keyValueString = (String) jsonObj.get(sourceField);
+  Map valueMap = new HashMap<>();
+  if (valueBorders != null) {
+keyValueString = preProcessBorders(keyValueString, valueMap);
+  }
+  
   String splitPattern = Pattern.quote(fieldSplit);
-  String[] tokens = valueObj.toString().split(splitPattern);
+  String[] tokens = keyValueString.split(splitPattern);
   for (String nv : tokens) {
 String[] nameValue = getNameValue(nv);
 String name = nameValue != null && nameValue.length == 2 ? 
nameValue[0] : null;
 String value = nameValue != null && nameValue.length == 2 ? 
nameValue[1] : null;
 if (name != null && value != null) {
-jsonObj.put(name, value);
- } else {
-   logParseError("name=" + name + ", pair=" + nv + ", field=" + 
sourceField + ", field_value=" + valueObj);
+  if (valueMap.containsKey(value)) {
+value = valueMap.get(value);
+  }
+  jsonObj.put(name, value);
+} else {
+ logParseError("name=" + name + ", pair=" + nv + ", field=" + 
sourceField + ", field_value=" + keyValueString);
 }
   }
 }
@@ -87,19 +96,41 @@ public class FilterKeyValue extends Filter {