http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java index ba872f8..4d6c43b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManager.java @@ -70,7 +70,7 @@ public class OutputManager { Input input = inputMarker.input; // Update the block with the context fields - for (Map.Entry<String, String> entry : input.getContextFields().entrySet()) { + for (Map.Entry<String, String> entry : input.getInputDescriptor().getAddFields().entrySet()) { if (jsonObj.get(entry.getKey()) == null || entry.getKey().equals("cluster") && "null".equals(jsonObj.get(entry.getKey()))) { jsonObj.put(entry.getKey(), entry.getValue()); } @@ -79,13 +79,13 @@ public class OutputManager { // TODO: Ideally most of the overrides should be configurable if (jsonObj.get("type") == null) { - jsonObj.put("type", input.getStringValue("type")); + jsonObj.put("type", input.getInputDescriptor().getType()); } if (jsonObj.get("path") == null && input.getFilePath() != null) { jsonObj.put("path", input.getFilePath()); } - if (jsonObj.get("path") == null && input.getStringValue("path") != null) { - jsonObj.put("path", input.getStringValue("path")); + if (jsonObj.get("path") == null && input.getInputDescriptor().getPath() != null) { + jsonObj.put("path", input.getInputDescriptor().getPath()); } if (jsonObj.get("host") == null && LogFeederUtil.hostName != null) { jsonObj.put("host", LogFeederUtil.hostName);
http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java index d0f51b2..076d12d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputS3File.java @@ -19,9 +19,6 @@ package org.apache.ambari.logfeeder.output; import com.google.common.annotations.VisibleForTesting; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import org.apache.ambari.logfeeder.common.ConfigHandler; import org.apache.ambari.logfeeder.common.LogFeederConstants; import org.apache.ambari.logfeeder.filter.Filter; import org.apache.ambari.logfeeder.input.InputMarker; @@ -31,11 +28,18 @@ import org.apache.ambari.logfeeder.output.spool.RolloverCondition; import org.apache.ambari.logfeeder.output.spool.RolloverHandler; import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.ambari.logfeeder.util.S3Util; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigGson; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputConfigImpl; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputS3FileDescriptorImpl; import org.apache.log4j.Logger; import java.io.File; -import java.util.*; -import java.util.Map.Entry; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; /** @@ -50,7 +54,6 @@ import java.util.Map.Entry; public class OutputS3File extends Output implements RolloverCondition, RolloverHandler { private static final Logger LOG = Logger.getLogger(OutputS3File.class); - public static final String INPUT_ATTRIBUTE_TYPE = "type"; public static final String GLOBAL_CONFIG_S3_PATH_SUFFIX = "global.config.json"; private LogSpooler logSpooler; @@ -72,9 +75,9 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH */ @Override public void copyFile(File inputFile, InputMarker inputMarker) { - String type = inputMarker.input.getStringValue(INPUT_ATTRIBUTE_TYPE); + String type = inputMarker.input.getInputDescriptor().getType(); S3Uploader s3Uploader = new S3Uploader(s3OutputConfiguration, false, type); - String resolvedPath = s3Uploader.uploadFile(inputFile, inputMarker.input.getStringValue(INPUT_ATTRIBUTE_TYPE)); + String resolvedPath = s3Uploader.uploadFile(inputFile, inputMarker.input.getInputDescriptor().getType()); uploadConfig(inputMarker, type, s3OutputConfiguration, resolvedPath); } @@ -82,43 +85,43 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH private void uploadConfig(InputMarker inputMarker, String type, S3OutputConfiguration s3OutputConfiguration, String resolvedPath) { - ArrayList<Map<String, Object>> filters = new ArrayList<>(); + ArrayList<FilterDescriptor> filters = new ArrayList<>(); addFilters(filters, inputMarker.input.getFirstFilter()); - Map<String, Object> inputConfig = new HashMap<>(); - inputConfig.putAll(inputMarker.input.getConfigs()); + InputS3FileDescriptor inputS3FileDescriptorOriginal = (InputS3FileDescriptor) inputMarker.input.getInputDescriptor(); + InputS3FileDescriptorImpl inputS3FileDescriptor = InputConfigGson.gson.fromJson( + InputConfigGson.gson.toJson(inputS3FileDescriptorOriginal), InputS3FileDescriptorImpl.class); String s3CompletePath = LogFeederConstants.S3_PATH_START_WITH + s3OutputConfiguration.getS3BucketName() + LogFeederConstants.S3_PATH_SEPARATOR + resolvedPath; - inputConfig.put("path", s3CompletePath); + inputS3FileDescriptor.setPath(s3CompletePath); - ArrayList<Map<String, Object>> inputConfigList = new ArrayList<>(); - inputConfigList.add(inputConfig); + ArrayList<InputDescriptorImpl> inputConfigList = new ArrayList<>(); + inputConfigList.add(inputS3FileDescriptor); // set source s3_file - // remove global config from filter config - removeGlobalConfig(inputConfigList); - removeGlobalConfig(filters); + // remove global config from input config + removeS3GlobalConfig(inputS3FileDescriptor); // write config into s3 file - Map<String, Object> config = new HashMap<>(); - config.put("filter", filters); - config.put("input", inputConfigList); - writeConfigToS3(config, getComponentConfigFileName(type), s3OutputConfiguration); + InputConfigImpl inputConfig = new InputConfigImpl(); + inputConfig.setInput(inputConfigList); + + writeConfigToS3(inputConfig, getComponentConfigFileName(type), s3OutputConfiguration); // write global config writeGlobalConfig(s3OutputConfiguration); } - private void addFilters(ArrayList<Map<String, Object>> filters, Filter filter) { + private void addFilters(ArrayList<FilterDescriptor> filters, Filter filter) { if (filter != null) { - Map<String, Object> filterConfig = new HashMap<String, Object>(); - filterConfig.putAll(filter.getConfigs()); - filters.add(filterConfig); + FilterDescriptor filterDescriptorOriginal = filter.getFilterDescriptor(); + FilterDescriptor filterDescriptor = InputConfigGson.gson.fromJson( + InputConfigGson.gson.toJson(filterDescriptorOriginal), filterDescriptorOriginal.getClass()); + filters.add(filterDescriptor); if (filter.getNextFilter() != null) { addFilters(filters, filter.getNextFilter()); } } } - private void writeConfigToS3(Map<String, Object> configToWrite, String s3KeySuffix, S3OutputConfiguration s3OutputConfiguration) { - Gson gson = new GsonBuilder().setPrettyPrinting().create(); - String configJson = gson.toJson(configToWrite); + private void writeConfigToS3(Object config, String s3KeySuffix, S3OutputConfiguration s3OutputConfiguration) { + String configJson = InputConfigGson.gson.toJson(config); String s3ResolvedKey = new S3LogPathResolver().getResolvedPath(getStringValue("s3_config_dir"), s3KeySuffix, s3OutputConfiguration.getCluster()); @@ -131,31 +134,14 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH return "input.config-" + componentName + ".json"; } - - private Map<String, Object> getGlobalConfig() { - Map<String, Object> globalConfig = ConfigHandler.globalConfigs; - if (globalConfig == null) { - globalConfig = new HashMap<>(); - } - return globalConfig; - } - - private void removeGlobalConfig(List<Map<String, Object>> configList) { - Map<String, Object> globalConfig = getGlobalConfig(); - if (configList != null && globalConfig != null) { - for (Entry<String, Object> globalConfigEntry : globalConfig.entrySet()) { - if (globalConfigEntry != null) { - String globalKey = globalConfigEntry.getKey(); - if (globalKey != null && !globalKey.trim().isEmpty()) { - for (Map<String, Object> config : configList) { - if (config != null) { - config.remove(globalKey); - } - } - } - } - } - } + private void removeS3GlobalConfig(InputS3FileDescriptorImpl inputS3FileDescriptor) { + inputS3FileDescriptor.setSource(null); + inputS3FileDescriptor.setCopyFile(null); + inputS3FileDescriptor.setProcessFile(null); + inputS3FileDescriptor.setTail(null); + inputS3FileDescriptor.getAddFields().remove("ip"); + inputS3FileDescriptor.getAddFields().remove("host"); + inputS3FileDescriptor.getAddFields().remove("bundle_id"); } /** @@ -164,7 +150,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH @SuppressWarnings("unchecked") private synchronized void writeGlobalConfig(S3OutputConfiguration s3OutputConfiguration) { if (!uploadedGlobalConfig) { - Map<String, Object> globalConfig = LogFeederUtil.cloneObject(getGlobalConfig()); + Map<String, Object> globalConfig = new HashMap<>(); //updating global config before write to s3 globalConfig.put("source", "s3_file"); globalConfig.put("copy_file", false); @@ -205,7 +191,7 @@ public class OutputS3File extends Output implements RolloverCondition, RolloverH public void write(String block, InputMarker inputMarker) throws Exception { if (logSpooler == null) { logSpooler = createSpooler(inputMarker.input.getFilePath()); - s3Uploader = createUploader(inputMarker.input.getStringValue(INPUT_ATTRIBUTE_TYPE)); + s3Uploader = createUploader(inputMarker.input.getInputDescriptor().getType()); } logSpooler.add(block); } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java index 1929178..d8a1fbb 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/LogFeederUtil.java @@ -27,13 +27,11 @@ import java.net.InetAddress; import java.net.UnknownHostException; import java.util.HashMap; import java.util.Hashtable; -import java.util.List; import java.util.Map; import java.util.Properties; import org.apache.ambari.logfeeder.LogFeeder; import org.apache.ambari.logfeeder.metrics.MetricData; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -205,55 +203,6 @@ public class LogFeederUtil { return retValue; } - @SuppressWarnings("unchecked") - public static boolean isEnabled(Map<String, Object> conditionConfigs, Map<String, Object> valueConfigs) { - Map<String, Object> conditions = (Map<String, Object>) conditionConfigs.get("conditions"); - if (MapUtils.isEmpty(conditions)) { - return toBoolean((String) valueConfigs.get("is_enabled"), true); - } - - for (String conditionType : conditions.keySet()) { - if (!conditionType.equalsIgnoreCase("fields")) { - continue; - } - - Map<String, Object> fields = (Map<String, Object>) conditions.get("fields"); - for (Map.Entry<String, Object> field : fields.entrySet()) { - if (field.getValue() instanceof String) { - if (isFieldConditionMatch(valueConfigs, field.getKey(), (String) field.getValue())) { - return true; - } - } else { - for (String stringValue : (List<String>) field.getValue()) { - if (isFieldConditionMatch(valueConfigs, field.getKey(), stringValue)) { - return true; - } - } - } - } - } - - return false; - } - - private static boolean isFieldConditionMatch(Map<String, Object> configs, String fieldName, String stringValue) { - boolean allow = false; - String fieldValue = (String) configs.get(fieldName); - if (fieldValue != null && fieldValue.equalsIgnoreCase(stringValue)) { - allow = true; - } else { - @SuppressWarnings("unchecked") - Map<String, Object> addFields = (Map<String, Object>) configs.get("add_fields"); - if (addFields != null && addFields.get(fieldName) != null) { - String addFieldValue = (String) addFields.get(fieldName); - if (stringValue.equalsIgnoreCase(addFieldValue)) { - allow = true; - } - } - } - return allow; - } - public static void logStatForMetric(MetricData metric, String prefixStr, String postFix) { long currStat = metric.value; long currMS = System.currentTimeMillis(); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java index 99565c5..8d7e86c 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterGrokTest.java @@ -18,12 +18,13 @@ package org.apache.ambari.logfeeder.filter; -import java.util.HashMap; import java.util.Map; import org.apache.ambari.logfeeder.input.Input; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.output.OutputManager; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterGrokDescriptorImpl; import org.apache.log4j.Logger; import org.easymock.Capture; import org.easymock.CaptureType; @@ -43,12 +44,12 @@ public class FilterGrokTest { private OutputManager mockOutputManager; private Capture<Map<String, Object>> capture; - public void init(Map<String, Object> config) throws Exception { + public void init(FilterGrokDescriptor filterGrokDescriptor) throws Exception { mockOutputManager = EasyMock.strictMock(OutputManager.class); capture = EasyMock.newCapture(CaptureType.LAST); filterGrok = new FilterGrok(); - filterGrok.loadConfig(config); + filterGrok.loadConfig(filterGrokDescriptor); filterGrok.setOutputManager(mockOutputManager); filterGrok.setInput(EasyMock.mock(Input.class)); filterGrok.init(); @@ -58,10 +59,10 @@ public class FilterGrokTest { public void testFilterGrok_parseMessage() throws Exception { LOG.info("testFilterGrok_parseMessage()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("message_pattern", "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}"); - config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})"); - init(config); + FilterGrokDescriptorImpl filterGrokDescriptor = new FilterGrokDescriptorImpl(); + filterGrokDescriptor.setMessagePattern("(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}"); + filterGrokDescriptor.setMultilinePattern("^(%{TIMESTAMP_ISO8601:logtime})"); + init(filterGrokDescriptor); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -84,10 +85,10 @@ public class FilterGrokTest { public void testFilterGrok_parseMultiLineMessage() throws Exception { LOG.info("testFilterGrok_parseMultiLineMessage()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("message_pattern", "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}"); - config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})"); - init(config); + FilterGrokDescriptorImpl filterGrokDescriptor = new FilterGrokDescriptorImpl(); + filterGrokDescriptor.setMessagePattern("(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}"); + filterGrokDescriptor.setMultilinePattern("^(%{TIMESTAMP_ISO8601:logtime})"); + init(filterGrokDescriptor); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -114,10 +115,10 @@ public class FilterGrokTest { public void testFilterGrok_notMatchingMesagePattern() throws Exception { LOG.info("testFilterGrok_notMatchingMesagePattern()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("message_pattern", "(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}"); - config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})"); - init(config); + FilterGrokDescriptorImpl filterGrokDescriptor = new FilterGrokDescriptorImpl(); + filterGrokDescriptor.setMessagePattern("(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}"); + filterGrokDescriptor.setMultilinePattern("^(%{TIMESTAMP_ISO8601:logtime})"); + init(filterGrokDescriptor); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall().anyTimes(); @@ -134,9 +135,9 @@ public class FilterGrokTest { public void testFilterGrok_noMesagePattern() throws Exception { LOG.info("testFilterGrok_noMesagePattern()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("multiline_pattern", "^(%{TIMESTAMP_ISO8601:logtime})"); - init(config); + FilterGrokDescriptorImpl filterGrokDescriptor = new FilterGrokDescriptorImpl(); + filterGrokDescriptor.setMultilinePattern("^(%{TIMESTAMP_ISO8601:logtime})"); + init(filterGrokDescriptor); EasyMock.replay(mockOutputManager); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java index 643dafc..8f75c3a 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterJSONTest.java @@ -21,7 +21,6 @@ package org.apache.ambari.logfeeder.filter; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.HashMap; import java.util.Map; import java.util.TimeZone; @@ -29,6 +28,7 @@ import org.apache.ambari.logfeeder.common.LogFeederConstants; import org.apache.ambari.logfeeder.common.LogfeederException; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.output.OutputManager; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterJsonDescriptorImpl; import org.apache.log4j.Logger; import org.easymock.Capture; import org.easymock.CaptureType; @@ -47,12 +47,12 @@ public class FilterJSONTest { private OutputManager mockOutputManager; private Capture<Map<String, Object>> capture; - public void init(Map<String, Object> params) throws Exception { + public void init(FilterJsonDescriptorImpl filterJsonDescriptor) throws Exception { mockOutputManager = EasyMock.strictMock(OutputManager.class); capture = EasyMock.newCapture(CaptureType.LAST); filterJson = new FilterJSON(); - filterJson.loadConfig(params); + filterJson.loadConfig(filterJsonDescriptor); filterJson.setOutputManager(mockOutputManager); filterJson.init(); } @@ -61,7 +61,7 @@ public class FilterJSONTest { public void testJSONFilterCode_convertFields() throws Exception { LOG.info("testJSONFilterCode_convertFields()"); - init(new HashMap<String, Object>()); + init(new FilterJsonDescriptorImpl()); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -86,7 +86,7 @@ public class FilterJSONTest { public void testJSONFilterCode_logTimeOnly() throws Exception { LOG.info("testJSONFilterCode_logTimeOnly()"); - init(new HashMap<String, Object>()); + init(new FilterJsonDescriptorImpl()); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -111,7 +111,7 @@ public class FilterJSONTest { public void testJSONFilterCode_lineNumberOnly() throws Exception { LOG.info("testJSONFilterCode_lineNumberOnly()"); - init(new HashMap<String, Object>()); + init(new FilterJsonDescriptorImpl()); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -131,7 +131,7 @@ public class FilterJSONTest { @Test public void testJSONFilterCode_invalidJson() throws Exception { LOG.info("testJSONFilterCode_invalidJson()"); - init(new HashMap<String, Object>()); + init(new FilterJsonDescriptorImpl()); String inputStr="invalid json"; try{ filterJson.apply(inputStr,new InputMarker(null, null, 0)); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java index 05647e6..ae978fb 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/FilterKeyValueTest.java @@ -18,10 +18,11 @@ package org.apache.ambari.logfeeder.filter; -import java.util.HashMap; import java.util.Map; import org.apache.ambari.logfeeder.output.OutputManager; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.FilterKeyValueDescriptorImpl; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.log4j.Logger; import org.easymock.Capture; @@ -41,12 +42,12 @@ public class FilterKeyValueTest { private OutputManager mockOutputManager; private Capture<Map<String, Object>> capture; - public void init(Map<String, Object> config) throws Exception { + public void init(FilterKeyValueDescriptor filterKeyValueDescriptor) throws Exception { mockOutputManager = EasyMock.strictMock(OutputManager.class); capture = EasyMock.newCapture(CaptureType.LAST); filterKeyValue = new FilterKeyValue(); - filterKeyValue.loadConfig(config); + filterKeyValue.loadConfig(filterKeyValueDescriptor); filterKeyValue.setOutputManager(mockOutputManager); filterKeyValue.init(); } @@ -55,11 +56,10 @@ public class FilterKeyValueTest { public void testFilterKeyValue_extraction() throws Exception { LOG.info("testFilterKeyValue_extraction()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("source_field", "keyValueField"); - config.put("field_split", "&"); - // using default value split: - init(config); + FilterKeyValueDescriptorImpl filterKeyValueDescriptor = new FilterKeyValueDescriptorImpl(); + filterKeyValueDescriptor.setSourceField("keyValueField"); + filterKeyValueDescriptor.setFieldSplit("&"); + init(filterKeyValueDescriptor); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -80,11 +80,11 @@ public class FilterKeyValueTest { public void testFilterKeyValue_extractionWithBorders() throws Exception { LOG.info("testFilterKeyValue_extractionWithBorders()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("source_field", "keyValueField"); - config.put("field_split", "&"); - config.put("value_borders", "()"); - init(config); + FilterKeyValueDescriptorImpl filterKeyValueDescriptor = new FilterKeyValueDescriptorImpl(); + filterKeyValueDescriptor.setSourceField("keyValueField"); + filterKeyValueDescriptor.setFieldSplit("&"); + filterKeyValueDescriptor.setValueBorders("()"); + init(filterKeyValueDescriptor); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall(); @@ -105,10 +105,9 @@ public class FilterKeyValueTest { public void testFilterKeyValue_missingSourceField() throws Exception { LOG.info("testFilterKeyValue_missingSourceField()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("field_split", "&"); - // using default value split: = - init(config); + FilterKeyValueDescriptorImpl filterKeyValueDescriptor = new FilterKeyValueDescriptorImpl(); + filterKeyValueDescriptor.setFieldSplit("&"); + init(filterKeyValueDescriptor); mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); EasyMock.expectLastCall().anyTimes(); @@ -124,10 +123,10 @@ public class FilterKeyValueTest { public void testFilterKeyValue_noSourceFieldPresent() throws Exception { LOG.info("testFilterKeyValue_noSourceFieldPresent()"); - Map<String, Object> config = new HashMap<String, Object>(); - config.put("source_field", "keyValueField"); - config.put("field_split", "&"); - init(config); + FilterKeyValueDescriptorImpl filterKeyValueDescriptor = new FilterKeyValueDescriptorImpl(); + filterKeyValueDescriptor.setSourceField("keyValueField"); + filterKeyValueDescriptor.setFieldSplit("&"); + init(filterKeyValueDescriptor); // using default value split: = mockOutputManager.write(EasyMock.capture(capture), EasyMock.anyObject(InputMarker.class)); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java index 522f6bb..3a5f31e 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/input/InputFileTest.java @@ -22,12 +22,11 @@ import java.io.File; import java.io.IOException; import java.nio.charset.Charset; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import org.apache.ambari.logfeeder.filter.Filter; import org.apache.ambari.logfeeder.input.InputMarker; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputFileDescriptorImpl; import org.apache.commons.io.FileUtils; import org.apache.log4j.Logger; import org.easymock.EasyMock; @@ -78,15 +77,14 @@ public class InputFileTest { } public void init(String path) throws Exception { - Map<String, Object> config = new HashMap<String, Object>(); - config.put("source", "file"); - config.put("tail", "true"); - config.put("gen_event_md5", "true"); - config.put("start_position", "beginning"); - - config.put("type", "hdfs_datanode"); - config.put("rowtype", "service"); - config.put("path", path); + InputFileDescriptorImpl inputFileDescriptor = new InputFileDescriptorImpl(); + inputFileDescriptor.setSource("file"); + inputFileDescriptor.setTail(true); + inputFileDescriptor.setGenEventMd5(true); + inputFileDescriptor.setStartPosition("beginning"); + inputFileDescriptor.setType("hdfs_datanode"); + inputFileDescriptor.setRowtype("service"); + inputFileDescriptor.setPath(path); Filter capture = new Filter() { @Override @@ -104,7 +102,7 @@ public class InputFileTest { }; inputFile = new InputFile(); - inputFile.loadConfig(config); + inputFile.loadConfig(inputFileDescriptor); inputFile.addFilter(capture); inputFile.init(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java index 44314c6..4123dad 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/logconfig/LogConfigHandlerTest.java @@ -21,13 +21,10 @@ package org.apache.ambari.logfeeder.logconfig; import java.util.Arrays; import java.util.Collections; import java.util.Date; -import java.util.HashMap; -import java.util.Map; import static org.easymock.EasyMock.*; import static org.junit.Assert.*; -import org.apache.ambari.logfeeder.common.LogFeederConstants; import org.apache.ambari.logfeeder.input.Input; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.loglevelfilter.FilterLogData; @@ -36,6 +33,7 @@ import org.apache.ambari.logfeeder.util.LogFeederUtil; import org.apache.ambari.logsearch.config.api.LogSearchConfig; import org.apache.ambari.logsearch.config.api.model.loglevelfilter.LogLevelFilter; import org.apache.commons.lang.time.DateUtils; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl; import org.junit.BeforeClass; import org.junit.Test; @@ -44,16 +42,18 @@ public class LogConfigHandlerTest { private static InputMarker inputMarkerAudit; private static InputMarker inputMarkerService; static { - Map<String, Object> auditMap = new HashMap<String, Object>(); - auditMap.put(LogFeederConstants.ROW_TYPE, "audit"); + InputDescriptorImpl auditInputDescriptor = new InputDescriptorImpl() {}; + auditInputDescriptor.setRowtype("audit"); + Input auditInput = strictMock(Input.class); - expect(auditInput.getConfigs()).andReturn(auditMap).anyTimes(); + expect(auditInput.getInputDescriptor()).andReturn(auditInputDescriptor).anyTimes(); inputMarkerAudit = new InputMarker(auditInput, null, 0); - Map<String, Object> serviceMap = new HashMap<String, Object>(); - serviceMap.put(LogFeederConstants.ROW_TYPE, "service"); + InputDescriptorImpl serviceInputDescriptor = new InputDescriptorImpl() {}; + serviceInputDescriptor.setRowtype("service"); + Input serviceInput = strictMock(Input.class); - expect(serviceInput.getConfigs()).andReturn(serviceMap).anyTimes(); + expect(serviceInput.getInputDescriptor()).andReturn(serviceInputDescriptor).anyTimes(); inputMarkerService = new InputMarker(serviceInput, null, 0); replay(auditInput, serviceInput); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java index 8beecda..0a0a9fd 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperDateTest.java @@ -25,6 +25,7 @@ import java.util.HashMap; import java.util.Map; import org.apache.ambari.logfeeder.common.LogFeederConstants; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapDateDescriptorImpl; import org.apache.commons.lang3.time.DateUtils; import org.apache.log4j.Logger; import org.junit.Test; @@ -40,11 +41,11 @@ public class MapperDateTest { public void testMapperDate_epoch() { LOG.info("testMapperDate_epoch()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("target_date_pattern", "epoch"); + MapDateDescriptorImpl mapDateDescriptor = new MapDateDescriptorImpl(); + mapDateDescriptor.setTargetDatePattern("epoch"); MapperDate mapperDate = new MapperDate(); - assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapDateDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); @@ -61,11 +62,11 @@ public class MapperDateTest { public void testMapperDate_pattern() throws Exception { LOG.info("testMapperDate_pattern()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("target_date_pattern", "yyyy-MM-dd HH:mm:ss.SSS"); + MapDateDescriptorImpl mapDateDescriptor = new MapDateDescriptorImpl(); + mapDateDescriptor.setTargetDatePattern("yyyy-MM-dd HH:mm:ss.SSS"); MapperDate mapperDate = new MapperDate(); - assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapDateDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); String dateString = "2016-04-08 15:55:23.548"; @@ -80,44 +81,35 @@ public class MapperDateTest { } @Test - public void testMapperDate_configNotMap() { - LOG.info("testMapperDate_configNotMap()"); - - MapperDate mapperDate = new MapperDate(); - assertFalse("Was able to initialize!", mapperDate.init(null, "someField", null, "")); - } - - @Test public void testMapperDate_noDatePattern() { LOG.info("testMapperDate_noDatePattern()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("some_param", "some_value"); + MapDateDescriptorImpl mapDateDescriptor = new MapDateDescriptorImpl(); MapperDate mapperDate = new MapperDate(); - assertFalse("Was able to initialize!", mapperDate.init(null, "someField", null, mapConfigs)); + assertFalse("Was not able to initialize!", mapperDate.init(null, "someField", null, mapDateDescriptor)); } @Test public void testMapperDate_notParsableDatePattern() { LOG.info("testMapperDate_notParsableDatePattern()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("target_date_pattern", "not_parsable_content"); + MapDateDescriptorImpl mapDateDescriptor = new MapDateDescriptorImpl(); + mapDateDescriptor.setTargetDatePattern("not_parsable_content"); MapperDate mapperDate = new MapperDate(); - assertFalse("Was able to initialize!", mapperDate.init(null, "someField", null, mapConfigs)); + assertFalse("Was not able to initialize!", mapperDate.init(null, "someField", null, mapDateDescriptor)); } @Test public void testMapperDate_invalidEpochValue() { LOG.info("testMapperDate_invalidEpochValue()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("target_date_pattern", "epoch"); + MapDateDescriptorImpl mapDateDescriptor = new MapDateDescriptorImpl(); + mapDateDescriptor.setTargetDatePattern("epoch"); MapperDate mapperDate = new MapperDate(); - assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapDateDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); String invalidValue = "abc"; @@ -131,11 +123,11 @@ public class MapperDateTest { public void testMapperDate_invalidDateStringValue() { LOG.info("testMapperDate_invalidDateStringValue()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("target_date_pattern", "yyyy-MM-dd HH:mm:ss.SSS"); + MapDateDescriptorImpl mapDateDescriptor = new MapDateDescriptorImpl(); + mapDateDescriptor.setTargetDatePattern("yyyy-MM-dd HH:mm:ss.SSS"); MapperDate mapperDate = new MapperDate(); - assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperDate.init(null, "someField", null, mapDateDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); String invalidValue = "abc"; http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java index 108c96e..4899dfc 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldCopyTest.java @@ -21,6 +21,7 @@ package org.apache.ambari.logfeeder.mapper; import java.util.HashMap; import java.util.Map; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapFieldCopyDescriptorImpl; import org.apache.log4j.Logger; import org.junit.Test; @@ -35,11 +36,11 @@ public class MapperFieldCopyTest { public void testMapperFieldCopy_copyField() { LOG.info("testMapperFieldCopy_copyField()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("copy_name", "someOtherField"); + MapFieldCopyDescriptorImpl mapFieldCopyDescriptor = new MapFieldCopyDescriptorImpl(); + mapFieldCopyDescriptor.setCopyName("someOtherField"); MapperFieldCopy mapperFieldCopy = new MapperFieldCopy(); - assertTrue("Could not initialize!", mapperFieldCopy.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperFieldCopy.init(null, "someField", null, mapFieldCopyDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); jsonObj.put("someField", "someValue"); @@ -52,20 +53,12 @@ public class MapperFieldCopyTest { } @Test - public void testMapperFielCopy_configNotMap() { - LOG.info("testMapperFieldCopy_configNotMap()"); - - MapperFieldCopy mapperFieldCopy = new MapperFieldCopy(); - assertFalse("Was able to initialize!", mapperFieldCopy.init(null, "someField", null, "")); - } - - @Test public void testMapperFieldCopy_noNewFieldName() { LOG.info("testMapperFieldCopy_noNewFieldName()"); - Map<String, Object> mapConfigs = new HashMap<>(); + MapFieldCopyDescriptorImpl mapFieldCopyDescriptor = new MapFieldCopyDescriptorImpl(); MapperFieldCopy mapperFieldCopy = new MapperFieldCopy(); - assertFalse("Was able to initialize!", mapperFieldCopy.init(null, "someField", null, mapConfigs)); + assertFalse("Was not able to initialize!", mapperFieldCopy.init(null, "someField", null, mapFieldCopyDescriptor)); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java index 8ecaad1..74b88fc 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldNameTest.java @@ -21,6 +21,7 @@ package org.apache.ambari.logfeeder.mapper; import java.util.HashMap; import java.util.Map; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapFieldNameDescriptorImpl; import org.apache.log4j.Logger; import org.junit.Test; @@ -35,11 +36,11 @@ public class MapperFieldNameTest { public void testMapperFieldName_replaceField() { LOG.info("testMapperFieldName_replaceField()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("new_fieldname", "someOtherField"); + MapFieldNameDescriptorImpl mapFieldNameDescriptor = new MapFieldNameDescriptorImpl(); + mapFieldNameDescriptor.setNewFieldName("someOtherField"); MapperFieldName mapperFieldName = new MapperFieldName(); - assertTrue("Could not initialize!", mapperFieldName.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperFieldName.init(null, "someField", null, mapFieldNameDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); jsonObj.put("someField", "someValue"); @@ -52,20 +53,12 @@ public class MapperFieldNameTest { } @Test - public void testMapperFieldName_configNotMap() { - LOG.info("testMapperFieldName_configNotMap()"); - - MapperFieldName mapperFieldName = new MapperFieldName(); - assertFalse("Was able to initialize!", mapperFieldName.init(null, "someField", null, "")); - } - - @Test public void testMapperFieldName_noNewFieldName() { LOG.info("testMapperFieldName_noNewFieldName()"); - Map<String, Object> mapConfigs = new HashMap<>(); + MapFieldNameDescriptorImpl mapFieldNameDescriptor = new MapFieldNameDescriptorImpl(); MapperFieldName mapperFieldName = new MapperFieldName(); - assertFalse("Was able to initialize!", mapperFieldName.init(null, "someField", null, mapConfigs)); + assertFalse("Was able to initialize!", mapperFieldName.init(null, "someField", null, mapFieldNameDescriptor)); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java index fce4308..1a33740 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/mapper/MapperFieldValueTest.java @@ -21,6 +21,7 @@ package org.apache.ambari.logfeeder.mapper; import java.util.HashMap; import java.util.Map; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.MapFieldValueDescriptorImpl; import org.apache.log4j.Logger; import org.junit.Test; @@ -35,12 +36,12 @@ public class MapperFieldValueTest { public void testMapperFieldValue_replaceValue() { LOG.info("testMapperFieldValue_replaceValue()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("pre_value", "someValue"); - mapConfigs.put("post_value", "someOtherValue"); + MapFieldValueDescriptorImpl mapFieldValueDescriptor = new MapFieldValueDescriptorImpl(); + mapFieldValueDescriptor.setPreValue("someValue"); + mapFieldValueDescriptor.setPostValue("someOtherValue"); MapperFieldValue mapperFieldValue = new MapperFieldValue(); - assertTrue("Could not initialize!", mapperFieldValue.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperFieldValue.init(null, "someField", null, mapFieldValueDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); @@ -52,33 +53,25 @@ public class MapperFieldValueTest { } @Test - public void testMapperFieldValue_configNotMap() { - LOG.info("testMapperFieldValue_configNotMap()"); - - MapperFieldValue mapperFieldValue = new MapperFieldValue(); - assertFalse("Was able to initialize!", mapperFieldValue.init(null, "someField", null, "")); - } - - @Test public void testMapperFieldValue_noPostValue() { LOG.info("testMapperFieldValue_noPostValue()"); - Map<String, Object> mapConfigs = new HashMap<>(); + MapFieldValueDescriptorImpl mapFieldValueDescriptor = new MapFieldValueDescriptorImpl(); MapperFieldValue mapperFieldValue = new MapperFieldValue(); - assertFalse("Was able to initialize!", mapperFieldValue.init(null, "someField", null, mapConfigs)); + assertFalse("Was not able to initialize!", mapperFieldValue.init(null, "someField", null, mapFieldValueDescriptor)); } @Test public void testMapperFieldValue_noPreValueFound() { LOG.info("testMapperFieldValue_noPreValueFound()"); - Map<String, Object> mapConfigs = new HashMap<>(); - mapConfigs.put("pre_value", "someValue"); - mapConfigs.put("post_value", "someOtherValue"); + MapFieldValueDescriptorImpl mapFieldValueDescriptor = new MapFieldValueDescriptorImpl(); + mapFieldValueDescriptor.setPreValue("someValue"); + mapFieldValueDescriptor.setPostValue("someOtherValue"); MapperFieldValue mapperFieldValue = new MapperFieldValue(); - assertTrue("Could not initialize!", mapperFieldValue.init(null, "someField", null, mapConfigs)); + assertTrue("Could not initialize!", mapperFieldValue.init(null, "someField", null, mapFieldValueDescriptor)); Map<String, Object> jsonObj = new HashMap<>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java index 1ccc319..6e108ab 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputLineFilterTest.java @@ -21,6 +21,8 @@ package org.apache.ambari.logfeeder.output; import org.apache.ambari.logfeeder.common.LogFeederConstants; import org.apache.ambari.logfeeder.input.Input; import org.apache.ambari.logfeeder.input.cache.LRUCache; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl; import org.easymock.EasyMock; import org.junit.Before; import org.junit.Test; @@ -49,7 +51,7 @@ public class OutputLineFilterTest { public void testApplyWithFilterOutByDedupInterval() { // GIVEN EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 100L, false)); - EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs()); + EasyMock.expect(inputMock.getInputDescriptor()).andReturn(generateInputDescriptor()); EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD); EasyMock.replay(inputMock); // WHEN @@ -63,7 +65,7 @@ public class OutputLineFilterTest { public void testApplyDoNotFilterOutDataByDedupInterval() { // GIVEN EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 10L, false)); - EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs()); + EasyMock.expect(inputMock.getInputDescriptor()).andReturn(generateInputDescriptor()); EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD); EasyMock.replay(inputMock); // WHEN @@ -77,7 +79,7 @@ public class OutputLineFilterTest { public void testApplyWithFilterOutByDedupLast() { // GIVEN EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 10L, true)); - EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs()); + EasyMock.expect(inputMock.getInputDescriptor()).andReturn(generateInputDescriptor()); EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD); EasyMock.replay(inputMock); // WHEN @@ -91,7 +93,7 @@ public class OutputLineFilterTest { public void testApplyDoNotFilterOutDataByDedupLast() { // GIVEN EasyMock.expect(inputMock.getCache()).andReturn(createLruCache("myMessage2", 10L, true)); - EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs()); + EasyMock.expect(inputMock.getInputDescriptor()).andReturn(generateInputDescriptor()); EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD); EasyMock.replay(inputMock); // WHEN @@ -117,7 +119,7 @@ public class OutputLineFilterTest { public void testApplyWithoutInMemoryTimestamp() { // GIVEN EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 100L, true)); - EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs()); + EasyMock.expect(inputMock.getInputDescriptor()).andReturn(generateInputDescriptor()); EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD); EasyMock.replay(inputMock); Map<String, Object> lineMap = generateLineMap(); @@ -133,7 +135,7 @@ public class OutputLineFilterTest { public void testApplyWithoutLogMessage() { // GIVEN EasyMock.expect(inputMock.getCache()).andReturn(createLruCache(DEFAULT_DUMMY_MESSAGE, 100L, true)); - EasyMock.expect(inputMock.getConfigs()).andReturn(generateInputConfigs()); + EasyMock.expect(inputMock.getInputDescriptor()).andReturn(generateInputDescriptor()); EasyMock.expect(inputMock.getCacheKeyField()).andReturn(CACHE_KEY_FIELD); EasyMock.replay(inputMock); Map<String, Object> lineMap = generateLineMap(); @@ -152,10 +154,10 @@ public class OutputLineFilterTest { return lineMap; } - private Map<String, Object> generateInputConfigs() { - Map<String, Object> inputConfigs = new HashMap<>(); - inputConfigs.put(LogFeederConstants.ROW_TYPE, "service"); - return inputConfigs; + private InputDescriptor generateInputDescriptor() { + InputDescriptorImpl inputDescriptor = new InputDescriptorImpl() {}; + inputDescriptor.setRowtype("service"); + return inputDescriptor; } private LRUCache createLruCache(String defaultKey, long defaultValue, boolean lastDedupEanabled) { http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java index cf1d25a..5abb720 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputManagerTest.java @@ -32,6 +32,7 @@ import java.util.Map; import org.apache.ambari.logfeeder.input.Input; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.metrics.MetricData; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl; import org.junit.Test; public class OutputManagerTest { @@ -91,15 +92,17 @@ public class OutputManagerTest { Input mockInput = strictMock(Input.class); InputMarker inputMarker = new InputMarker(mockInput, null, 0); + InputDescriptorImpl inputDescriptor = new InputDescriptorImpl() {}; + inputDescriptor.setAddFields(Collections.<String, String> emptyMap()); Output output1 = strictMock(Output.class); Output output2 = strictMock(Output.class); Output output3 = strictMock(Output.class); - expect(mockInput.getContextFields()).andReturn(Collections.<String, String> emptyMap()); + expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor); expect(mockInput.isUseEventMD5()).andReturn(false); expect(mockInput.isGenEventMD5()).andReturn(false); - expect(mockInput.getConfigs()).andReturn(Collections.<String, Object> emptyMap()); + expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor); expect(mockInput.getCache()).andReturn(null); expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3)); @@ -125,12 +128,13 @@ public class OutputManagerTest { Input mockInput = strictMock(Input.class); InputMarker inputMarker = new InputMarker(mockInput, null, 0); + InputDescriptorImpl inputDescriptor = new InputDescriptorImpl() {}; Output output1 = strictMock(Output.class); Output output2 = strictMock(Output.class); Output output3 = strictMock(Output.class); - expect(mockInput.getConfigs()).andReturn(Collections.<String, Object> emptyMap()); + expect(mockInput.getInputDescriptor()).andReturn(inputDescriptor); expect(mockInput.getOutputList()).andReturn(Arrays.asList(output1, output2, output3)); output1.write(jsonString, inputMarker); expectLastCall(); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java index 1872135..7c6aca2 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/output/OutputS3FileTest.java @@ -22,6 +22,7 @@ import org.apache.ambari.logfeeder.input.Input; import org.apache.ambari.logfeeder.input.InputMarker; import org.apache.ambari.logfeeder.output.spool.LogSpooler; import org.apache.ambari.logfeeder.output.spool.LogSpoolerContext; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputDescriptorImpl; import org.junit.Before; import org.junit.Test; @@ -33,7 +34,6 @@ import static org.easymock.EasyMock.*; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; - public class OutputS3FileTest { private Map<String, Object> configMap; @@ -71,8 +71,11 @@ public class OutputS3FileTest { Input input = mock(Input.class); InputMarker inputMarker = new InputMarker(input, null, 0); + InputDescriptorImpl inputDescriptor = new InputDescriptorImpl() {}; + inputDescriptor.setType("hdfs-namenode"); + expect(input.getFilePath()).andReturn("/var/log/hdfs-namenode.log"); - expect(input.getStringValue(OutputS3File.INPUT_ATTRIBUTE_TYPE)).andReturn("hdfs-namenode"); + expect(input.getInputDescriptor()).andReturn(inputDescriptor); final LogSpooler spooler = mock(LogSpooler.class); spooler.add("log event block"); final S3Uploader s3Uploader = mock(S3Uploader.class); @@ -99,8 +102,11 @@ public class OutputS3FileTest { public void shouldReuseSpoolerForSamePath() throws Exception { Input input = mock(Input.class); InputMarker inputMarker = new InputMarker(input, null, 0); + InputDescriptorImpl inputDescriptor = new InputDescriptorImpl() {}; + inputDescriptor.setType("hdfs-namenode"); + expect(input.getFilePath()).andReturn("/var/log/hdfs-namenode.log"); - expect(input.getStringValue(OutputS3File.INPUT_ATTRIBUTE_TYPE)).andReturn("hdfs-namenode"); + expect(input.getInputDescriptor()).andReturn(inputDescriptor); final LogSpooler spooler = mock(LogSpooler.class); spooler.add("log event block1"); spooler.add("log event block2"); @@ -169,8 +175,11 @@ public class OutputS3FileTest { public void shouldUploadFileOnRollover() throws Exception { Input input = mock(Input.class); InputMarker inputMarker = new InputMarker(input, null, 0); + InputDescriptorImpl inputDescriptor = new InputDescriptorImpl() {}; + inputDescriptor.setType("hdfs-namenode"); + expect(input.getFilePath()).andReturn("/var/log/hdfs-namenode.log"); - expect(input.getStringValue(OutputS3File.INPUT_ATTRIBUTE_TYPE)).andReturn("hdfs-namenode"); + expect(input.getInputDescriptor()).andReturn(inputDescriptor); final LogSpooler spooler = mock(LogSpooler.class); spooler.add("log event block1"); final S3Uploader s3Uploader = mock(S3Uploader.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java index 1118233..44d91a9 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ShipperConfigManager.java @@ -21,7 +21,9 @@ package org.apache.ambari.logsearch.manager; import java.util.List; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; import org.apache.ambari.logsearch.configurer.LogSearchConfigConfigurer; +import org.apache.ambari.logsearch.model.common.LSServerInputConfig; import org.apache.ambari.logsearch.model.common.LSServerLogLevelFilterMap; import org.apache.log4j.Logger; @@ -50,8 +52,9 @@ public class ShipperConfigManager extends JsonManagerBase { return LogSearchConfigConfigurer.getConfig().getServices(clusterName); } - public String getInputConfig(String clusterName, String serviceName) { - return LogSearchConfigConfigurer.getConfig().getInputConfig(clusterName, serviceName); + public LSServerInputConfig getInputConfig(String clusterName, String serviceName) { + InputConfig inputConfig = LogSearchConfigConfigurer.getConfig().getInputConfig(clusterName, serviceName); + return new LSServerInputConfig(inputConfig); } public Response createInputConfig(String clusterName, String serviceName, String inputConfig) { http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerConditions.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerConditions.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerConditions.java new file mode 100644 index 0000000..9cd9710 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerConditions.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.Conditions; + +import io.swagger.annotations.ApiModel; + +@ApiModel +public class LSServerConditions { + private LSServerFields fields; + + public LSServerConditions(Conditions conditions) { + this.fields = new LSServerFields(conditions.getFields()); + } + + public LSServerFields getFields() { + return fields; + } + + public void setFields(LSServerFields fields) { + this.fields = fields; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFields.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFields.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFields.java new file mode 100644 index 0000000..5f570da --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFields.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import java.util.Set; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.Fields; + +import io.swagger.annotations.ApiModel; + +@ApiModel +public class LSServerFields { + private Set<String> type; + + public LSServerFields(Fields fields) { + this.type = fields.getType(); + } + + public Set<String> getType() { + return type; + } + + public void setType(Set<String> type) { + this.type = type; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java new file mode 100644 index 0000000..0190c01 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilter.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.PostMapValues; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonInclude.Include; + +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonInclude(Include.NON_NULL) +public abstract class LSServerFilter { + private String filter; + + private LSServerConditions conditions; + + @JsonProperty("sort_order") + private Integer sortOrder; + + private String sourceField; + + @JsonProperty("remove_source_field") + private Boolean removeSourceField; + + private Map<String, List<LSServerPostMapValues>> postMapValues; + + @JsonProperty("is_enabled") + private Boolean isEnabled; + + public LSServerFilter(FilterDescriptor filterDescriptor) { + this.filter = filterDescriptor.getFilter(); + this.conditions = new LSServerConditions(filterDescriptor.getConditions()); + this.sortOrder = filterDescriptor.getSortOrder(); + this.sourceField = filterDescriptor.getSourceField(); + this.removeSourceField = filterDescriptor.isRemoveSourceField(); + + postMapValues = new HashMap<String, List<LSServerPostMapValues>>(); + for (Map.Entry<String, ? extends List<? extends PostMapValues>> e : filterDescriptor.getPostMapValues().entrySet()) { + List<LSServerPostMapValues> lsServerPostMapValues = new ArrayList<>(); + for (PostMapValues pmv : e.getValue()) { + lsServerPostMapValues.add(new LSServerPostMapValues(pmv)); + } + postMapValues.put(e.getKey(), lsServerPostMapValues); + } + + this.isEnabled = filterDescriptor.isEnabled(); + } + + public String getFilter() { + return filter; + } + + public void setFilter(String filter) { + this.filter = filter; + } + + public LSServerConditions getConditions() { + return conditions; + } + + public void setConditions(LSServerConditions conditions) { + this.conditions = conditions; + } + + public Integer getSortOrder() { + return sortOrder; + } + + public void setSortOrder(Integer sortOrder) { + this.sortOrder = sortOrder; + } + + public String getSourceField() { + return sourceField; + } + + public void setSourceField(String sourceField) { + this.sourceField = sourceField; + } + + public Boolean getRemoveSourceField() { + return removeSourceField; + } + + public void setRemoveSourceField(Boolean removeSourceField) { + this.removeSourceField = removeSourceField; + } + + public Map<String, List<LSServerPostMapValues>> getPostMapValues() { + return postMapValues; + } + + public void setPostMapValues(Map<String, List<LSServerPostMapValues>> postMapValues) { + this.postMapValues = postMapValues; + } + + public Boolean getIsEnabled() { + return isEnabled; + } + + public void setIsEnabled(Boolean isEnabled) { + this.isEnabled = isEnabled; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterGrok.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterGrok.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterGrok.java new file mode 100644 index 0000000..a8c4a7a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterGrok.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import io.swagger.annotations.ApiModel; + +@ApiModel +public class LSServerFilterGrok extends LSServerFilter { + @JsonProperty("log4j_format") + private String log4jFormat; + + @JsonProperty("multiline_pattern") + private String multilinePattern; + + @JsonProperty("message_pattern") + private String messagePattern; + + public LSServerFilterGrok(FilterDescriptor filterDescriptor) { + super(filterDescriptor); + if (filterDescriptor instanceof FilterGrokDescriptor) { + FilterGrokDescriptor filterGrokDescriptor = (FilterGrokDescriptor)filterDescriptor; + this.log4jFormat = filterGrokDescriptor.getLog4jFormat(); + this.multilinePattern = filterGrokDescriptor.getMultilinePattern(); + this.messagePattern = filterGrokDescriptor.getMessagePattern(); + } + } + + public String getLog4jFormat() { + return log4jFormat; + } + + public void setLog4jFormat(String log4jFormat) { + this.log4jFormat = log4jFormat; + } + + public String getMultilinePattern() { + return multilinePattern; + } + + public void setMultilinePattern(String multilinePattern) { + this.multilinePattern = multilinePattern; + } + + public String getMessagePattern() { + return messagePattern; + } + + public void setMessagePattern(String messagePattern) { + this.messagePattern = messagePattern; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterJson.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterJson.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterJson.java new file mode 100644 index 0000000..3c0ed17 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterJson.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; + +import io.swagger.annotations.ApiModel; + +@ApiModel +public class LSServerFilterJson extends LSServerFilter { + public LSServerFilterJson(FilterDescriptor filterDescriptor) { + super(filterDescriptor); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterKeyValue.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterKeyValue.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterKeyValue.java new file mode 100644 index 0000000..dcee25d --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerFilterKeyValue.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import io.swagger.annotations.ApiModel; + +@ApiModel +public class LSServerFilterKeyValue extends LSServerFilter { + @JsonProperty("field_split") + private String fieldSplit; + + @JsonProperty("value_split") + private String valueSplit; + + @JsonProperty("value_borders") + private String valueBorders; + + public LSServerFilterKeyValue(FilterDescriptor filterDescriptor) { + super(filterDescriptor); + FilterKeyValueDescriptor filterKeyValueDescriptor = (FilterKeyValueDescriptor)filterDescriptor; + this.fieldSplit = filterKeyValueDescriptor.getFieldSplit(); + this.valueSplit = filterKeyValueDescriptor.getValueSplit(); + this.valueBorders = filterKeyValueDescriptor.getValueBorders(); + } + + public String getFieldSplit() { + return fieldSplit; + } + + public void setFieldSplit(String fieldSplit) { + this.fieldSplit = fieldSplit; + } + + public String getValueSplit() { + return valueSplit; + } + + public void setValueSplit(String valueSplit) { + this.valueSplit = valueSplit; + } + + public String getValueBorders() { + return valueBorders; + } + + public void setValueBorders(String valueBorders) { + this.valueBorders = valueBorders; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java new file mode 100644 index 0000000..fe83fe4 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInput.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import java.util.Map; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.annotation.JsonProperty; + +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonInclude(Include.NON_NULL) +public abstract class LSServerInput { + private final String type; + private final String rowtype; + private final String path; + + @JsonProperty("add_fields") + private final Map<String, String> addFields; + + private final String source; + private final Boolean tail; + + @JsonProperty("gen_event_md5") + private final Boolean genEventMd5; + + @JsonProperty("use_event_md5_as_id") + private final Boolean useEventMd5AsId; + + @JsonProperty("start_position") + private final String startPosition; + + @JsonProperty("cache_enabled") + private final Boolean cacheEnabled; + + @JsonProperty("cache_key_field") + private final String cacheKeyField; + + @JsonProperty("cache_last_dedup_enabled") + private final Boolean cacheLastDedupEnabled; + + @JsonProperty("cache_size") + private final Integer cacheSize; + + @JsonProperty("cache_dedup_interval") + private final Long cacheDedupInterval; + + @JsonProperty("is_enabled") + private final Boolean isEnabled; + + public LSServerInput(InputDescriptor inputDescriptor) { + this.type = inputDescriptor.getType(); + this.rowtype = inputDescriptor.getRowtype(); + this.path = inputDescriptor.getPath(); + this.addFields = inputDescriptor.getAddFields(); + this.source = inputDescriptor.getSource(); + this.tail = inputDescriptor.isTail(); + this.genEventMd5 = inputDescriptor.isGenEventMd5(); + this.useEventMd5AsId = inputDescriptor.isUseEventMd5AsId(); + this.startPosition = inputDescriptor.getStartPosition(); + this.cacheEnabled = inputDescriptor.isCacheEnabled(); + this.cacheKeyField = inputDescriptor.getCacheKeyField(); + this.cacheLastDedupEnabled = inputDescriptor.getCacheLastDedupEnabled(); + this.cacheSize = inputDescriptor.getCacheSize(); + this.cacheDedupInterval = inputDescriptor.getCacheDedupInterval(); + this.isEnabled = inputDescriptor.isEnabled(); + } + + public String getType() { + return type; + } + + public String getRowtype() { + return rowtype; + } + + public String getPath() { + return path; + } + + public Map<String, String> getAddFields() { + return addFields; + } + + public String getSource() { + return source; + } + + public Boolean getTail() { + return tail; + } + + public Boolean getGenEventMd5() { + return genEventMd5; + } + + public Boolean getUseEventMd5AsId() { + return useEventMd5AsId; + } + + public String getStartPosition() { + return startPosition; + } + + public Boolean getCacheEnabled() { + return cacheEnabled; + } + + public String getCacheKeyField() { + return cacheKeyField; + } + + public Boolean getCacheLastDedupEnabled() { + return cacheLastDedupEnabled; + } + + public Integer getCacheSize() { + return cacheSize; + } + + public Long getCacheDedupInterval() { + return cacheDedupInterval; + } + + public Boolean getIsEnabled() { + return isEnabled; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/fd4a7a46/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInputConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInputConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInputConfig.java new file mode 100644 index 0000000..e3dc0d1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/common/LSServerInputConfig.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.model.common; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterGrokDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterJsonDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.FilterKeyValueDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputConfig; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputFileBaseDescriptor; +import org.apache.ambari.logsearch.config.api.model.inputconfig.InputS3FileDescriptor; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; + +@ApiModel +public class LSServerInputConfig { + @ApiModelProperty + private List<LSServerInput> input; + + @ApiModelProperty + private List<LSServerFilter> filter; + + public LSServerInputConfig(InputConfig inputConfig) { + input = new ArrayList<>(); + for (InputDescriptor inputDescriptor : inputConfig.getInput()) { + if (inputDescriptor instanceof InputFileBaseDescriptor) { + LSServerInput inputItem = new LSServerInputFile(inputDescriptor); + input.add(inputItem); + } else if (inputDescriptor instanceof InputS3FileDescriptor) { + LSServerInput inputItem = new LSServerInputS3File(inputDescriptor); + input.add(inputItem); + } + } + + filter = new ArrayList<>(); + for (FilterDescriptor filterDescriptor : inputConfig.getFilter()) { + if (filterDescriptor instanceof FilterGrokDescriptor) { + LSServerFilter filterItem = new LSServerFilterGrok(filterDescriptor); + filter.add(filterItem); + } else if (filterDescriptor instanceof FilterKeyValueDescriptor) { + LSServerFilter filterItem = new LSServerFilterKeyValue(filterDescriptor); + filter.add(filterItem); + } else if (filterDescriptor instanceof FilterJsonDescriptor) { + LSServerFilter filterItem = new LSServerFilterJson(filterDescriptor); + filter.add(filterItem); + } + } + } + + public List<LSServerInput> getInput() { + return input; + } + + public void setInput(List<LSServerInput> input) { + this.input = input; + } + + public List<LSServerFilter> getFilter() { + return filter; + } + + public void setFilter(List<LSServerFilter> filter) { + this.filter = filter; + } +}