This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch branch-feature-AMBARI-22842 in repository https://gitbox.apache.org/repos/asf/ambari.git
commit 794bc557e4910815a51b27c8dab015beda6ea4db Author: Oliver Szabo <oleew...@gmail.com> AuthorDate: Wed Jan 24 17:47:32 2018 +0100 AMBARI-22842. Provide field / component UI metadata from backend --- ambari-logsearch/.gitignore | 10 + .../api/model/inputconfig/InputDescriptor.java | 2 + .../inputconfig/impl/InputDescriptorImpl.java | 20 +- .../ambari/logfeeder/plugin/filter/Filter.java | 2 +- .../ambari/logfeeder/plugin/input/Input.java | 2 +- .../ambari/logfeeder/output/OutputManagerImpl.java | 3 + .../shipper-conf/input.config-sample.json | 3 +- .../configsets/hadoop_logs/conf/managed-schema | 1 + .../common/HadoopServiceConfigHelper.java | 80 ---- .../logsearch/common/LabelFallbackHandler.java | 110 +++++ .../logsearch/common/LogSearchConstants.java | 22 + .../logsearch/common/PropertiesSplitter.java | 93 ++++ .../ambari/logsearch/conf/UIMappingConfig.java | 431 ++++++++++++++++++ .../apache/ambari/logsearch/doc/DocConstants.java | 1 - .../ambari/logsearch/manager/AuditLogsManager.java | 75 +++- .../logsearch/manager/ServiceLogsManager.java | 55 ++- .../metadata/AuditFieldMetadataResponse.java} | 41 +- .../model/metadata/ComponentMetadata.java} | 44 +- .../logsearch/model/metadata/FieldMetadata.java} | 59 ++- .../logsearch/model/metadata/Filterable.java} | 33 +- .../logsearch/model/metadata/Groupable.java} | 33 +- .../logsearch/model/metadata/Labelable.java} | 33 +- .../ambari/logsearch/model/metadata/Metadata.java} | 44 +- .../metadata/ServiceComponentMetadataWrapper.java} | 45 +- .../ambari/logsearch/model/metadata/Visible.java} | 33 +- .../logsearch/model/response/ServiceLogData.java | 5 + .../ambari/logsearch/rest/AuditLogsResource.java | 15 +- .../ambari/logsearch/rest/ServiceLogsResource.java | 17 +- .../logsearch/solr/ResponseDataGenerator.java | 73 ++- .../ambari/logsearch/solr/SolrConstants.java | 1 + .../logsearch/solr/model/SolrServiceLogData.java | 13 + .../src/main/resources/HadoopServiceConfig.json | 488 --------------------- .../logsearch/common/LabelFallbackHandlerTest.java | 129 ++++++ .../logsearch/common/PropertiesSplitterTest.java | 123 ++++++ .../src/test/resources/HadoopServiceConfig.json | 17 - .../shipper-conf/input.config-ambari.json | 18 +- .../logfeeder/shipper-conf/input.config-hst.json | 3 +- .../shipper-conf/input.config-secure_log.json | 3 +- .../shipper-conf/input.config-system_message.json | 3 +- .../shipper-conf/input.config-zookeeper.json | 3 +- 40 files changed, 1284 insertions(+), 902 deletions(-) diff --git a/ambari-logsearch/.gitignore b/ambari-logsearch/.gitignore new file mode 100644 index 0000000..7ea6a1f --- /dev/null +++ b/ambari-logsearch/.gitignore @@ -0,0 +1,10 @@ +target +.settings +.classpath +.project +/bin/ +node_modules/ +logs/ +node/ +*.pid + diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java index 82e9504..d3a43c1 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java @@ -49,4 +49,6 @@ public interface InputDescriptor { Long getCacheDedupInterval(); Boolean isEnabled(); + + String getGroup(); } diff --git a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java index 2e3405d..0994b9f 100644 --- a/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java +++ b/ambari-logsearch/ambari-logsearch-config-zookeeper/src/main/java/org/apache/ambari/logsearch/config/zookeeper/model/inputconfig/impl/InputDescriptorImpl.java @@ -54,6 +54,15 @@ public abstract class InputDescriptorImpl implements InputDescriptor { private String rowtype; @ShipperConfigElementDescription( + path = "/input/[]/group", + type = "string", + description = "Group of the input type.", + examples = {"Ambari", "Yarn"} + ) + @Expose + private String group; + + @ShipperConfigElementDescription( path = "/input/[]/path", type = "string", description = "The path of the source, may contain '*' characters too.", @@ -179,8 +188,6 @@ public abstract class InputDescriptorImpl implements InputDescriptor { @SerializedName("is_enabled") private Boolean isEnabled; - private Map<String, Object> allProperties; - public String getType() { return type; } @@ -292,4 +299,13 @@ public abstract class InputDescriptorImpl implements InputDescriptor { public void setIsEnabled(Boolean isEnabled) { this.isEnabled = isEnabled; } + + @Override + public String getGroup() { + return this.group; + } + + public void setGroup(String group) { + this.group = group; + } } diff --git a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/filter/Filter.java b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/filter/Filter.java index 9bf69b9..e2180d7 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/filter/Filter.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/filter/Filter.java @@ -162,7 +162,7 @@ public abstract class Filter<PROP_TYPE extends LogFeederProperties> extends Conf @Override public boolean isEnabled() { - return filterDescriptor.isEnabled(); + return filterDescriptor.isEnabled() != null ? filterDescriptor.isEnabled() : true; } @Override diff --git a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java index e2158bc..368f7d5 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder-plugin-api/src/main/java/org/apache/ambari/logfeeder/plugin/input/Input.java @@ -154,7 +154,7 @@ public abstract class Input<PROP_TYPE extends LogFeederProperties, INPUT_MARKER @Override public boolean isEnabled() { - return inputDescriptor.isEnabled(); + return inputDescriptor.isEnabled() != null ? inputDescriptor.isEnabled() : true; } @Override diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java index 47135b6..e7e0eef 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputManagerImpl.java @@ -149,6 +149,9 @@ public class OutputManagerImpl extends OutputManager { if (jsonObj.get("event_count") == null) { jsonObj.put("event_count", new Integer(1)); } + if (StringUtils.isNotBlank(input.getInputDescriptor().getGroup())) { + jsonObj.put("group", input.getInputDescriptor().getGroup()); + } if (inputMarker.getAllProperties().containsKey("line_number") && (Integer) inputMarker.getAllProperties().get("line_number") > 0) { jsonObj.put("logfile_line_number", inputMarker.getAllProperties().get("line_number")); diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json index e54c974..3951e4b 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log-samples/shipper-conf/input.config-sample.json @@ -3,7 +3,8 @@ { "type": "service_sample", "rowtype": "service", - "path": "target/classes/log-samples/logs/service_sample.txt" + "path": "target/classes/log-samples/logs/service_sample.txt", + "group": "Ambari" } ], "filter": [ diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema index 93b2d6b..9fb0cf7 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema +++ b/ambari-logsearch/ambari-logsearch-server/src/main/configsets/hadoop_logs/conf/managed-schema @@ -110,6 +110,7 @@ <field name="text" type="text_std_token_lower_case" multiValued="true" indexed="true" stored="false"/> <field name="thread_name" type="key_lower_case" multiValued="false" omitNorms="false"/> <field name="type" type="key_lower_case" multiValued="false"/> + <field name="group" type="string" multiValued="false" docValues="true" stored="true"/> <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/> <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/> diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/HadoopServiceConfigHelper.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/HadoopServiceConfigHelper.java deleted file mode 100644 index 0e2087f..0000000 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/HadoopServiceConfigHelper.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ambari.logsearch.common; - -import java.io.File; -import java.io.IOException; -import java.util.Iterator; -import java.util.Set; -import java.util.TreeSet; - -import org.apache.ambari.logsearch.util.JSONUtil; -import org.apache.commons.io.FileUtils; -import org.apache.log4j.Logger; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - -import com.google.gson.JsonParseException; - -public class HadoopServiceConfigHelper { - private static final Logger LOG = Logger.getLogger(HadoopServiceConfigHelper.class); - - public static String getHadoopServiceConfigJSON() { - String fileContent = null; - - try { - ClassLoader classLoader = HadoopServiceConfigHelper.class.getClassLoader(); - File file = new File(classLoader.getResource("HadoopServiceConfig.json").getFile()); - fileContent = FileUtils.readFileToString(file); - } catch (IOException e) { - LOG.error("Unable to read HadoopServiceConfig.json", e); - } - - return JSONUtil.isJSONValid(fileContent) ? fileContent : null; - } - - @SuppressWarnings("unchecked") - public static Set<String> getAllLogIds() { - Set<String> logIds = new TreeSet<>(); - - String key = null; - JSONArray componentArray = null; - try { - String hadoopServiceConfigJSON = getHadoopServiceConfigJSON(); - JSONObject hadoopServiceJsonObject = new JSONObject(hadoopServiceConfigJSON).getJSONObject("service"); - Iterator<String> hadoopSerivceKeys = hadoopServiceJsonObject.keys(); - while (hadoopSerivceKeys.hasNext()) { - key = hadoopSerivceKeys.next(); - componentArray = hadoopServiceJsonObject.getJSONObject(key).getJSONArray("components"); - for (int i = 0; i < componentArray.length(); i++) { - JSONObject componentJsonObject = (JSONObject) componentArray.get(i); - String logId = componentJsonObject.getString("name"); - logIds.add(logId); - } - } - } catch (JsonParseException | JSONException je) { - LOG.error("Error parsing JSON. key=" + key + ", componentArray=" + componentArray, je); - return null; - } - - return logIds; - } -} diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LabelFallbackHandler.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LabelFallbackHandler.java new file mode 100644 index 0000000..5549343 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LabelFallbackHandler.java @@ -0,0 +1,110 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +import org.apache.ambari.logsearch.conf.UIMappingConfig; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.text.WordUtils; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.List; + +@Named +public class LabelFallbackHandler { + + private final UIMappingConfig uiMappingConfig; + + @Inject + public LabelFallbackHandler(UIMappingConfig uiMappingConfig) { + this.uiMappingConfig = uiMappingConfig; + } + + public String fallbackIfRequired(String field, String label, boolean replaceUnderscore, + boolean replaceUppercaseInWord, boolean capitalizeAll) { + if (isEnabled() && StringUtils.isBlank(label)) { + return fallback(field,replaceUnderscore, replaceUppercaseInWord, capitalizeAll); + } + return label; + } + + public String fallbackIfRequired(String field, String label, boolean replaceUnderscore, + boolean replaceUppercaseInWord, boolean capitalizeAll, List<String> prefixesToRemove) { + if (isEnabled() && StringUtils.isBlank(label)) { + return fallback(field,replaceUnderscore, replaceUppercaseInWord, capitalizeAll, prefixesToRemove); + } + return label; + } + + public String fallback(String field, boolean replaceUnderscore, boolean replaceUppercaseInWord, boolean capitalizeAll) { + String result = null; + if (StringUtils.isNotBlank(field)) { + if (replaceUppercaseInWord) { + result = capitalize(deCamelCase(field), false); + } + if (replaceUnderscore) { + result = capitalize(deUnderScore(result != null ? result : field), capitalizeAll); + } + } + return result; + } + + public String fallback(String field, boolean replaceUnderscore, boolean replaceUppercaseInWord, boolean capitalizeAll, List<String> prefixesToRemove) { + String fieldWithoutPrefix = null; + if (!CollectionUtils.isEmpty(prefixesToRemove)) { + for (String prefix : prefixesToRemove) { + if (StringUtils.isNotBlank(field) && field.startsWith(prefix)) { + fieldWithoutPrefix = field.substring(prefix.length()); + } + } + } + return fallback(fieldWithoutPrefix != null ? fieldWithoutPrefix : field, replaceUnderscore, replaceUppercaseInWord, capitalizeAll); + } + + private String deUnderScore(String input) { + return input.replaceAll("_", " "); + } + + private String capitalize(String input, boolean capitalizeAll) { + if (capitalizeAll) { + return WordUtils.capitalizeFully(input); + } else { + Character firstLetter = Character.toUpperCase(input.charAt(0)); + return input.length() > 1 ? firstLetter + input.substring(1) : firstLetter.toString(); + } + } + + private String deCamelCase(String input) { + StringBuilder result = new StringBuilder(); + for(int i=0 ; i < input.length() ; i++) { + char c = input.charAt(i); + if(i != 0 && Character.isUpperCase(c)) { + result.append(' '); + } + result.append(c); + } + return result.toString(); + } + + public boolean isEnabled() { + return uiMappingConfig.isLabelFallbackEnabled(); + } + +} diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java index 4722b79..82dfcd5 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java @@ -65,6 +65,28 @@ public class LogSearchConstants { public static final String SHIPPER_CONFIG_API_KEY = "metadata_patterns"; public static final String AUTH_FEATURE_KEY = "auth"; + // service field / component label defaults + public static final String SERVICE_GROUP_LABELS_DEFAULTS = ""; + public static final String SERVICE_COMPONENT_LABELS_DEFAULTS = ""; + public static final String SERVICE_FIELD_LABELS_DEFAULTS = "log_message:Message,type:Component,logtime:Log Time,thread_name:Thread"; + public static final String SERVICE_FIELD_VISIBLE_DEFAULTS = "log_message,level,logtime,type"; + public static final String SERVICE_FIELD_EXCLUDES_DEFAULTS = "id,tags,text,message,seq_num,case_id,bundle_id,rowtype,event_count"; + public static final String SERVICE_FIELD_FILTERABLE_EXLUDE_DEFAULTS = ""; + public static final String SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS = "ws_,sdi_"; + + // audit field / component label defaults + public static final String AUDIT_COMPONENT_LABELS_DEFAULTS = "ambari:Ambari,hdfs:Hdfs,RangerAudit:Ranger"; + public static final String AUDIT_FIELD_LABELS_DEFAULTS = ""; + public static final String AUDIT_FIELD_COMMON_LABELS_DEFAULTS = "enforcer:Access Enforcer,access:Access Type,cliIP:Client Ip,cliType:Client Type," + + "dst:DST,evtTime:Event Time,ip:IP,logtime:Log Time,sess:Session,ugi:UGI,reqUser:User"; + public static final String AUDIT_FIELD_VISIBLE_DEFAULTS = ""; + public static final String AUDIT_FIELD_VISIBLE_COMMON_DEFAULTS = "access,cliIP,evtTime,repo,resource,result,reqUser"; + public static final String AUDIT_FIELD_EXCLUDES_DEFAULTS = ""; + public static final String AUDIT_FIELD_EXCLUDES_COMMON_DEFAULTS = "tags,tags_str,seq_num"; + public static final String AUDIT_FIELD_FILTERABLE_EXCLUDES_DEFAULTS = ""; + public static final String AUDIT_FIELD_FILTERABLE_EXCLUDES_COMMON_DEFAULTS = ""; + public static final String AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS = "ws_,std_"; + //Facet Constant public static final String FACET_FIELD = "facet.field"; public static final String FACET_PIVOT = "facet.pivot"; diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/PropertiesSplitter.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/PropertiesSplitter.java new file mode 100644 index 0000000..57e33e9 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/common/PropertiesSplitter.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +import com.google.common.base.Splitter; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; + +import javax.inject.Named; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Named +public class PropertiesSplitter { + + public List<String> parseList(String listStr) { + return parseList(listStr, ","); + } + + public Map<String, String> parseMap(String mapStr) { + return parseMap(mapStr, ",", ":"); + } + + private List<String> parseList(String listStr, String separator) { + return StringUtils.isNotBlank(listStr) ? Splitter.on(separator).splitToList(listStr) : new ArrayList<>(); + } + + public Map<String, String> parseMap(String mapStr, String separator, String keyValueSeparator) { + Map<String, String> resultMap = new HashMap<>(); + if (StringUtils.isNotBlank(mapStr)) { + List<String> keyValueList = parseList(mapStr, separator); + if (!keyValueList.isEmpty()) { + for (String keyValueElement : keyValueList) { + if (StringUtils.isNotEmpty(keyValueElement)) { + List<String> keyValueElementList = parseList(keyValueElement, keyValueSeparator); + if (!CollectionUtils.isEmpty(keyValueElementList) && keyValueElementList.size() >= 2 + && StringUtils.isNotBlank(keyValueElementList.get(0))) { + resultMap.put(keyValueElementList.get(0), keyValueElementList.get(1)); + } + } + } + } + } + return resultMap; + } + + public Map<String, Map<String, String>> parseMapInMap(String mapInMapStr) { + Map<String, Map<String, String>> mapInMap = new HashMap<>(); + Map<String, String> outerMap = parseMap(mapInMapStr, ";", "#"); + if (!outerMap.isEmpty()) { + for (Map.Entry<String, String> entry : outerMap.entrySet()) { + Map<String, String> keyValueMap = parseMap(entry.getValue()); + if (!keyValueMap.isEmpty()) { + mapInMap.put(entry.getKey(), keyValueMap); + } + } + } + return mapInMap; + } + + public Map<String, List<String>> parseListInMap(String listInMapStr) { + return parseListInMap(listInMapStr, ";", ":", ","); + } + + public Map<String, List<String>> parseListInMap(String listInMapStr, String mapSeparator, String keyValueSeparator, String listSeparator) { + Map<String, List<String>> listInMap = new HashMap<>(); + Map<String, String> typeKeyValueMap = parseMap(listInMapStr, mapSeparator, keyValueSeparator); + for (Map.Entry<String, String> entry : typeKeyValueMap.entrySet()) { + List<String> valuesList = parseList(entry.getValue(), listSeparator); + listInMap.put(entry.getKey(), valuesList); + } + return listInMap; + } + +} diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/UIMappingConfig.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/UIMappingConfig.java new file mode 100644 index 0000000..57abe16 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/conf/UIMappingConfig.java @@ -0,0 +1,431 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.conf; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.config.api.LogSearchPropertyDescription; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_COMPONENT_LABELS_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_COMMON_LABELS_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_EXCLUDES_COMMON_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_EXCLUDES_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FILTERABLE_EXCLUDES_COMMON_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_FILTERABLE_EXCLUDES_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_LABELS_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_VISIBLE_COMMON_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.AUDIT_FIELD_VISIBLE_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.LOGSEARCH_PROPERTIES_FILE; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_FILTERABLE_EXLUDE_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_GROUP_LABELS_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_COMPONENT_LABELS_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_LABELS_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_EXCLUDES_DEFAULTS; +import static org.apache.ambari.logsearch.common.LogSearchConstants.SERVICE_FIELD_VISIBLE_DEFAULTS; + +@Configuration +public class UIMappingConfig { + + @Value("#{propertiesSplitter.parseMap('${logsearch.web.service_logs.group.labels:" + SERVICE_GROUP_LABELS_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.service_logs.group.labels", + description = "Map of serivce group labels", + examples = {"ambari:Ambari,yarn:YARN"}, + defaultValue = SERVICE_GROUP_LABELS_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, String> serviceGroupLabels; + + @Value("#{propertiesSplitter.parseMap('${logsearch.web.service_logs.component.labels:" + SERVICE_COMPONENT_LABELS_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.service_logs.component.labels", + description = "Map of serivce component labels.", + examples = {"ambari_agent:Ambari Agent,ambari_server:Ambari Servcer"}, + defaultValue = SERVICE_COMPONENT_LABELS_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, String> serviceComponentLabels; + + @Value("#{propertiesSplitter.parseMap('${logsearch.web.service_logs.field.labels:" + SERVICE_FIELD_LABELS_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.service_logs.field.labels", + description = "Map of serivce field labels.", + examples = {"log_message:Message,ip:IP Address"}, + defaultValue = SERVICE_FIELD_LABELS_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, String> serviceFieldLabels; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.service_logs.field.excludes:" + SERVICE_FIELD_EXCLUDES_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.service_logs.field.excludes", + description = "List of fields that will be excluded from metadata schema responses.", + examples = {"seq_num,tag"}, + defaultValue = SERVICE_FIELD_EXCLUDES_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> serviceFieldExcludeList; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.service_logs.field.visible:" + SERVICE_FIELD_VISIBLE_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.service_logs.field.visible", + description = "List of fields that will be displayed by default on the UI.", + examples = {"log_message,path,logtime"}, + defaultValue = SERVICE_FIELD_VISIBLE_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> serviceFieldVisibleList; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.service_logs.field.filterable.excludes:" + SERVICE_FIELD_FILTERABLE_EXLUDE_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.service_logs.field.filterable.excludes", + description = "List of fields that will be excluded from filter selection on the UI.", + examples = {"path,method,logger_name"}, + defaultValue = SERVICE_FIELD_FILTERABLE_EXLUDE_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> serviceFieldFilterableExcludesList; + + @Value("#{propertiesSplitter.parseMap('${logsearch.web.audit_logs.component.labels:" + AUDIT_COMPONENT_LABELS_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.component.labels", + description = "Map of component component labels.", + examples = {"ambari:Ambari,RangerAudit:ranger"}, + defaultValue = AUDIT_COMPONENT_LABELS_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, String> auditComponentLabels; + + @Value("#{propertiesSplitter.parseMapInMap('${logsearch.web.audit_logs.field.labels:" + AUDIT_FIELD_LABELS_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.labels", + description = "Map of fields (key-value pairs) labels for different component types.", + examples = {"ambari#reqUser:Ambari User,ws_response:Response;RangerAudit#reqUser:Req User"}, + defaultValue = AUDIT_FIELD_LABELS_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, Map<String, String>> auditFieldLabels; + + @Value("#{propertiesSplitter.parseMap('${logsearch.web.audit_logs.field.common.labels:" + AUDIT_FIELD_COMMON_LABELS_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.common.labels", + description = "Map of fields labels for audits (common).", + examples = {"reqUser:Req User,resp:Response"}, + defaultValue = AUDIT_FIELD_COMMON_LABELS_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, String> auditFieldCommonLabels; + + @Value("#{propertiesSplitter.parseListInMap('${logsearch.web.audit_logs.field.visible:" + AUDIT_FIELD_VISIBLE_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.visible", + description = "List of fields that will be displayed by default on the UI for different audit components.", + examples = {"ambari:reqUser,resp;RangerAudit:reqUser,repo"}, + defaultValue = AUDIT_FIELD_VISIBLE_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, List<String>> auditFieldVisibleleMap; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.audit_logs.field.common.visible:" + AUDIT_FIELD_VISIBLE_COMMON_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.common.visible", + description = "List of fields that will be displayed by default on the UI for every audit components.", + examples = {"reqUser,resp"}, + defaultValue = AUDIT_FIELD_VISIBLE_COMMON_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> auditFieldCommonVisibleList; + + @Value("#{propertiesSplitter.parseListInMap('${logsearch.web.audit_logs.field.excludes:" + AUDIT_FIELD_EXCLUDES_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.excludes", + description = "List of fields that will be excluded from metadata schema responses for different audit components.", + examples = {"ambari:reqUser,resp,hdfs:ws_user,ws_role"}, + defaultValue = AUDIT_FIELD_EXCLUDES_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, List<String>> auditFieldExcludeMap; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.audit_logs.field.common.excludes:" + AUDIT_FIELD_EXCLUDES_COMMON_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.common.excludes", + description = "List of fields that will be excluded from metadata schema responses for every audit components.", + examples = {"reqUser,resp,tag_str"}, + defaultValue = AUDIT_FIELD_EXCLUDES_COMMON_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> auditFieldCommonExcludeList; + + @Value("#{propertiesSplitter.parseListInMap('${logsearch.web.audit_logs.field.filterable.excludes:" + AUDIT_FIELD_FILTERABLE_EXCLUDES_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.filterable.excludes", + description = "List of fields that will be excluded from filter selection on the UI for different audit components.", + examples = {"ambari:tag_str,resp,tag_str;RangerAudit:path,ip"}, + defaultValue = AUDIT_FIELD_FILTERABLE_EXCLUDES_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private Map<String, List<String>> auditFieldFilterableExcludeMap; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.audit_logs.field.common.filterable.common.excludes:" + AUDIT_FIELD_FILTERABLE_EXCLUDES_COMMON_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.common.filterable.common.excludes", + description = "List of fields that will be excluded from filter selection on the UI for every audit components.", + examples = {"tag_str,resp,tag_str"}, + defaultValue = AUDIT_FIELD_FILTERABLE_EXCLUDES_COMMON_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> auditFieldCommonFilterableExcludeList; + + @Value("${logsearch.web.labels.fallback.enabled:true}") + @LogSearchPropertyDescription( + name = "logsearch.web.audit_logs.field.filterable.excludes", + description = "Enable label fallback. (replace _ with spaces and capitalize properly)", + examples = {"false"}, + defaultValue = "true", + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private boolean labelFallbackEnabled; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.labels.service_logs.field.fallback.prefixes:" + SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS +"}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.labels.service_logs.field.fallback.prefixes", + description = "List of prefixes that should be removed during fallback of service field labels.", + examples = {"ws_,std_,sdi_"}, + defaultValue = SERVICE_FIELD_FALLBACK_PREFIX_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> serviceFieldFallbackPrefixes; + + @Value("#{propertiesSplitter.parseList('${logsearch.web.labels.service_logs.field.fallback.prefixes:" + AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS + "}')}") + @LogSearchPropertyDescription( + name = "logsearch.web.labels.service_logs.field.fallback.prefixes", + description = "List of prefixes that should be removed during fallback of audit field labels.", + examples = {"ws_,std_,sdi_"}, + defaultValue = AUDIT_FIELD_FALLBACK_PREFIX_DEFAULTS, + sources = {LOGSEARCH_PROPERTIES_FILE} + ) + private List<String> auditFieldFallbackPrefixes; + + private final Map<String, Map<String, String>> mergedAuditFieldLabelMap = new HashMap<>(); + + private final Map<String, List<String>> mergedAuditFieldVisibleMap = new HashMap<>(); + + private final Map<String, List<String>> mergedAuditFieldExcludeMap = new HashMap<>(); + + private final Map<String, List<String>> mergedAuditFieldFilterableExcludesMap = new HashMap<>(); + + public Map<String, String> getServiceGroupLabels() { + return serviceGroupLabels; + } + + public void setServiceGroupLabels(Map<String, String> serviceGroupLabels) { + this.serviceGroupLabels = serviceGroupLabels; + } + + public Map<String, String> getServiceComponentLabels() { + return serviceComponentLabels; + } + + public void setServiceComponentLabels(Map<String, String> serviceComponentLabels) { + this.serviceComponentLabels = serviceComponentLabels; + } + + public Map<String, String> getAuditComponentLabels() { + return auditComponentLabels; + } + + public void setAuditComponentLabels(Map<String, String> auditComponentLabels) { + this.auditComponentLabels = auditComponentLabels; + } + + public Map<String, String> getServiceFieldLabels() { + return serviceFieldLabels; + } + + public void setServiceFieldLabels(Map<String, String> serviceFieldLabels) { + this.serviceFieldLabels = serviceFieldLabels; + } + + public Map<String, Map<String, String>> getAuditFieldLabels() { + return auditFieldLabels; + } + + public void setAuditFieldLabels(Map<String, Map<String, String>> auditFieldLabels) { + this.auditFieldLabels = auditFieldLabels; + } + + public List<String> getServiceFieldExcludeList() { + return serviceFieldExcludeList; + } + + public void setServiceFieldExcludeList(List<String> serviceFieldExcludeList) { + this.serviceFieldExcludeList = serviceFieldExcludeList; + } + + public List<String> getServiceFieldVisibleList() { + return serviceFieldVisibleList; + } + + public void setServiceFieldVisibleList(List<String> serviceFieldVisibleList) { + this.serviceFieldVisibleList = serviceFieldVisibleList; + } + + public Map<String, List<String>> getAuditFieldVisibleleMap() { + return auditFieldVisibleleMap; + } + + public void setAuditFieldVisibleleMap(Map<String, List<String>> auditFieldVisibleleMap) { + this.auditFieldVisibleleMap = auditFieldVisibleleMap; + } + + public List<String> getAuditFieldCommonVisibleList() { + return auditFieldCommonVisibleList; + } + + public void setAuditFieldCommonVisibleList(List<String> auditFieldCommonVisibleList) { + this.auditFieldCommonVisibleList = auditFieldCommonVisibleList; + } + + public Map<String, List<String>> getAuditFieldExcludeMap() { + return auditFieldExcludeMap; + } + + public void setAuditFieldExcludeMap(Map<String, List<String>> auditFieldExcludeMap) { + this.auditFieldExcludeMap = auditFieldExcludeMap; + } + + public List<String> getAuditFieldCommonExcludeList() { + return auditFieldCommonExcludeList; + } + + public void setAuditFieldCommonExcludeList(List<String> auditFieldCommonExcludeList) { + this.auditFieldCommonExcludeList = auditFieldCommonExcludeList; + } + + public Map<String, String> getAuditFieldCommonLabels() { + return auditFieldCommonLabels; + } + + public void setAuditFieldCommonLabels(Map<String, String> auditFieldCommonLabels) { + this.auditFieldCommonLabels = auditFieldCommonLabels; + } + + public boolean isLabelFallbackEnabled() { + return labelFallbackEnabled; + } + + public void setLabelFallbackEnabled(boolean labelFallbackEnabled) { + this.labelFallbackEnabled = labelFallbackEnabled; + } + + public List<String> getServiceFieldFallbackPrefixes() { + return serviceFieldFallbackPrefixes; + } + + public void setServiceFieldFallbackPrefixes(List<String> serviceFieldFallbackPrefixes) { + this.serviceFieldFallbackPrefixes = serviceFieldFallbackPrefixes; + } + + public List<String> getAuditFieldFallbackPrefixes() { + return auditFieldFallbackPrefixes; + } + + public void setAuditFieldFallbackPrefixes(List<String> auditFieldFallbackPrefixes) { + this.auditFieldFallbackPrefixes = auditFieldFallbackPrefixes; + } + + public List<String> getServiceFieldFilterableExcludesList() { + return serviceFieldFilterableExcludesList; + } + + public void setServiceFieldFilterableExcludesList(List<String> serviceFieldFilterableExcludesList) { + this.serviceFieldFilterableExcludesList = serviceFieldFilterableExcludesList; + } + + public Map<String, List<String>> getMergedAuditFieldVisibleMap() { + return mergedAuditFieldVisibleMap; + } + + public Map<String, List<String>> getMergedAuditFieldExcludeMap() { + return mergedAuditFieldExcludeMap; + } + + public Map<String, Map<String, String>> getMergedAuditFieldLabelMap() { + return mergedAuditFieldLabelMap; + } + + public Map<String, List<String>> getMergedAuditFieldFilterableExcludesMap() { + return mergedAuditFieldFilterableExcludesMap; + } + + @PostConstruct + public void init() { + mergeCommonAndSpecMapValues(auditFieldLabels, auditFieldCommonLabels, mergedAuditFieldLabelMap); + mergeCommonAndSpecListValues(auditFieldVisibleleMap, auditFieldCommonVisibleList, mergedAuditFieldVisibleMap); + mergeCommonAndSpecListValues(auditFieldExcludeMap, auditFieldCommonExcludeList, mergedAuditFieldExcludeMap); + mergeCommonAndSpecListValues(auditFieldFilterableExcludeMap, auditFieldCommonFilterableExcludeList, mergedAuditFieldFilterableExcludesMap); + } + + private void mergeCommonAndSpecListValues(Map<String, List<String>> specMap, List<String> commonList, + Map<String, List<String>> mergedMap) { + Set<String> componentFilterableKeys = specMap.keySet(); + for (String component : componentFilterableKeys) { + List<String> specAuditDataList = specMap.get(component); + List<String> mergedDataList = new ArrayList<>(); + if (specAuditDataList != null) { + mergedDataList.addAll(specAuditDataList); + for (String commonData : commonList) { + if (!specAuditDataList.contains(commonData)) { + mergedDataList.add(commonData); + } + } + mergedMap.put(component, mergedDataList); + } + } + } + + private void mergeCommonAndSpecMapValues(Map<String, Map<String, String>> specMap, Map<String, String> commonMap, + Map<String, Map<String, String>> mergedMap) { + Set<String> componentFilterableKeys = specMap.keySet(); + for (String component : componentFilterableKeys) { + Map<String, String> specAuditDataMap = specMap.get(component); + Map<String, String> mergedAuditDataMap = new HashMap<>(); + if (specAuditDataMap != null) { + mergedAuditDataMap.putAll(specAuditDataMap); + for (Map.Entry<String, String> entry : commonMap.entrySet()) { + if (!specAuditDataMap.containsKey(entry.getKey())) { + mergedAuditDataMap.put(entry.getKey(), entry.getValue()); + } + } + mergedMap.put(component, mergedAuditDataMap); + } + } + } + +} diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java index 0563d79..b0fdeb4 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java @@ -95,7 +95,6 @@ public class DocConstants { public static final String GET_ANY_GRAPH_COUNT_DATA_OD = "Get the data generic enough to use for graph plots (yAzis is always count)"; public static final String GET_HOST_LIST_BY_COMPONENT_OD = "Get host list of components"; public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "Get service logs schema fields"; - public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch"; public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data"; public static final String REQUEST_CANCEL = "Cancel an ongoing solr request"; public static final String GET_HOST_LOGFILES_OD = "Get the log files of the components of a host"; diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java index 4f3fcbb..412366d 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java @@ -40,8 +40,12 @@ import freemarker.template.TemplateException; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.common.StatusMessage; +import org.apache.ambari.logsearch.common.LabelFallbackHandler; +import org.apache.ambari.logsearch.conf.UIMappingConfig; import org.apache.ambari.logsearch.dao.AuditSolrDao; import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; +import org.apache.ambari.logsearch.model.metadata.AuditFieldMetadataResponse; +import org.apache.ambari.logsearch.model.metadata.FieldMetadata; import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest; import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest; import org.apache.ambari.logsearch.model.request.impl.AuditLogRequest; @@ -51,7 +55,6 @@ import org.apache.ambari.logsearch.model.request.impl.UserExportRequest; import org.apache.ambari.logsearch.model.response.AuditLogData; import org.apache.ambari.logsearch.model.response.AuditLogResponse; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; -import org.apache.ambari.logsearch.model.response.GroupListResponse; import org.apache.ambari.logsearch.model.response.LogData; import org.apache.ambari.logsearch.solr.ResponseDataGenerator; import org.apache.ambari.logsearch.solr.SolrConstants; @@ -59,6 +62,7 @@ import org.apache.ambari.logsearch.solr.model.SolrAuditLogData; import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData; import org.apache.ambari.logsearch.util.DownloadUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.ambari.logsearch.common.VResponse; import org.apache.log4j.Logger; @@ -89,6 +93,10 @@ public class AuditLogsManager extends ManagerBase<AuditLogData, AuditLogResponse private Configuration freemarkerConfiguration; @Inject private SolrSchemaFieldDao solrSchemaFieldDao; + @Inject + private UIMappingConfig uiMappingConfig; + @Inject + private LabelFallbackHandler labelFallbackHandler; public AuditLogResponse getLogs(AuditLogRequest request) { String event = "/audit/logs"; @@ -123,11 +131,14 @@ public class AuditLogsManager extends ManagerBase<AuditLogData, AuditLogResponse return docList; } - public GroupListResponse getAuditComponents(AuditComponentRequest request) { - GroupListResponse componentResponse = new GroupListResponse(); - List<LogData> docList = getComponents(request); - componentResponse.setGroupList(docList); - return componentResponse; + public Map<String, String> getAuditComponents(String clusters) { + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setQuery("*:*"); + solrQuery.setRows(0); + SolrUtil.setFacetField(solrQuery, AUDIT_COMPONENT); + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + return responseDataGenerator.generateComponentMetadata(queryResponse, AUDIT_COMPONENT, + uiMappingConfig.getAuditComponentLabels()); } public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphRequest request) { @@ -142,8 +153,56 @@ public class AuditLogsManager extends ManagerBase<AuditLogData, AuditLogResponse return responseDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0); } - public String getAuditLogsSchemaFieldsName() { - return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.AUDIT)); + public AuditFieldMetadataResponse getAuditLogSchemaMetadata() { + Map<String, List<FieldMetadata>> overrides = new HashMap<>(); + List<FieldMetadata> defaults = new ArrayList<>(); + + Map<String, String> schemaFieldsMap = solrSchemaFieldDao.getSchemaFieldNameMap(LogType.AUDIT); + + Map<String, Map<String, String>> fieldLabelMap = uiMappingConfig.getMergedAuditFieldLabelMap(); + Map<String, List<String>> fieldVisibleeMap = uiMappingConfig.getMergedAuditFieldVisibleMap(); + Map<String, List<String>> fieldExcludeMap = uiMappingConfig.getMergedAuditFieldExcludeMap(); + Map<String, List<String>> fieldFilterableExcludeMap = uiMappingConfig.getMergedAuditFieldFilterableExcludesMap(); + + Map<String, String> commonFieldLabels = uiMappingConfig.getAuditFieldCommonLabels(); + List<String> commonFieldVisibleList = uiMappingConfig.getAuditFieldCommonVisibleList(); + List<String> commonFieldExcludeList = uiMappingConfig.getAuditFieldCommonExcludeList(); + List<String> commonFieldFilterableExcludeList = uiMappingConfig.getAuditFieldCommonExcludeList(); + + Map<String, String> componentLabels = uiMappingConfig.getAuditComponentLabels(); + + for (Map.Entry<String, String> component : componentLabels.entrySet()) { + String componentName = component.getKey(); + List<FieldMetadata> auditComponentFieldMetadataList = new ArrayList<>(); + for (Map.Entry<String, String> fieldEntry : schemaFieldsMap.entrySet()) { + String field = fieldEntry.getKey(); + if (!fieldExcludeMap.containsKey(field) && !commonFieldExcludeList.contains(field)) { + String fieldLabel = fieldLabelMap.get(componentName) != null ? fieldLabelMap.get(componentName).get(field): null; + String fallbackedFieldLabel = labelFallbackHandler.fallbackIfRequired(field, fieldLabel, + true, true, true, + uiMappingConfig.getAuditFieldFallbackPrefixes()); + + Boolean excludeFromFilter = fieldFilterableExcludeMap.get(componentName) != null && fieldFilterableExcludeMap.get(componentName).contains(field); + Boolean visible = fieldVisibleeMap.get(componentName) != null && fieldVisibleeMap.get(componentName).contains(field); + auditComponentFieldMetadataList.add(new FieldMetadata(field, fallbackedFieldLabel, !excludeFromFilter, visible)); + } + overrides.put(componentName, auditComponentFieldMetadataList); + } + } + + for (Map.Entry<String, String> fieldEntry : schemaFieldsMap.entrySet()) { + String field = fieldEntry.getKey(); + if (!commonFieldExcludeList.contains(field)) { + String fieldLabel = commonFieldLabels.get(field); + Boolean visible = commonFieldVisibleList.contains(field); + Boolean excludeFromFilter = commonFieldFilterableExcludeList.contains(field); + String fallbackedFieldLabel = labelFallbackHandler.fallbackIfRequired(field, fieldLabel, + true, true, true, + uiMappingConfig.getAuditFieldFallbackPrefixes()); + defaults.add(new FieldMetadata(field, fallbackedFieldLabel, !excludeFromFilter, visible)); + } + } + return new AuditFieldMetadataResponse(defaults, overrides); } public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) { diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java index 061cc2c..d022b4d 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java @@ -28,6 +28,7 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.stream.Collectors; import javax.inject.Inject; import javax.inject.Named; @@ -40,13 +41,16 @@ import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; -import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper; +import org.apache.ambari.logsearch.common.LabelFallbackHandler; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.common.StatusMessage; +import org.apache.ambari.logsearch.conf.UIMappingConfig; import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; +import org.apache.ambari.logsearch.model.metadata.FieldMetadata; +import org.apache.ambari.logsearch.model.metadata.ServiceComponentMetadataWrapper; import org.apache.ambari.logsearch.model.request.impl.HostLogFilesRequest; import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest; import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest; @@ -124,6 +128,10 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe private Configuration freemarkerConfiguration; @Inject private SolrSchemaFieldDao solrSchemaFieldDao; + @Inject + private UIMappingConfig uiMappingConfig; + @Inject + private LabelFallbackHandler labelFallbackHandler; public ServiceLogResponse searchLogs(ServiceLogRequest request) { String event = "/service/logs"; @@ -161,10 +169,6 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe return getFields(HOST, clusters, SolrHostLogData.class); } - public GroupListResponse getComponents(String clusters) { - return getFields(COMPONENT, clusters, SolrComponentTypeLogData.class); - } - public GraphDataListResponse getAggregatedInfo(ServiceLogAggregatedInfoRequest request) { SimpleQuery solrDataQuery = new BaseServiceLogRequestQueryConverter().convert(request); SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(solrDataQuery); @@ -439,8 +443,22 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe return responseDataGenerator.generateOneLevelServiceNodeTree(response, String.format("%s,%s", COMPONENT, LEVEL)); } - public String getServiceLogsSchemaFieldsName() { - return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.SERVICE)); + public List<FieldMetadata> getServiceLogsSchemaFieldsName() { + Map<String, String> schemaFieldsMap = solrSchemaFieldDao.getSchemaFieldNameMap(LogType.SERVICE); + return schemaFieldsMap + .entrySet() + .stream() + .filter(e -> !uiMappingConfig.getServiceFieldExcludeList().contains(e.getKey())) + .map(e -> + new FieldMetadata( + e.getKey(), + labelFallbackHandler.fallbackIfRequired( + e.getKey(), uiMappingConfig.getServiceFieldLabels().get(e.getKey()), + true, false, true, + uiMappingConfig.getServiceFieldFallbackPrefixes()), + !uiMappingConfig.getServiceFieldFilterableExcludesList().contains(e.getKey()), + uiMappingConfig.getServiceFieldVisibleList().contains(e.getKey()))) + .collect(Collectors.toList()); } public BarGraphDataListResponse getAnyGraphCountData(ServiceAnyGraphRequest request) { @@ -596,15 +614,6 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe return (T)result; } - - - public String getHadoopServiceConfigJSON() { - String hadoopServiceConfigJSON = HadoopServiceConfigHelper.getHadoopServiceConfigJSON(); - if (hadoopServiceConfigJSON == null) { - throw RESTErrorUtil.createRESTException("Could not load HadoopServiceConfig.json", MessageEnums.ERROR_SYSTEM); - } - return hadoopServiceConfigJSON; - } public HostLogFilesResponse getHostLogFileData(HostLogFilesRequest request) { SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); @@ -621,4 +630,18 @@ public class ServiceLogsManager extends ManagerBase<ServiceLogData, ServiceLogRe public List<String> getClusters() { return getClusters(serviceLogsSolrDao, CLUSTER, "/service/logs/clusters"); } + + + public ServiceComponentMetadataWrapper getComponentMetadata(String clusters) { + String pivotFields = COMPONENT + ",group"; + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setQuery("*:*"); + solrQuery.setRows(0); + solrQuery.set("facet", true); + solrQuery.set("facet.pivot", pivotFields); + SolrUtil.addListFilterToSolrQuery(solrQuery, CLUSTER, clusters); + QueryResponse queryResponse = serviceLogsSolrDao.process(solrQuery, "/serivce/logs/components"); + return responseDataGenerator.generateGroupedComponentMetadataResponse( + queryResponse, pivotFields, uiMappingConfig.getServiceGroupLabels(), uiMappingConfig.getServiceComponentLabels()); + } } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/AuditFieldMetadataResponse.java similarity index 58% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/AuditFieldMetadataResponse.java index 82e9504..3ef96ff 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/AuditFieldMetadataResponse.java @@ -16,37 +16,30 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +import io.swagger.annotations.ApiModel; +import java.util.List; import java.util.Map; -public interface InputDescriptor { - String getType(); +@ApiModel +public class AuditFieldMetadataResponse { - String getRowtype(); + private final List<FieldMetadata> defaults; - String getPath(); + private final Map<String, List<FieldMetadata>> overrides; - Map<String, String> getAddFields(); + public AuditFieldMetadataResponse(List<FieldMetadata> defaults, Map<String, List<FieldMetadata>> overrides) { + this.defaults = defaults; + this.overrides = overrides; + } - String getSource(); + public List<FieldMetadata> getDefaults() { + return defaults; + } - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); + public Map<String, List<FieldMetadata>> getOverrides() { + return overrides; + } } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/ComponentMetadata.java similarity index 61% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/ComponentMetadata.java index 82e9504..41b6b2d 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/ComponentMetadata.java @@ -16,37 +16,23 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.swagger.annotations.ApiModel; -import java.util.Map; +@ApiModel +@JsonIgnoreProperties(ignoreUnknown = true) +public class ComponentMetadata extends Metadata implements Groupable { + private final String group; -public interface InputDescriptor { - String getType(); + public ComponentMetadata(String name, String label, String group) { + super(name, label); + this.group = group; + } - String getRowtype(); - - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); + @Override + public String getGroup() { + return group; + } } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/FieldMetadata.java similarity index 54% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/FieldMetadata.java index 82e9504..2eec14f 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/FieldMetadata.java @@ -16,37 +16,30 @@ * specific language governing permissions and limitations * under the License. */ - -package org.apache.ambari.logsearch.config.api.model.inputconfig; - -import java.util.Map; - -public interface InputDescriptor { - String getType(); - - String getRowtype(); - - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); +package org.apache.ambari.logsearch.model.metadata; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.swagger.annotations.ApiModel; + +@ApiModel +@JsonIgnoreProperties(ignoreUnknown = true) +public class FieldMetadata extends Metadata implements Filterable, Visible { + private final boolean filterable; + private final boolean visible; + + public FieldMetadata(String name, String label, Boolean filterable, Boolean visible) { + super(name, label); + this.visible = visible; + this.filterable = filterable; + } + + @Override + public Boolean isFilterable() { + return this.filterable; + } + + @Override + public boolean isVisible() { + return this.visible; + } } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Filterable.java similarity index 61% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Filterable.java index 82e9504..e076e81e 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Filterable.java @@ -16,37 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +public interface Filterable { -import java.util.Map; + Boolean isFilterable(); -public interface InputDescriptor { - String getType(); - - String getRowtype(); - - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Groupable.java similarity index 61% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Groupable.java index 82e9504..49d9f71 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Groupable.java @@ -16,37 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +public interface Groupable { -import java.util.Map; + String getGroup(); -public interface InputDescriptor { - String getType(); - - String getRowtype(); - - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Labelable.java similarity index 61% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Labelable.java index 82e9504..eb82b81 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Labelable.java @@ -16,37 +16,12 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +public interface Labelable { -import java.util.Map; + String getName(); -public interface InputDescriptor { - String getType(); + String getLabel(); - String getRowtype(); - - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Metadata.java similarity index 61% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Metadata.java index 82e9504..12fd664 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Metadata.java @@ -16,37 +16,25 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +public class Metadata implements Labelable { -import java.util.Map; + private final String name; + private final String label; -public interface InputDescriptor { - String getType(); + Metadata(String name, String label) { + this.name = name; + this.label = label; + } - String getRowtype(); + @Override + public String getName() { + return name; + } - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); + @Override + public String getLabel() { + return label; + } } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/ServiceComponentMetadataWrapper.java similarity index 55% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/ServiceComponentMetadataWrapper.java index 82e9504..0c537c3 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/ServiceComponentMetadataWrapper.java @@ -16,37 +16,30 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.swagger.annotations.ApiModel; +import java.util.List; import java.util.Map; -public interface InputDescriptor { - String getType(); +@ApiModel +@JsonIgnoreProperties(ignoreUnknown = true) +public class ServiceComponentMetadataWrapper { + private final Map<String, String> groups; + private final List<ComponentMetadata> metadata; - String getRowtype(); + public ServiceComponentMetadataWrapper(List<ComponentMetadata> metadata, Map<String, String> groups) { + this.groups = groups; + this.metadata = metadata; + } - String getPath(); + public Map<String, String> getGroups() { + return groups; + } - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); + public List<ComponentMetadata> getMetadata() { + return metadata; + } } diff --git a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Visible.java similarity index 61% copy from ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java copy to ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Visible.java index 82e9504..f105b73 100644 --- a/ambari-logsearch/ambari-logsearch-config-api/src/main/java/org/apache/ambari/logsearch/config/api/model/inputconfig/InputDescriptor.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/metadata/Visible.java @@ -16,37 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +package org.apache.ambari.logsearch.model.metadata; -package org.apache.ambari.logsearch.config.api.model.inputconfig; +public interface Visible { -import java.util.Map; + boolean isVisible(); -public interface InputDescriptor { - String getType(); - - String getRowtype(); - - String getPath(); - - Map<String, String> getAddFields(); - - String getSource(); - - Boolean isTail(); - - Boolean isGenEventMd5(); - - Boolean isUseEventMd5AsId(); - - Boolean isCacheEnabled(); - - String getCacheKeyField(); - - Boolean getCacheLastDedupEnabled(); - - Integer getCacheSize(); - - Long getCacheDedupInterval(); - - Boolean isEnabled(); } diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ServiceLogData.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ServiceLogData.java index 16aed16..e17b87d 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ServiceLogData.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/model/response/ServiceLogData.java @@ -60,4 +60,9 @@ public interface ServiceLogData extends CommonLogData, ComponentTypeLogData, Hos String getHost(); void setHost(String host); + + @JsonProperty("group") + String getGroup(); + + void setGroup(String group); } diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java index 27a3e83..66863c9 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/AuditLogsResource.java @@ -19,6 +19,7 @@ package org.apache.ambari.logsearch.rest; +import javax.annotation.Nullable; import javax.inject.Inject; import javax.inject.Named; import javax.ws.rs.BeanParam; @@ -26,25 +27,27 @@ import javax.ws.rs.DELETE; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.Response; import freemarker.template.TemplateException; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; +import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.StatusMessage; +import org.apache.ambari.logsearch.model.metadata.AuditFieldMetadataResponse; import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest; -import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest; import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest; import org.apache.ambari.logsearch.model.request.impl.TopFieldAuditLogRequest; import org.apache.ambari.logsearch.model.request.impl.UserExportRequest; import org.apache.ambari.logsearch.model.response.AuditLogResponse; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; -import org.apache.ambari.logsearch.model.response.GroupListResponse; import org.apache.ambari.logsearch.model.request.impl.AuditLogRequest; import org.apache.ambari.logsearch.manager.AuditLogsManager; import org.springframework.context.annotation.Scope; import java.util.List; +import java.util.Map; import static org.apache.ambari.logsearch.doc.DocConstants.AuditOperationDescriptions.*; @@ -61,8 +64,8 @@ public class AuditLogsResource { @Path("/schema/fields") @Produces({"application/json"}) @ApiOperation(GET_AUDIT_SCHEMA_FIELD_LIST_OD) - public String getSolrFieldList() { - return auditLogsManager.getAuditLogsSchemaFieldsName(); + public AuditFieldMetadataResponse getSolrFieldList() { + return auditLogsManager.getAuditLogSchemaMetadata(); } @GET @@ -83,8 +86,8 @@ public class AuditLogsResource { @Path("/components") @Produces({"application/json"}) @ApiOperation(GET_AUDIT_COMPONENTS_OD) - public GroupListResponse getAuditComponents(@BeanParam AuditComponentRequest request) { - return auditLogsManager.getAuditComponents(request); + public Map<String, String> getAuditComponents(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) { + return auditLogsManager.getAuditComponents(clusters); } @GET diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java index e02acb8..a891671 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java @@ -36,6 +36,8 @@ import io.swagger.annotations.ApiOperation; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.StatusMessage; +import org.apache.ambari.logsearch.model.metadata.FieldMetadata; +import org.apache.ambari.logsearch.model.metadata.ServiceComponentMetadataWrapper; import org.apache.ambari.logsearch.model.request.impl.HostLogFilesRequest; import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest; import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest; @@ -97,8 +99,8 @@ public class ServiceLogsResource { @Path("/components") @Produces({"application/json"}) @ApiOperation(GET_COMPONENTS_OD) - public GroupListResponse getComponents(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) { - return serviceLogsManager.getComponents(clusters); + public ServiceComponentMetadataWrapper getComponents(@QueryParam(LogSearchConstants.REQUEST_PARAM_CLUSTER_NAMES) @Nullable String clusters) { + return serviceLogsManager.getComponentMetadata(clusters); } @GET @@ -172,11 +174,12 @@ public class ServiceLogsResource { public NodeListResponse getComponentListWithLevelCounts(@BeanParam ServiceLogComponentLevelRequest request) { return serviceLogsManager.getComponentListWithLevelCounts(request); } + @GET @Path("/schema/fields") @Produces({"application/json"}) @ApiOperation(GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD) - public String getServiceLogsSchemaFieldsName() { + public List<FieldMetadata> getServiceLogsSchemaFieldsName() { return serviceLogsManager.getServiceLogsSchemaFieldsName(); } @@ -197,14 +200,6 @@ public class ServiceLogsResource { } @GET - @Path("/serviceconfig") - @Produces({"application/json"}) - @ApiOperation(GET_HADOOP_SERVICE_CONFIG_JSON_OD) - public String getHadoopServiceConfigJSON() { - return serviceLogsManager.getHadoopServiceConfigJSON(); - } - - @GET @Path("/request/cancel") @Produces({"application/json"}) @ApiOperation(REQUEST_CANCEL) diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/ResponseDataGenerator.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/ResponseDataGenerator.java index e3c1a24..ad0e6dc 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/ResponseDataGenerator.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/ResponseDataGenerator.java @@ -24,6 +24,7 @@ import static org.apache.ambari.logsearch.solr.SolrConstants.ServiceLogConstants import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -31,6 +32,9 @@ import java.util.Map; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import org.apache.ambari.logsearch.common.LabelFallbackHandler; +import org.apache.ambari.logsearch.model.metadata.ComponentMetadata; +import org.apache.ambari.logsearch.model.metadata.ServiceComponentMetadataWrapper; import org.apache.ambari.logsearch.model.response.BarGraphData; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; import org.apache.ambari.logsearch.model.response.CountData; @@ -49,13 +53,18 @@ import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RangeFacet; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.util.NamedList; +import javax.inject.Inject; import javax.inject.Named; @Named public class ResponseDataGenerator { + @Inject + private LabelFallbackHandler labelFallbackHandler; + public BarGraphDataListResponse generateBarGraphDataResponseWithRanges(QueryResponse response, String typeField, boolean typeUppercase) { BarGraphDataListResponse dataList = new BarGraphDataListResponse(); if (response == null) { @@ -396,7 +405,7 @@ public class ResponseDataGenerator { if (response == null) { return graphInfo; } - List<List<PivotField>> hirarchicalPivotField = new ArrayList<List<PivotField>>(); + List<List<PivotField>> hirarchicalPivotField = new ArrayList<>(); List<GraphData> dataList = new ArrayList<>(); NamedList<List<PivotField>> namedList = response.getFacetPivot(); if (namedList != null) { @@ -450,4 +459,66 @@ public class ResponseDataGenerator { return response; } + + public Map<String, String> generateComponentMetadata(QueryResponse queryResponse, + String facetField, Map<String, String> componetnLabels) { + Map<String, String> result = new HashMap<>(); + if (queryResponse == null) { + return result; + } + FacetField facetFields = queryResponse.getFacetField(facetField); + if (facetFields == null) { + return result; + } + List<Count> counts = facetFields.getValues(); + if (counts == null) { + return result; + } + for (Count count : counts) { + if (count.getName() != null) { + String label = componetnLabels.get(count.getName()); + String fallbackedLabel = labelFallbackHandler.fallbackIfRequired(count.getName(), label, true, false, true); + result.put(count.getName(), fallbackedLabel); + } + } + return result; + } + + public ServiceComponentMetadataWrapper generateGroupedComponentMetadataResponse(QueryResponse queryResponse, String pivotFields, + Map<String, String> groupLabels, + Map<String, String> componentLabels) { + List<ComponentMetadata> componentMetadata = new ArrayList<>(); + Map<String, String> groupsMetadata = new HashMap<>(); + + if (queryResponse == null) { + return new ServiceComponentMetadataWrapper(componentMetadata, groupsMetadata); + } + NamedList<List<PivotField>> facetPivotResponse = queryResponse.getFacetPivot(); + if (facetPivotResponse == null || facetPivotResponse.size() < 1) { + return new ServiceComponentMetadataWrapper(componentMetadata, groupsMetadata); + } + if (CollectionUtils.isEmpty(facetPivotResponse.get(pivotFields))) { + return new ServiceComponentMetadataWrapper(componentMetadata, groupsMetadata); + } + + for (PivotField pivotField : facetPivotResponse.get(pivotFields)) { + if (pivotField != null && pivotField.getValue() != null) { + String componentName = pivotField.getValue().toString(); + String groupName = null; + if (CollectionUtils.isNotEmpty(pivotField.getPivot())) { + Object groupValue = pivotField.getPivot().get(0).getValue(); + if (groupValue != null) { + groupName = groupValue.toString(); + groupsMetadata.put(groupName, groupLabels.getOrDefault(groupName, null)); + } + } + String label = componentLabels.get(componentName); + String fallbackedLabel = labelFallbackHandler.fallbackIfRequired(componentName, label, true, false, true); + componentMetadata.add((new ComponentMetadata(componentName, fallbackedLabel, groupName))); + + } + } + + return new ServiceComponentMetadataWrapper(componentMetadata, groupsMetadata); + } } diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/SolrConstants.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/SolrConstants.java index 6554bcf..6b2669b 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/SolrConstants.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/SolrConstants.java @@ -57,6 +57,7 @@ public class SolrConstants { public static final String LOG_MESSAGE = "log_message"; public static final String KEY_LOG_MESSAGE = "key_log_message"; public static final String HOST = "host"; + public static final String GROUP = "group"; public static final String LEVEL = "level"; public static final String THREAD_NAME = "thread_name"; public static final String LOGGER_NAME = "logger_name"; diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java index 224ed8c..5db04ee 100644 --- a/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java +++ b/ambari-logsearch/ambari-logsearch-server/src/main/java/org/apache/ambari/logsearch/solr/model/SolrServiceLogData.java @@ -49,6 +49,9 @@ public class SolrServiceLogData extends SolrCommonLogData implements ServiceLogD @Field(HOST) private String host; + @Field(GROUP) + private String group; + @Field(SDI_DYNAMIC_FIELDS) private Map<String, Object> sdiDynamicFields; @@ -93,6 +96,16 @@ public class SolrServiceLogData extends SolrCommonLogData implements ServiceLogD } @Override + public String getGroup() { + return group; + } + + @Override + public void setGroup(String group) { + this.group = group; + } + + @Override public Date getLogTime() { return logTime; } diff --git a/ambari-logsearch/ambari-logsearch-server/src/main/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-server/src/main/resources/HadoopServiceConfig.json deleted file mode 100644 index dd4bf6b..0000000 --- a/ambari-logsearch/ambari-logsearch-server/src/main/resources/HadoopServiceConfig.json +++ /dev/null @@ -1,488 +0,0 @@ -{ - "service": { - "accumulo": { - "label": "Accumulo", - "components": [ - { - "name": "accumulo_gc" - }, - { - "name": "accumulo_master" - }, - { - "name": "accumulo_monitor" - }, - { - "name": "accumulo_tracer" - }, - { - "name": "accumulo_tserver" - } - ], - "dependencies": [ - ] - }, - "atlas": { - "label": "Atlas", - "components": [ - { - "name": "atlas_app" - } - ], - "dependencies": [ - ] - }, - "ambari": { - "label": "Ambari", - "components": [ - { - "name": "ambari_agent" - }, - { - "name": "ambari_server" - }, - { - "name": "ambari_alerts" - }, - { - "name": "ambari_audit" - }, - { - "name": "ambari_config_changes" - }, - { - "name": "ambari_eclipselink" - }, - { - "name": "ambari_server_check_database" - } - ], - "dependencies": [ - ] - }, - "ams": { - "label": "AMS", - "components": [ - { - "name": "ams_hbase_master" - }, - { - "name": "ams_hbase_regionserver" - }, - { - "name": "ams_collector" - }, - { - "name": "ams_monitor" - }, - { - "name": "ams_grafana" - } - ], - "dependencies": [ - ] - }, - "falcon": { - "label": "Falcon", - "components": [ - { - "name": "falcon_app" - } - ], - "dependencies": [ - ] - }, - "flume": { - "label": "Flume", - "components": [ - { - "name": "flume_handler" - } - ], - "dependencies": [ - ] - }, - "hbase": { - "label": "HBase", - "components": [ - { - "name": "hbase_master" - }, - { - "name": "hbase_regionserver" - }, - { - "name": "hbase_phoenix_server" - } - ], - "dependencies": [ - { - "service": "hdfs", - "components": [ - "hdfs_namenode" - ] - } - ] - }, - "hdfs": { - "label": "HDFS", - "components": [ - { - "name": "hdfs_datanode" - }, - { - "name": "hdfs_namenode" - }, - { - "name": "hdfs_journalnode" - }, - { - "name": "hdfs_secondarynamenode" - }, - { - "name": "hdfs_zkfc" - }, - { - "name": "hdfs_nfs3" - }, - { - "name": "hdfs_audit", - "rowtype": "audit" - } - ], - "dependencies": [ - ] - }, - "hive": { - "label": "Hive", - "components": [ - { - "name": "hive_hiveserver2" - }, - { - "name": "hive_metastore" - }, - { - "name": "webhcat_server" - } - ], - "dependencies": [ - { - "service": "hdfs", - "components": [ - "hdfs_namenode" - ] - } - ] - }, - "infra" : { - "label" : "Infra", - "components": [ - { - "name": "infra_solr" - } - ], - "dependencies": [ - ] - }, - "kafka": { - "label": "Kafka", - "components": [ - { - "name": "kafka_controller" - }, - { - "name": "kafka_request" - }, - { - "name": "kafka_logcleaner" - }, - { - "name": "kafka_server" - }, - { - "name": "kafka_statechange" - } - ], - "dependencies": [ - { - "service": "zookeeper", - "components": [ - "zookeeper" - ] - } - ] - }, - "knox": { - "label": "Knox", - "components": [ - { - "name": "knox_gateway" - }, - { - "name": "knox_cli" - }, - { - "name": "knox_ldap" - } - ], - "dependencies": [ - ] - }, - "nifi": { - "label": "NiFi", - "components": [ - { - "name": "nifi_app" - }, - { - "name": "nifi_bootstrap" - }, - { - "name": "nifi_setup" - }, - { - "name": "nifi_user" - } - ], - "dependencies": [ - ] - }, - "mapred": { - "label": "MapReduce", - "components": [ - { - "name": "mapred_historyserver" - } - ], - "dependencies": [ - ] - }, - "logsearch": { - "label": "Logsearch", - "components": [ - { - "name": "logsearch_app" - }, - { - "name": "logsearch_feeder" - }, - { - "name": "logsearch_perf" - }, - { - "name": "logsearch_solr" - } - ], - "dependencies": [ - ] - }, - "ranger": { - "label": "Ranger", - "components": [ - { - "name": "ranger_admin" - }, - { - "name": "ranger_dbpatch" - }, - { - "name": "ranger_kms" - }, - { - "name": "ranger_usersync" - } - ], - "dependencies": [ - { - "service": "hdfs", - "required": "optional", - "components": [ - "hdfs_namenode" - ] - }, - { - "service": "hbase", - "required": "optional", - "components": [ - "hbase_master", - "hbase_regionserver" - ] - }, - { - "service": "hive", - "required": "optional", - "components": [ - "hive_hiveserver2" - ] - }, - { - "service": "kafka", - "required": "optional", - "components": [ - "kafka_ranger" - ] - }, - { - "service": "knox", - "required": "optional", - "components": [ - "knox_gateway" - ] - }, - { - "service": "storm", - "required": "optional", - "components": [ - "storm_supervisor" - ] - }, - { - "service": "yarn", - "required": "optional", - "components": [ - "yarn_resourcemanager" - ] - } - ] - }, - "oozie": { - "label": "Oozie", - "components": [ - { - "name": "oozie_app" - } - ], - "dependencies": [ - ] - }, - "hst": { - "label": "SmartSense", - "components": [ - { - "name": "hst_server" - }, - { - "name": "hst_agent" - }, - { - "name": "activity_analyzer" - }, - { - "name": "activity_explorer" - } - ], - "dependencies": [ - ] - }, - "spark": { - "label": "Spark", - "components": [ - { - "name": "spark_jobhistory_server" - }, - { - "name": "spark_thriftserver" - }, - { - "name": "livy_server" - } - ], - "dependencies": [ - ] - }, - "spark2": { - "label": "Spark 2", - "components": [ - { - "name": "spark2_jobhistory_server" - }, - { - "name": "spark2_thriftserver" - } - ], - "dependencies": [ - ] - }, - "storm": { - "label": "Storm", - "components": [ - { - "name": "storm_drpc" - }, - { - "name": "storm_logviewer" - }, - { - "name": "storm_nimbus" - }, - { - "name": "storm_supervisor" - }, - { - "name": "storm_ui" - }, - { - "name": "storm_worker" - } - ], - "dependencies": [ - ] - }, - "yarn": { - "label": "YARN", - "components": [ - { - "name": "yarn_nodemanager" - }, - { - "name": "yarn_resourcemanager" - }, - { - "name": "yarn_timelineserver" - }, - { - "name": "yarn_historyserver" - }, - { - "name": "yarn_jobsummary" - } - ], - "dependencies": [ - ] - }, - "zeppelin": { - "label": "Zeppelin", - "components": [ - { - "name": "zeppelin" - } - ], - "dependencies": [ - ] - }, - "zookeeper": { - "label": "ZooKeeper", - "components": [ - { - "name": "zookeeper" - } - ], - "dependencies": [ - ] - }, - "System": { - "label": "System", - "components": [ - { - "name": "system_message" - }, - { - "name": "secure_log" - } - ], - "dependencies": [ - ] - } - } -} diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/LabelFallbackHandlerTest.java b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/LabelFallbackHandlerTest.java new file mode 100644 index 0000000..e765f96 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/LabelFallbackHandlerTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +import org.apache.ambari.logsearch.conf.UIMappingConfig; +import org.junit.Before; +import org.junit.Test; + +import java.util.Arrays; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class LabelFallbackHandlerTest { + + private LabelFallbackHandler underTest; + + @Before + public void setUp() { + final UIMappingConfig uiMappingConfig = new UIMappingConfig(); + uiMappingConfig.setLabelFallbackEnabled(true); + underTest = new LabelFallbackHandler(uiMappingConfig); + } + + @Test + public void testFallbackIgnore() { + // GIVEN + String testInput = "my_field"; + // WHEN + String result = underTest.fallbackIfRequired(testInput, "spec label", true, false, true, null); + // THEN + assertEquals("spec label", result); + } + + @Test + public void testFallbackUnderscore() { + // GIVEN + String testInput = "my_field"; + // WHEN + String result = underTest.fallback(testInput, true, false, true); + // THEN + assertEquals("My Field", result); + } + + @Test + public void testFallbackUnderscoreWithNull() { + // GIVEN + // WHEN + String result = underTest.fallback(null, true, false, true); + // THEN + assertNull(result); + } + + @Test + public void testFallbackCamelCase() { + // GIVEN + String testInput = "myField"; + // WHEN + String result = underTest.fallback(testInput, false, true, true); + // THEN + assertEquals("My Field", result); + } + + @Test + public void testFallbackCamelCaseWithEmptyString() { + // GIVEN + String testInput = ""; + // WHEN + String result = underTest.fallback(testInput, true, true, true); + // THEN + assertNull(result); + } + + @Test + public void testFallbackCamelCaseWithNull() { + // GIVEN + // WHEN + String result = underTest.fallback(null, true, true, true); + // THEN + assertNull(result); + } + + @Test + public void testFallbackCamelCaseWith1Letter() { + // GIVEN + String testInput = "d"; + // WHEN + String result = underTest.fallback(testInput, true, true, true); + // THEN + assertEquals("D", result); + } + + @Test + public void testFallbackWithRemovingPrefixes() { + // GIVEN + String testInput = "ws_request_id"; + // WHEN + String result = underTest.fallback(testInput, true, true, true, Arrays.asList("ws_", "std_")); + // THEN + assertEquals("Request Id", result); + } + + @Test + public void testFallbackWithRemovingPrefixesWithoutAnyPrefix() { + // GIVEN + String testInput = "request_id"; + // WHEN + String result = underTest.fallback(testInput, true, true, true, Arrays.asList("ws_", "std_")); + // THEN + assertEquals("Request Id", result); + } + +} diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/PropertiesSplitterTest.java b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/PropertiesSplitterTest.java new file mode 100644 index 0000000..30cd76f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-server/src/test/java/org/apache/ambari/logsearch/common/PropertiesSplitterTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +import org.junit.Before; +import org.junit.Test; + +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class PropertiesSplitterTest { + + private PropertiesSplitter underTest; + + @Before + public void setUp() { + underTest = new PropertiesSplitter(); + } + + @Test + public void testParseList() { + // GIVEN + // WHEN + List<String> values = underTest.parseList("v1,v2"); + // THEN + assertTrue(values.contains("v1")); + assertTrue(values.contains("v2")); + } + + @Test + public void testParseListWithEmptyString() { + // GIVEN + // WHEN + List<String> values = underTest.parseList(""); + // THEN + assertTrue(values.isEmpty()); + } + + @Test + public void testParseMap() { + // GIVEN + // WHEN + Map<String, String> keyValues = underTest.parseMap("k1:v1,k2:v2"); + // THEN + assertEquals("v1", keyValues.get("k1")); + assertEquals("v2", keyValues.get("k2")); + } + + @Test + public void testParseMapWithEmptyValue() { + // GIVEN + // WHEN + Map<String, String> keyValues = underTest.parseMap("k1:v1,k2:"); + // THEN + assertEquals("v1", keyValues.get("k1")); + assertEquals("", keyValues.get("k2")); + } + + @Test + public void testParseMapWithMissingKey() { + // GIVEN + // WHEN + Map<String, String> keyValues = underTest.parseMap("k1:v1,:v2"); + // THEN + assertEquals("v1", keyValues.get("k1")); + assertNull(keyValues.get("k2")); + assertEquals(1, keyValues.size()); + } + + @Test + public void testParseMapInMap() { + // GIVEN + // WHEN + Map<String, Map<String, String>> keyMapValues = underTest.parseMapInMap("K1#k1:v1,k2:v2;K2#k3:v3,k4:v4"); + // THEN + Map<String, String> keyValues1 = keyMapValues.get("K1"); + Map<String, String> keyValues2 = keyMapValues.get("K2"); + assertNotNull(keyValues1); + assertNotNull(keyValues2); + assertEquals("v1", keyValues1.get("k1")); + assertEquals("v2", keyValues1.get("k2")); + assertEquals("v3", keyValues2.get("k3")); + assertEquals("v4", keyValues2.get("k4")); + } + + @Test + public void testParseListInMap() { + // GIVEN + // WHEN + Map<String, List<String>> listInMap = underTest.parseListInMap("K1:v1,v2;K2:v3,v4"); + // THEN + List<String> valueList1 = listInMap.get("K1"); + List<String> valueList2 = listInMap.get("K2"); + assertNotNull(valueList1); + assertNotNull(valueList2); + assertEquals("v1", valueList1.get(0)); + assertEquals("v2", valueList1.get(1)); + assertEquals("v3", valueList2.get(0)); + assertEquals("v4", valueList2.get(1)); + } + +} diff --git a/ambari-logsearch/ambari-logsearch-server/src/test/resources/HadoopServiceConfig.json b/ambari-logsearch/ambari-logsearch-server/src/test/resources/HadoopServiceConfig.json deleted file mode 100644 index 344dc3d..0000000 --- a/ambari-logsearch/ambari-logsearch-server/src/test/resources/HadoopServiceConfig.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "service": { - "accumulo": { - "label": "TestService", - "components": [ - { - "name": "test_component1" - }, - { - "name": "test_component2" - } - ], - "dependencies": [ - ] - } - } -} \ No newline at end of file diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json index 2dfe6d3..e3f6f12 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json @@ -3,12 +3,14 @@ { "type":"ambari_agent", "rowtype":"service", - "path":"/root/test-logs/ambari-server/ambari-agent.log" + "path":"/root/test-logs/ambari-server/ambari-agent.log", + "group": "Ambari" }, { "type":"ambari_server", "rowtype":"service", - "path":"/root/test-logs/ambari-server/ambari-server.log" + "path":"/root/test-logs/ambari-server/ambari-server.log", + "group": "Ambari" }, { "type":"ambari_alerts", @@ -16,22 +18,26 @@ "add_fields":{ "level":"INFO" }, - "path":"/root/test-logs/ambari-server/ambari-alerts.log" + "path":"/root/test-logs/ambari-server/ambari-alerts.log", + "group": "Ambari" }, { "type":"ambari_config_changes", "rowtype":"service", - "path":"/root/test-logs/ambari-server/ambari-config-changes.log" + "path":"/root/test-logs/ambari-server/ambari-config-changes.log", + "group": "Ambari" }, { "type":"ambari_eclipselink", "rowtype":"service", - "path":"/root/test-logs/ambari-server/ambari-eclipselink.log" + "path":"/root/test-logs/ambari-server/ambari-eclipselink.log", + "group": "Ambari" }, { "type":"ambari_server_check_database", "rowtype":"service", - "path":"/root/test-logs/ambari-server/ambari-server-check-database.log" + "path":"/root/test-logs/ambari-server/ambari-server-check-database.log", + "group": "Ambari" }, { "type":"ambari_audit", diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json index 2ef13a2..9ec78d6 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-hst.json @@ -3,7 +3,8 @@ { "type": "hst_agent", "rowtype": "service", - "path": "/root/test-logs/smartsense/hst-agent-test-log.txt" + "path": "/root/test-logs/smartsense/hst-agent-test-log.txt", + "group": "Smartsense" } ], "filter": [ diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-secure_log.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-secure_log.json index 93f6b1e..99412e6 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-secure_log.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-secure_log.json @@ -3,7 +3,8 @@ { "type": "secure_log", "rowtype": "service", - "path": "/root/test-logs/secure_log/secure-log.txt" + "path": "/root/test-logs/secure_log/secure-log.txt", + "group": "System" } ], "filter": [ diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-system_message.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-system_message.json index aef586d..065ef87 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-system_message.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-system_message.json @@ -3,7 +3,8 @@ { "type": "system_message", "rowtype": "service", - "path": "/root/test-logs/system_message/message_logs.txt" + "path": "/root/test-logs/system_message/message_logs.txt", + "group": "System" } ], "filter": [ diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json index fd1af97..65185a1 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-zookeeper.json @@ -6,7 +6,8 @@ "path": "/root/test-logs/zookeeper/zookeeper-test-log.txt", "cache_enabled" : "true", "cache_size" : "10", - "cache_dedup_interval" : "1000" + "cache_dedup_interval" : "1000", + "group": "Zookeeper" } ], "filter": [ -- To stop receiving notification emails like this one, please contact oleew...@apache.org.