http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java new file mode 100644 index 0000000..c32dc36 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.graph; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; + +import org.apache.ambari.logsearch.util.DateUtil; +import org.apache.ambari.logsearch.view.VBarGraphData; +import org.apache.ambari.logsearch.view.VNameValue; +import org.apache.solr.client.solrj.response.RangeFacet; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.codehaus.jackson.map.ObjectMapper; +import org.codehaus.jackson.map.ObjectWriter; +import org.springframework.beans.factory.annotation.Autowired; + +public class GraphDataGeneratorBase { + + @Autowired + DateUtil dateUtil; + + @SuppressWarnings("unchecked") + protected void extractRangeStackValuesFromBucket( + SimpleOrderedMap<Object> jsonFacetResponse, String outerField, + String innerField, List<VBarGraphData> histogramData) { + NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse + .get(outerField); + ArrayList<Object> stackBuckets = (ArrayList<Object>) stack + .get("buckets"); + for (Object temp : stackBuckets) { + VBarGraphData vBarGraphData = new VBarGraphData(); + + SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp; + String name = ((String) level.getVal(0)).toUpperCase(); + vBarGraphData.setName(name); + + Collection<VNameValue> vNameValues = new ArrayList<VNameValue>(); + vBarGraphData.setDataCounts(vNameValues); + ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level + .get(innerField)).get("buckets"); + for (Object temp1 : levelBuckets) { + SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1; + String value = dateUtil + .convertDateWithMillisecondsToSolrDate((Date) countValue + .getVal(0)); + + String count = "" + countValue.getVal(1); + VNameValue vNameValue = new VNameValue(); + vNameValue.setName(value); + vNameValue.setValue(count); + vNameValues.add(vNameValue); + } + histogramData.add(vBarGraphData); + } + } + + @SuppressWarnings("unchecked") + protected boolean extractNonRangeStackValuesFromBucket( + SimpleOrderedMap<Object> jsonFacetResponse, String level, + Collection<VBarGraphData> vGraphDatas, String typeXAxis) { + + boolean zeroFlag = true; + if (jsonFacetResponse.get(level).toString().equals("{count=0}")) { + return false; + } + + NamedList<Object> list = (NamedList<Object>) jsonFacetResponse + .get(level); + + ArrayList<Object> list3 = (ArrayList<Object>) list.get("buckets"); + int i = 0; + for (i = 0; i < list3.size(); i++) { + VBarGraphData vGraphData = new VBarGraphData(); + + + Collection<VNameValue> levelCounts = new ArrayList<VNameValue>(); + vGraphData.setDataCounts(levelCounts); + + SimpleOrderedMap<Object> valueCount = (SimpleOrderedMap<Object>) list3 + .get(i); + String name = ("" + valueCount.getVal(0)).trim(); + if (isTypeNumber(typeXAxis)) { + VNameValue nameValue = new VNameValue(); + String value = ("" + valueCount.getVal(2)).trim().substring(0, ("" + valueCount.getVal(2)).indexOf(".")); + nameValue.setName(name); + nameValue.setValue(value); + levelCounts.add(nameValue); + } else { + SimpleOrderedMap<Object> l1 = (SimpleOrderedMap<Object>) valueCount + .getVal(2); + ArrayList<Object> l2 = (ArrayList<Object>) l1.get("buckets"); + for (int j = 0; l2 != null && j < l2.size(); j++) { + VNameValue nameValue = new VNameValue(); + SimpleOrderedMap<Object> innerValueCount = (SimpleOrderedMap<Object>) l2 + .get(j); + nameValue.setName(("" + innerValueCount.getVal(0)).trim()); + nameValue.setValue(("" + innerValueCount.getVal(1)).trim()); + levelCounts.add(nameValue); + } + } + + vGraphData.setName(name); + vGraphDatas.add(vGraphData); + } + return zeroFlag; + } + + @SuppressWarnings("unchecked") + protected boolean extractValuesFromJson( + SimpleOrderedMap<Object> jsonFacetResponse, String level, + VBarGraphData histogramData, List<RangeFacet.Count> counts) { + histogramData.setName(level); + Collection<VNameValue> levelCounts = new ArrayList<VNameValue>(); + histogramData.setDataCounts(levelCounts); + boolean zeroFlag = true; + if (jsonFacetResponse.get(level).toString().equals("{count=0}")) { + for (RangeFacet.Count date : counts) { + VNameValue nameValue = new VNameValue(); + + nameValue.setName(date.getValue()); + nameValue.setValue("0"); + + levelCounts.add(nameValue); + } + return false; + } + NamedList<Object> list = (NamedList<Object>) jsonFacetResponse + .get(level); + NamedList<Object> list2 = (NamedList<Object>) list.getVal(1); + ArrayList<Object> list3 = (ArrayList<Object>) list2.get("buckets"); + int i = 0; + for (RangeFacet.Count date : counts) { + VNameValue nameValue = new VNameValue(); + SimpleOrderedMap<Object> valueCount = (SimpleOrderedMap<Object>) list3 + .get(i); + String count = ("" + valueCount.getVal(1)).trim(); + if (!"0".equals(count)) { + zeroFlag = false; + } + nameValue.setName(date.getValue()); + nameValue.setValue(count); + + levelCounts.add(nameValue); + i++; + } + + return zeroFlag; + } + + protected boolean isTypeNumber(String typeXAxis) { + return "long".contains(typeXAxis) || "int".contains(typeXAxis) + || "float".contains(typeXAxis) || "double".contains(typeXAxis); + } + + public String convertObjToString(Object obj) throws IOException { + if (obj == null) { + return ""; + } + ObjectMapper mapper = new ObjectMapper(); + ObjectWriter w = mapper.writerWithDefaultPrettyPrinter(); + return w.writeValueAsString(obj); + } + +}
http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java new file mode 100644 index 0000000..44143f4 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGnerator.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.graph; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.SearchCriteria; +import org.apache.ambari.logsearch.dao.SolrDaoBase; +import org.apache.ambari.logsearch.query.QueryGeneration; +import org.apache.ambari.logsearch.util.ConfigUtil; +import org.apache.ambari.logsearch.util.DateUtil; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.StringUtil; +import org.apache.ambari.logsearch.view.VBarDataList; +import org.apache.ambari.logsearch.view.VBarGraphData; +import org.apache.ambari.logsearch.view.VNameValue; +import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FacetField.Count; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.RangeFacet; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class GraphDataGnerator extends GraphDataGeneratorBase { + + @Autowired + StringUtil stringUtil; + + @Autowired + QueryGeneration queryGenerator; + + @Autowired + RESTErrorUtil restErrorUtil; + + @Autowired + DateUtil dateUtil; + + private static Logger logger = Logger.getLogger(GraphDataGnerator.class); + + public String getAnyGraphData(SearchCriteria searchCriteria, + SolrDaoBase solrDaoBase, SolrQuery solrQuery) { + // X axis credentials + String xAxisField = (String) searchCriteria.getParamValue("xAxis"); + String stackField = (String) searchCriteria.getParamValue("stackBy"); + String from = (String) searchCriteria.getParamValue("from"); + String to = (String) searchCriteria.getParamValue("to"); + String unit = (String) searchCriteria.getParamValue("unit"); + String suffix = (String) searchCriteria.getParamValue("suffix"); + String typeXAxis = ConfigUtil.schemaFieldsName.get(xAxisField + suffix); + typeXAxis = (stringUtil.isEmpty(typeXAxis)) ? "string" : typeXAxis; + + // Y axis credentials + String yAxisField = (String) searchCriteria.getParamValue("yAxis"); + + searchCriteria.addParam("type", typeXAxis); + String fieldTime = (String) searchCriteria.getParamValue("feildTime"); + + int garphType = getGraphType(searchCriteria); + + switch (garphType) { + case 1: + return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, + typeXAxis, fieldTime, solrQuery); + case 2: + return rangeNonStackGraph(xAxisField, yAxisField, from, to, unit, + solrDaoBase, typeXAxis, fieldTime, solrQuery); + case 3: + return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, + to, solrDaoBase, typeXAxis, fieldTime, solrQuery); + case 4: + return rangeStackGraph(xAxisField, yAxisField, stackField, from, + to, unit, solrDaoBase, typeXAxis, fieldTime, solrQuery); + default: + return null; + } + } + + private int getGraphType(SearchCriteria searchCriteria) { + // X axis credentials + String xAxisField = (String) searchCriteria.getParamValue("xAxis"); + String stackField = (String) searchCriteria.getParamValue("stackBy"); + String from = (String) searchCriteria.getParamValue("from"); + String to = (String) searchCriteria.getParamValue("to"); + String xType = (String) searchCriteria.getParamValue("type"); + + if (xType == null) + return 0; + + // Y axis credentials + String yAxisField = (String) searchCriteria.getParamValue("yAxis"); + if (stringUtil.isEmpty(xAxisField) || stringUtil.isEmpty(yAxisField)) { + } + // Normal Graph Type + else if (stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to) + && !stringUtil.isEmpty(from) + && !(xType.contains("date") || xType.contains("time"))) + return 1; + // Range(Non-Stack) Graph Type + else if (stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to) + && !stringUtil.isEmpty(from) + && (xType.contains("date") || xType.contains("time"))) + return 2; + // Non-Range Stack Graph Type + else if (!stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to) + && !stringUtil.isEmpty(from) + && !(xType.contains("date") || xType.contains("time"))) + return 3; + // Range Stack GraphType + else if (!stringUtil.isEmpty(stackField) && !stringUtil.isEmpty(to) + && !stringUtil.isEmpty(from) + && (xType.contains("date") || xType.contains("time"))) + return 4; + + return 0; + } + + @SuppressWarnings("unchecked") + private String normalGraph(String xAxisField, String yAxisField, + String from, String to, SolrDaoBase solrDaoBase, String typeXAxis, + String fieldTime, SolrQuery solrQuery) { + VBarDataList dataList = new VBarDataList(); + Collection<VBarGraphData> vBarGraphDatas = new ArrayList<VBarGraphData>(); + VBarGraphData vBarGraphData = new VBarGraphData(); + Collection<VNameValue> vNameValues = new ArrayList<VNameValue>(); + + queryGenerator.setMainQuery(solrQuery, null); + queryGenerator.setSingleIncludeFilter(solrQuery, fieldTime, "[" + from + + " TO " + to + "]"); + if (typeXAxis.contains("string") + || typeXAxis.contains("key_lower_case") + || typeXAxis.contains("text")) { + queryGenerator.setFacetField(solrQuery, xAxisField); + try { + QueryResponse response = solrDaoBase.process(solrQuery); + Long count = response.getResults().getNumFound(); + + if (response != null && count > 0) { + FacetField facetField = response.getFacetField(xAxisField); + if (facetField != null) { + List<Count> countValues = facetField.getValues(); + for (Count cnt : countValues) { + VNameValue vNameValue = new VNameValue(); + vNameValue.setName(cnt.getName()); + vNameValue.setValue("" + cnt.getCount()); + vNameValues.add(vNameValue); + } + vBarGraphData.setDataCounts(vNameValues); + vBarGraphData.setName(xAxisField); + vBarGraphDatas.add(vBarGraphData); + dataList.setGraphData(vBarGraphDatas); + } + } + return convertObjToString(dataList); + } catch (SolrException | SolrServerException | IOException e) { + + } + } else { + queryGenerator.setRowCount(solrQuery, 0); + String yAxis = yAxisField.contains("count") ? "sum" : yAxisField; + String jsonQuery = queryGenerator + .buildJSONFacetAggregatedFuncitonQuery(yAxis, + xAxisField); + queryGenerator.setJSONFacet(solrQuery, jsonQuery); + try { + QueryResponse response = solrDaoBase.process(solrQuery); + + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response + .getResponse().get("facets"); + if (jsonFacetResponse.toString().equals("{count=0}")) + return convertObjToString(dataList); + VNameValue value = new VNameValue(); + String sum = jsonFacetResponse.getVal(1).toString(); + value.setName(xAxisField); + value.setValue(sum.substring(0, sum.indexOf("."))); + + vNameValues.add(value); + vBarGraphData.setDataCounts(vNameValues); + vBarGraphData.setName(xAxisField); + vBarGraphDatas.add(vBarGraphData); + dataList.setGraphData(vBarGraphDatas); + return convertObjToString(dataList); + } catch (SolrException | SolrServerException | IOException e) { + + } + } + + return null; + } + + @SuppressWarnings("unchecked") + private String nonRangeStackGraph(String xAxisField, String yAxisField, + String stackField, String from, String to, SolrDaoBase solrDaoBase, + String typeXAxis, String fieldTime, SolrQuery solrQuery) { + VBarDataList dataList = new VBarDataList(); + Collection<VBarGraphData> vGraphData = new ArrayList<VBarGraphData>(); + + String mainQuery = queryGenerator.buildInclusiveRangeFilterQuery( + fieldTime, from, to); + queryGenerator.setMainQuery(solrQuery, mainQuery); + queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + String jsonQuery = ""; + + if (isTypeNumber(typeXAxis)) { + String function = (yAxisField.contains("count")) ? "sum" + : yAxisField; + jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber( + stackField, xAxisField, function); + } else { + jsonQuery = queryGenerator.buildJsonFacetTermsRangeQuery( + stackField, xAxisField); + } + + try { + queryGenerator.setJSONFacet(solrQuery, jsonQuery); + dataList.setGraphData(vGraphData); + + QueryResponse response = solrDaoBase.process(solrQuery); + if (response == null) { + response = new QueryResponse(); + } + Long count = response.getResults().getNumFound(); + if (count <= 0) + return convertObjToString(dataList); + + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response + .getResponse().get("facets"); + if (jsonFacetResponse == null + || jsonFacetResponse.toString().equals("{count=0}")) { + return convertObjToString(dataList); + } + + extractNonRangeStackValuesFromBucket(jsonFacetResponse, stackField, vGraphData, + typeXAxis); + + if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField) + && LogSearchConstants.SOLR_LEVEL + .equalsIgnoreCase(xAxisField)) { + Collection<VBarGraphData> levelVGraphData = dataList.getGraphData(); + List<String> logLevels = new ArrayList<String>(); + + logLevels.add(LogSearchConstants.FATAL); + logLevels.add(LogSearchConstants.ERROR); + logLevels.add(LogSearchConstants.WARN); + logLevels.add(LogSearchConstants.INFO); + logLevels.add(LogSearchConstants.DEBUG); + logLevels.add(LogSearchConstants.TRACE); + + for (VBarGraphData garphData : levelVGraphData) { + Collection<VNameValue> valueList = garphData.getDataCount(); + Collection<VNameValue> valueListSorted = new ArrayList<VNameValue>(); + for (String level : logLevels) { + String val = "0"; + for (VNameValue value : valueList) { + if (value.getName().equalsIgnoreCase(level)) { + val = value.getValue(); + break; + } + } + VNameValue v1 = new VNameValue(); + v1.setName(level.toUpperCase()); + v1.setValue(val); + valueListSorted.add(v1); + } + garphData.setDataCounts(valueListSorted); + } + } + + return convertObjToString(dataList); + } catch (SolrException | IOException | SolrServerException e) { + logger.error(e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.DATA_NOT_FOUND); + } + + } + + @SuppressWarnings("unchecked") + private String rangeNonStackGraph(String xAxisField, String yAxisField, + String from, String to, String unit, SolrDaoBase solrDaoBase, + String typeXAxis, String fieldTime, SolrQuery solrQuery) { + VBarDataList dataList = new VBarDataList(); + Collection<VBarGraphData> vBarGraphDatas = new ArrayList<VBarGraphData>(); + VBarGraphData vBarGraphData = new VBarGraphData(); + Collection<VNameValue> vNameValues = new ArrayList<VNameValue>(); + + queryGenerator.setMainQuery(solrQuery, null); + + if (isTypeNumber(typeXAxis)) { + queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to); + return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, + typeXAxis, fieldTime, solrQuery); + } else { + try { + queryGenerator.setFacetRange(solrQuery, xAxisField, from, to, + unit); + QueryResponse response = solrDaoBase.process(solrQuery); + if (response == null) + response = new QueryResponse(); + Long count = response.getResults().getNumFound(); + if (count > 0) { + + @SuppressWarnings("rawtypes") + List<RangeFacet> rangeFacet = response.getFacetRanges(); + if (rangeFacet == null) + return convertObjToString(dataList); + + List<RangeFacet.Count> listCount = rangeFacet.get(0) + .getCounts(); + if (listCount != null) { + for (RangeFacet.Count cnt : listCount) { + VNameValue vNameValue = new VNameValue(); + vNameValue.setName(cnt.getValue()); + vNameValue.setValue("" + cnt.getCount()); + vNameValues.add(vNameValue); + } + vBarGraphData.setDataCounts(vNameValues); + vBarGraphDatas.add(vBarGraphData); + vBarGraphData.setName(xAxisField); + dataList.setGraphData(vBarGraphDatas); + } + } + return convertObjToString(dataList); + } catch (SolrException | SolrServerException | IOException e) { + + } + } + return null; + } + + @SuppressWarnings("unchecked") + private String rangeStackGraph(String xAxisField, String yAxisField, + String stackField, String from, String to, String unit, + SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) { + VBarDataList dataList = new VBarDataList(); + List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>(); + + queryGenerator.setMainQuery(solrQuery, null); + queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + + String jsonHistogramQuery = queryGenerator + .buildJSONFacetTermTimeRangeQuery(stackField, xAxisField, from, + to, unit).replace("\\", ""); + + try { + solrQuery.set("json.facet", jsonHistogramQuery); + queryGenerator.setRowCount(solrQuery, 0); + QueryResponse response = solrDaoBase.process(solrQuery); + if (response == null) + response = new QueryResponse(); + + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response + .getResponse().get("facets"); + + if (jsonFacetResponse == null + || jsonFacetResponse.toString().equals("{count=0}")) + return convertObjToString(dataList); + + extractRangeStackValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); + + dataList.setGraphData(histogramData); + return convertObjToString(dataList); + + } catch (SolrException | IOException | SolrServerException e) { + } + + return null; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java new file mode 100644 index 0000000..e16faa0 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java @@ -0,0 +1,810 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.manager; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.List; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.ManageStartEndTime; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.SearchCriteria; +import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.graph.GraphDataGnerator; +import org.apache.ambari.logsearch.query.QueryGeneration; +import org.apache.ambari.logsearch.util.BizUtil; +import org.apache.ambari.logsearch.util.ConfigUtil; +import org.apache.ambari.logsearch.util.DateUtil; +import org.apache.ambari.logsearch.util.JSONUtil; +import org.apache.ambari.logsearch.util.PropertiesUtil; +import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.StringUtil; +import org.apache.ambari.logsearch.view.VBarDataList; +import org.apache.ambari.logsearch.view.VBarGraphData; +import org.apache.ambari.logsearch.view.VGroupList; +import org.apache.ambari.logsearch.view.VNameValue; +import org.apache.ambari.logsearch.view.VNameValueList; +import org.apache.ambari.logsearch.view.VSolrLogList; +import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FacetField.Count; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.RangeFacet; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class AuditMgr extends MgrBase { + static Logger logger = Logger.getLogger(AuditMgr.class); + + @Autowired + QueryGeneration queryGenerator; + + @Autowired + AuditSolrDao auditSolrDao; + + @Autowired + RESTErrorUtil restErrorUtil; + + @Autowired + JSONUtil jsonUtil; + + @Autowired + StringUtil stringUtil; + + @Autowired + BizUtil bizUtil; + + @Autowired + DateUtil dateUtil; + + @Autowired + GraphDataGnerator graphDataGnerator; + + public String getAllSolrFields() { + + Collection<String> fieldNames = new ArrayList<String>(); + SolrQuery solrQuery = new SolrQuery(); + + int numberDocument = 0; + int size = 0; + + try { + queryGenerator.setMainQuery(solrQuery, null); + queryGenerator.setRowCount(solrQuery, 10); + + SearchCriteria searchCriteria = new SearchCriteria(); + searchCriteria.setSortBy(LogSearchConstants.AUDIT_EVTTIME); + searchCriteria.setSortType(SolrQuery.ORDER.asc.toString()); + + queryGenerator.setSortOrderDefaultServiceLog(solrQuery, + searchCriteria); + SolrDocumentList docList = auditSolrDao.process(solrQuery) + .getResults(); + + if (docList.size() > 0) { + for (int i = 0; i < 10; i++) { + if (docList.get(i).size() > size) { + size = docList.get(i).size(); + numberDocument = i; + } + } + fieldNames = docList.get(numberDocument).getFieldNames(); + return convertObjToString(fieldNames); + } + return convertObjToString(fieldNames); + + } catch (SolrException | SolrServerException | IOException e) { + logger.error(e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + public String getLogs(SearchCriteria searchCriteria) { + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + try { + QueryResponse response = auditSolrDao.process(solrQuery); + SolrDocumentList docList = response.getResults(); + VSolrLogList collection = new VSolrLogList(docList); + collection.setStartIndex((int) docList.getStart()); + collection.setTotalCount(docList.getNumFound()); + collection.setPageSize(searchCriteria.getMaxRows()); + return convertObjToString(collection); + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + public SolrDocumentList getComponents(SearchCriteria searchCriteria) { + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + try { + queryGenerator.setFacetField(solrQuery, + LogSearchConstants.AUDIT_COMPONENT); + queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + List<FacetField> facetFields = null; + List<Count> componentsCount = new ArrayList<Count>(); + FacetField facetField = null; + + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse == null) { + queryResponse = new QueryResponse(); + } + + facetFields = queryResponse.getFacetFields(); + if (facetFields == null) { + return new SolrDocumentList(); + } + if (!facetFields.isEmpty()) { + facetField = facetFields.get(0); + } + if (facetField != null) { + componentsCount = facetField.getValues(); + } + SolrDocumentList docList = new SolrDocumentList(); + for (Count compnonet : componentsCount) { + SolrDocument solrDocument = new SolrDocument(); + solrDocument.addField("type", compnonet.getName()); + docList.add(solrDocument); + } + return docList; + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + public String getAuditComponents(SearchCriteria searchCriteria) { + VGroupList vGroupList = new VGroupList(); + + try { + + SolrDocumentList docList = getComponents(searchCriteria); + + vGroupList.setGroupDocuments(docList); + return convertObjToString(vGroupList); + } catch (SolrException | IOException e) { + logger.error("Error", e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + @SuppressWarnings("unchecked") + public String getAuditLineGraphData(SearchCriteria searchCriteria) { + VBarDataList dataList = new VBarDataList(); + SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + + String from = (String) searchCriteria.getParamValue("startTime"); + String to = (String) searchCriteria.getParamValue("endTime"); + String unit = (String) searchCriteria.getParamValue("unit"); + + List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>(); + String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery( + LogSearchConstants.AUDIT_COMPONENT, + LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", + ""); + + try { + queryGenerator.setJSONFacet(solrQuery, jsonHistogramQuery); + queryGenerator.setRowCount(solrQuery, 0); + QueryResponse response = auditSolrDao.process(solrQuery); + if (response == null) + response = new QueryResponse(); + + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response + .getResponse().get("facets"); + + if (jsonFacetResponse == null + || jsonFacetResponse.toString().equals("{count=0}")) + return convertObjToString(dataList); + + extractValuesFromBucket(jsonFacetResponse, "x", "y", + histogramData); + + dataList.setGraphData(histogramData); + return convertObjToString(dataList); + + } catch (SolrServerException | SolrException | IOException e) { + logger.error(e); + throw restErrorUtil.createRESTException("No Request Parameter", + MessageEnums.ERROR_SYSTEM); + + } + } + + public String getTopAuditFieldCount(SearchCriteria searchCriteria) { + Integer top = (Integer) searchCriteria.getParamValue("top"); + String facetField = (String) searchCriteria.getParamValue("field"); + if (top == null) + top = new Integer(10); + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + try { + + List<VNameValue> nameValues = new ArrayList<VNameValue>(); + + VNameValueList nameValueList = new VNameValueList(nameValues); + + queryGenerator.setFacetField(solrQuery, facetField); + queryGenerator.setFacetSort(solrQuery, "count"); + queryGenerator.setFacetLimit(solrQuery, top.intValue()); + + List<Count> countList = new ArrayList<FacetField.Count>(); + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse.getFacetField(facetField) != null) { + FacetField queryFacetField = queryResponse + .getFacetField(facetField); + if (queryFacetField != null) { + countList = queryFacetField.getValues(); + } + } + + for (Count cnt : countList) { + VNameValue nameValue = new VNameValue(); + nameValue.setName(cnt.getName()); + + nameValue.setValue("" + cnt.getCount()); + nameValues.add(nameValue); + } + return convertObjToString(nameValueList); + + } catch (SolrException | IOException | SolrServerException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + @SuppressWarnings("unchecked") + public String getLiveLogCounts() { + SolrQuery solrQuery = new SolrQuery(); + solrQuery.setParam("event", "/getLiveLogsCount"); + try { + String startDate = dateUtil + .convertGivenDateFormatToSolrDateFormat(ManageStartEndTime.startDate); + + String endDate = dateUtil + .convertGivenDateFormatToSolrDateFormat(ManageStartEndTime.endDate); + + queryGenerator.setMainQuery(solrQuery, null); + queryGenerator.setFacetRange(solrQuery, + LogSearchConstants.AUDIT_EVTTIME, startDate, endDate, + "+2MINUTE"); + List<RangeFacet.Count> listCount; + + QueryResponse response = auditSolrDao.process(solrQuery); + @SuppressWarnings("rawtypes") + List<RangeFacet> rangeFacet = response.getFacetRanges(); + if (rangeFacet == null) + return convertObjToString(new VNameValueList()); + listCount = rangeFacet.get(0).getCounts(); + + List<VNameValue> nameValues = new ArrayList<VNameValue>(); + int count = 0; + for (RangeFacet.Count cnt : listCount) { + VNameValue nameValue = new VNameValue(); + nameValue.setName("" + count); + nameValue.setValue("" + cnt.getCount()); + nameValues.add(nameValue); + count++; + } + VNameValueList nameValueList = new VNameValueList(nameValues); + + return convertObjToString(nameValueList); + + } catch (SolrException | SolrServerException | ParseException + | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + public String topTenUsers(SearchCriteria searchCriteria) { + + String jsonUserQuery = "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}"; + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + solrQuery.set("json.facet", jsonUserQuery); + queryGenerator.setRowCount(solrQuery, 0); + try { + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + + NamedList<Object> namedList = queryResponse.getResponse(); + VBarDataList vBarDataList = new VBarDataList(); + if (namedList == null) { + return convertObjToString(vBarDataList); + } + + @SuppressWarnings("unchecked") + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList + .get("facets"); + vBarDataList = bizUtil.buildSummaryForTopCounts(jsonFacetResponse); + return convertObjToString(vBarDataList); + + } catch (SolrServerException | SolrException | IOException e) { + logger.error("Error during solrQuery=" + e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + public String topTenResources(SearchCriteria searchCriteria) { + + String jsonUserQuery = "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}"; + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + solrQuery.set("json.facet", jsonUserQuery); + queryGenerator.setRowCount(solrQuery, 0); + try { + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + + NamedList<Object> namedList = queryResponse.getResponse(); + VBarDataList vBarDataList = new VBarDataList(); + if (namedList == null) { + return convertObjToString(vBarDataList); + } + + @SuppressWarnings("unchecked") + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList + .get("facets"); + + vBarDataList = bizUtil.buildSummaryForTopCounts(jsonFacetResponse); + return convertObjToString(vBarDataList); + + } catch (SolrServerException | SolrException | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + @SuppressWarnings("unchecked") + public String getRequestUserLineGraph(SearchCriteria searchCriteria) { + + String from = (String) searchCriteria.getParamValue("startTime"); + String to = (String) searchCriteria.getParamValue("endTime"); + String unit = (String) searchCriteria.getParamValue("unit"); + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + + VBarDataList dataList = new VBarDataList(); + List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>(); + + queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + + String jsonHistogramQuery = queryGenerator + .buildJSONFacetTermTimeRangeQuery( + LogSearchConstants.AUDIT_REQUEST_USER, + LogSearchConstants.AUDIT_EVTTIME, from, to, unit) + .replace("\\", ""); + + try { + solrQuery.set("json.facet", jsonHistogramQuery); + queryGenerator.setRowCount(solrQuery, 0); + QueryResponse response = auditSolrDao.process(solrQuery); + if (response == null) + response = new QueryResponse(); + + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response + .getResponse().get("facets"); + + if (jsonFacetResponse == null + || jsonFacetResponse.toString().equals("{count=0}")) + return convertObjToString(dataList); + + extractValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); + + dataList.setGraphData(histogramData); + return convertObjToString(dataList); + + } catch (SolrException | IOException | SolrServerException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + + } + + public SolrDocumentList getRequestUser(SearchCriteria searchCriteria) { + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + try { + queryGenerator.setFacetField(solrQuery, + LogSearchConstants.AUDIT_REQUEST_USER); + queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); + List<FacetField> facetFields = null; + List<Count> componentsCount = new ArrayList<Count>(); + FacetField facetField = null; + + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + if (queryResponse == null) { + queryResponse = new QueryResponse(); + } + + facetFields = queryResponse.getFacetFields(); + if (facetFields == null) { + return new SolrDocumentList(); + } + if (!facetFields.isEmpty()) { + facetField = facetFields.get(0); + } + if (facetField != null) { + componentsCount = facetField.getValues(); + } + SolrDocumentList docList = new SolrDocumentList(); + for (Count compnonet : componentsCount) { + SolrDocument solrDocument = new SolrDocument(); + solrDocument.addField("type", compnonet.getName()); + docList.add(solrDocument); + } + return docList; + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error during solrQuery=" + solrQuery, e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + public String getAuditLogsSchemaFieldsName() { + String suffix = PropertiesUtil.getProperty("auditlog.solr.core.logs"); + String excludeArray[] = PropertiesUtil + .getPropertyStringList("auditlog.exclude.columnlist"); + List<String> fieldNames = new ArrayList<String>(); + HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>(); + ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames); + + for (String fieldName : fieldNames) { + String uiField = ConfigUtil.auditLogsColumnMapping.get(fieldName + + LogSearchConstants.SOLR_SUFFIX); + if (uiField == null) { + uiFieldColumnMapping.put(fieldName, fieldName); + } else { + uiFieldColumnMapping.put(fieldName, uiField); + } + } + + try { + uiFieldColumnMapping = bizUtil + .sortHashMapByValuesD(uiFieldColumnMapping); + return convertObjToString(uiFieldColumnMapping); + } catch (IOException e) { + logger.error(e); + } + throw restErrorUtil.createRESTException( + "Cache is Empty for FieldsName", MessageEnums.DATA_NOT_FOUND); + } + + public String getAnyGraphData(SearchCriteria searchCriteria) { + searchCriteria.addParam("feildTime", LogSearchConstants.AUDIT_EVTTIME); + String suffix = PropertiesUtil.getProperty("auditlog.solr.core.logs"); + searchCriteria.addParam("suffix", suffix); + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + String result = graphDataGnerator.getAnyGraphData(searchCriteria, + auditSolrDao, solrQuery); + if (result != null) + return result; + try { + return convertObjToString(new VBarDataList()); + } catch (IOException e) { + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + @SuppressWarnings("unchecked") + public void extractValuesFromBucket( + SimpleOrderedMap<Object> jsonFacetResponse, String outerField, + String innerField, List<VBarGraphData> histogramData) { + NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse + .get(outerField); + ArrayList<Object> stackBuckets = (ArrayList<Object>) stack + .get("buckets"); + for (Object temp : stackBuckets) { + VBarGraphData vBarGraphData = new VBarGraphData(); + + SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp; + String name = ((String) level.getVal(0)).toUpperCase(); + vBarGraphData.setName(name); + + Collection<VNameValue> vNameValues = new ArrayList<VNameValue>(); + vBarGraphData.setDataCounts(vNameValues); + ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level + .get(innerField)).get("buckets"); + for (Object temp1 : levelBuckets) { + SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1; + String value = dateUtil + .convertDateWithMillisecondsToSolrDate((Date) countValue + .getVal(0)); + + String count = "" + countValue.getVal(1); + VNameValue vNameValue = new VNameValue(); + vNameValue.setName(value); + vNameValue.setValue(count); + vNameValues.add(vNameValue); + } + histogramData.add(vBarGraphData); + } + } + + @SuppressWarnings({"unchecked", "resource"}) + public Response exportUserTableToTextFile(SearchCriteria searchCriteria) { + String jsonUserQuery = "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}"; + + SolrQuery solrQuery = queryGenerator + .commonAuditFilterQuery(searchCriteria); + String startTime = (String) searchCriteria.getParamValue("startTime"); + String endTime = (String) searchCriteria.getParamValue("endTime"); + + startTime = startTime == null ? "" : startTime; + endTime = endTime == null ? "" : "_" + endTime; + + queryGenerator.setJSONFacet(solrQuery, jsonUserQuery); + queryGenerator.setRowCount(solrQuery, 0); + + String dataFormat = (String) searchCriteria.getParamValue("format"); + try { + QueryResponse queryResponse = auditSolrDao.process(solrQuery); + + NamedList<Object> namedList = queryResponse.getResponse(); + VBarDataList vBarUserDataList = new VBarDataList(); + VBarDataList vBarResourceDataList = new VBarDataList(); + if (namedList == null) { + + } + + SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList + .get("facets"); + vBarUserDataList = bizUtil + .buildSummaryForTopCounts(jsonFacetResponse); + vBarResourceDataList = bizUtil + .buildSummaryForResourceCounts(jsonFacetResponse); + String data = ""; + String summary = ""; + if ("text".equals(dataFormat)) { + int users = 0; + int resources = 0; + summary += "\n\n\n\n"; + data += addBlank("Users") + "Components/Access" + "\n"; + data += "--------------------------------------------------------------------------\n"; + Collection<VBarGraphData> tableUserData = vBarUserDataList + .getGraphData(); + for (VBarGraphData graphData : tableUserData) { + String userName = graphData.getName(); + String largeUserName = ""; + + if (userName.length() > 45) { + largeUserName = userName.substring(0, 45); + data += addBlank(largeUserName); + } else + data += addBlank(userName); + + Collection<VNameValue> vnameValueList = graphData + .getDataCount(); + int count = 0; + String blank = ""; + for (VNameValue vNameValue : vnameValueList) { + data += blank + vNameValue.getName() + " " + + vNameValue.getValue() + "\n"; + if (count == 0) + blank = addBlank(blank); + count++; + + } + while (largeUserName.length() > 0) { + data += largeUserName.substring(0, 45) + "\n"; + } + + users += 1; + } + data += "\n\n\n\n\n\n"; + data += addBlank("Resources") + "Components/Access" + "\n"; + data += "--------------------------------------------------------------------------\n"; + Collection<VBarGraphData> tableResourceData = vBarResourceDataList + .getGraphData(); + for (VBarGraphData graphData : tableResourceData) { + String resourceName = graphData.getName(); + String largeResourceName = resourceName; + if (largeResourceName.length() > 45) { + resourceName = largeResourceName.substring(0, 45); + largeResourceName = largeResourceName.substring(45, largeResourceName.length()); + } else { + largeResourceName = ""; + } + + //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n"); + data += addBlank(resourceName); + Collection<VNameValue> vnameValueList = graphData + .getDataCount(); + int count = 0; + String blank = ""; + for (VNameValue vNameValue : vnameValueList) { + data += blank + vNameValue.getName() + " " + + vNameValue.getValue() + "\n"; + if (count == 0) + blank = addBlank(blank); + count++; + } + String tempLargeResourceName = largeResourceName; + while (largeResourceName.length() > 45) { + largeResourceName = tempLargeResourceName.substring(0, 45); + tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length()); + data += largeResourceName + "\n"; + } + if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) { + data += largeResourceName + "\n"; + } + resources += 1; + } + String header = "--------------------------------SUMMARY-----------------------------------\n"; + summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary; + data = summary + data; + } else { + data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}"; + dataFormat = "json"; + } + String fileName = "Users_Resource" + startTime + endTime + + "."; + File file = File.createTempFile(fileName, dataFormat); + + FileOutputStream fis = new FileOutputStream(file); + fis.write(data.getBytes()); + return Response + .ok(file, MediaType.APPLICATION_OCTET_STREAM) + .header("Content-Disposition", + "attachment;filename=" + fileName + dataFormat) + .build(); + + } catch (SolrServerException | SolrException | IOException e) { + logger.error("Error during solrQuery=" + e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + } + + private String addBlank(String field) { + int blanks = 50; + int strSize = field.length(); + String fieldWithBlank = field; + for (int i = 0; i < blanks - strSize; i++) { + fieldWithBlank += " "; + } + return fieldWithBlank; + } + + public String getServiceLoad(SearchCriteria searchCriteria) { + VBarDataList dataList = new VBarDataList(); + Collection<VBarGraphData> vaDatas = new ArrayList<VBarGraphData>(); + dataList.setGraphData(vaDatas); + + SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + + try { + queryGenerator.setFacetField(serivceLoadQuery, + LogSearchConstants.AUDIT_COMPONENT); + QueryResponse serviceLoadResponse = auditSolrDao + .process(serivceLoadQuery); + if (serviceLoadResponse == null) + return "[]"; + List<Count> serviceLoadFacets = serviceLoadResponse.getFacetField( + LogSearchConstants.AUDIT_COMPONENT).getValues(); + for (Count cnt : serviceLoadFacets) { + List<VNameValue> valueList = new ArrayList<VNameValue>(); + VBarGraphData vBarGraphData = new VBarGraphData(); + vaDatas.add(vBarGraphData); + VNameValue vNameValue = new VNameValue(); + vNameValue.setName(cnt.getName()); + vBarGraphData.setName(cnt.getName().toUpperCase()); + vNameValue.setValue("" + cnt.getCount()); + valueList.add(vNameValue); + vBarGraphData.setDataCounts(valueList); + } + + + return convertObjToString(dataList); + + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error during solrQuery=" + e); + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } + + /* + String preDefinedJSON = getHadoopServiceConfigJSON(); + try { + JSONObject serviceJSON = new JSONObject(preDefinedJSON).getJSONObject("service"); + HashMap<String, Object> serviceMap = jsonUtil.jsonToMapObject(serviceJSON.toString()); + Iterator<Entry<String, Object>> serviceMapIterator= serviceMap.entrySet().iterator(); + List<VNameValue> newValueList = new ArrayList<VNameValue>(); + for (VNameValue vNameValue : valueList) { + String name=vNameValue.getName(); + while (serviceMapIterator.hasNext()) { + Map.Entry<String, Object> tempMap = serviceMapIterator + .next(); + + String keyName = tempMap.getKey(); + + JSONObject valueObj = new JSONObject(tempMap.toString().replace(keyName+"=", "")); + if(name.contains(keyName.toLowerCase())){ + vNameValue.setName(valueObj.getString("label")); + break; + } + JSONArray componentsArray = valueObj.getJSONArray("components"); + + for(int i =0;i< componentsArray.length();i++){ + JSONObject jObj = componentsArray.getJSONObject(i); + String jsonName = jObj.getString("name"); + if(name.contains(jsonName.toLowerCase())){ + vNameValue.setName(valueObj.getString("label")); + break; + } + } + + } + if(newValueList.isEmpty()){ + newValueList.add(vNameValue); + }else{ + boolean isMatch = false; + for(VNameValue vValue: newValueList){ + if(vValue.getName().equalsIgnoreCase(vNameValue.getName())){ + isMatch =true; + Integer cnt1 = Integer.parseInt(vValue.getValue()); + Integer cnt2 = Integer.parseInt(vNameValue.getValue()); + vValue.setValue((cnt1+cnt2)+""); + } + } + if(!isMatch) + newValueList.add(vNameValue); + } + } + vBarGraphData.setDataCounts(newValueList); + vBarGraphData.setName("ServiceList"); + return convertObjToString(dataList); + + } catch (Exception e) { + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + }*/ + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/39c85bb8/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java new file mode 100644 index 0000000..0388366 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogFileMgr.java @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.manager; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.SearchCriteria; +import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; +import org.apache.ambari.logsearch.dao.SolrDaoBase; +import org.apache.ambari.logsearch.view.VLogFile; +import org.apache.ambari.logsearch.view.VLogFileList; +import org.apache.ambari.logsearch.view.VSolrLogList; +import org.apache.commons.io.FilenameUtils; +import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.FacetField.Count; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + + +@Component +public class LogFileMgr extends MgrBase { + + private static Logger logger = Logger.getLogger(LogFileMgr.class); + + private enum LOG_TYPE { + SERVICE, AUDIT + } + + @Autowired + ServiceLogsSolrDao serviceLogsSolrDao; + + @Autowired + AuditSolrDao auditSolrDao; + + @Autowired + LogsMgr logMgr; + + /** + * Search logFiles + * + * @param searchCriteria + * @return + */ + public String searchLogFiles(SearchCriteria searchCriteria) { + VLogFileList logFileList = new VLogFileList(); + List<VLogFile> logFiles = new ArrayList<VLogFile>(); + String componentName = (String) searchCriteria.getParamValue("component"); + String host = (String) searchCriteria.getParamValue("host"); + int minCount = 1;// to remove zero count facet + SolrQuery solrQuery = new SolrQuery(); + queryGenrator.setMainQuery(solrQuery, null); + queryGenrator.setFacetField(solrQuery, LogSearchConstants.SOLR_PATH, minCount); + // adding filter + queryGenrator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_COMPONENT, componentName); + queryGenrator.setSingleIncludeFilter(solrQuery, LogSearchConstants.SOLR_HOST, host); + try { + String logType = (String) searchCriteria.getParamValue("logType"); + if (stringUtil.isEmpty(logType)) { + logType = LOG_TYPE.SERVICE.name();// default is service Log + } + SolrDaoBase daoMgr = null; + if (logType.equalsIgnoreCase(LOG_TYPE.SERVICE.name())) { + daoMgr = serviceLogsSolrDao; + } else if (logType.equalsIgnoreCase(LOG_TYPE.AUDIT.name())) { + daoMgr = auditSolrDao; + } else { + throw restErrorUtil.createRESTException(logType + " is not a valid logType", MessageEnums.INVALID_INPUT_DATA); + } + QueryResponse queryResponse = daoMgr.process(solrQuery); + if (queryResponse.getFacetField(LogSearchConstants.SOLR_PATH) != null) { + FacetField queryFacetField = queryResponse.getFacetField(LogSearchConstants.SOLR_PATH); + if (queryFacetField != null) { + List<Count> countList = queryFacetField.getValues(); + for (Count count : countList) { + VLogFile vLogFile = new VLogFile(); + String filePath = count.getName(); + String fileName = FilenameUtils.getName(filePath); + vLogFile.setPath(filePath); + vLogFile.setName(fileName); + logFiles.add(vLogFile); + } + } + } + } catch (SolrException | SolrServerException | IOException e) { + logger.error("Error in solr query :" + e.getLocalizedMessage() + "\n Query :" + solrQuery.toQueryString(), + e.getCause()); + throw restErrorUtil.createRESTException("Please try later.", MessageEnums.ERROR_SYSTEM); + } + logFileList.setLogFiles(logFiles); + String jsonStr = ""; + try { + jsonStr = convertObjToString(logFileList); + } catch (IOException e) { + logger.error(e); + throw restErrorUtil.createRESTException("Please try later.", MessageEnums.ERROR_SYSTEM); + } + return jsonStr; + } + + public String getLogFileTail(SearchCriteria searchCriteria) { + String host = (String) searchCriteria.getParamValue("host"); + String logFile = (String) searchCriteria.getParamValue("name"); + String component = (String) searchCriteria.getParamValue("component"); + String tailSize = (String) searchCriteria.getParamValue("tailSize"); + if (stringUtil.isEmpty(host)) { + throw restErrorUtil.createRESTException("missing Host Name", + MessageEnums.ERROR_SYSTEM); + } + tailSize = (stringUtil.isEmpty(tailSize)) ? "10" : tailSize; + SolrQuery logFileTailQuery = new SolrQuery(); + try { + int tail = Integer.parseInt(tailSize); + tail = tail > 100 ? 100 : tail; + queryGenrator.setMainQuery(logFileTailQuery, null); + queryGenrator.setSingleIncludeFilter(logFileTailQuery, + LogSearchConstants.SOLR_HOST, host); + if (!stringUtil.isEmpty(logFile)) { + queryGenrator.setSingleIncludeFilter(logFileTailQuery, + LogSearchConstants.SOLR_PATH, + solrUtil.makeSolrSearchString(logFile)); + } else if (!stringUtil.isEmpty(component)) { + queryGenrator.setSingleIncludeFilter(logFileTailQuery, + LogSearchConstants.SOLR_COMPONENT, component); + } else { + throw restErrorUtil.createRESTException("component or logfile parameter must be present", + MessageEnums.ERROR_SYSTEM); + } + + queryGenrator.setRowCount(logFileTailQuery, tail); + queryGenrator.setSortOrderDefaultServiceLog(logFileTailQuery, new SearchCriteria()); + VSolrLogList solrLogList = getLogAsPaginationProvided(logFileTailQuery, serviceLogsSolrDao); + return convertObjToString(solrLogList); + + } catch (SolrException | IOException e) { + throw restErrorUtil.createRESTException(e.getMessage(), + MessageEnums.ERROR_SYSTEM); + } catch (NumberFormatException ne) { + + throw restErrorUtil.createRESTException(ne.getMessage(), + MessageEnums.ERROR_SYSTEM); + + } + } +}
