http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java deleted file mode 100644 index c57e0e9..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGeneratorBase.java +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.ambari.logsearch.graph; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.List; - -import org.apache.ambari.logsearch.manager.ManagerBase; -import org.apache.ambari.logsearch.model.response.BarGraphData; -import org.apache.ambari.logsearch.model.response.NameValueData; -import org.apache.ambari.logsearch.util.DateUtil; -import org.apache.commons.lang.StringUtils; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; - -class GraphDataGeneratorBase { - - private static final String BUCKETS = "buckets"; - - private static enum DataType { - LONG("long"), - DOUBLE("double"), - FLOAT("long"), - INT("long"); - - private String type; - - DataType(String type) { - this.type = type; - } - - String getType() { - return type; - } - } - - protected static enum GraphType { - UNKNOWN, - NORMAL_GRAPH, - RANGE_NON_STACK_GRAPH, - NON_RANGE_STACK_GRAPH, - RANGE_STACK_GRAPH; - } - - @SuppressWarnings("unchecked") - protected void extractRangeStackValuesFromBucket(SimpleOrderedMap<Object> jsonFacetResponse, String outerField, - String innerField, List<BarGraphData> histogramData) { - if (jsonFacetResponse != null) { - NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse.get(outerField); - if (stack != null) { - ArrayList<Object> stackBuckets = (ArrayList<Object>) stack.get(BUCKETS); - if (stackBuckets != null) { - for (Object stackBucket : stackBuckets) { - BarGraphData vBarGraphData = new BarGraphData(); - SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) stackBucket; - if (level != null) { - String name = level.getVal(0) != null ? level.getVal(0).toString().toUpperCase() : ""; - vBarGraphData.setName(name); - Collection<NameValueData> vNameValues = new ArrayList<NameValueData>(); - NamedList<Object> innerFiledValue = (NamedList<Object>) level.get(innerField); - if (innerFiledValue != null) { - ArrayList<Object> levelBuckets = (ArrayList<Object>) innerFiledValue.get(BUCKETS); - if (levelBuckets != null) { - for (Object levelBucket : levelBuckets) { - SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) levelBucket; - if (countValue != null) { - String innerName = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0)); - String innerValue = countValue.getVal(1) != null ? countValue.getVal(1).toString() : ""; - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(innerName); - vNameValue.setValue(innerValue); - vNameValues.add(vNameValue); - } - } - } - } - vBarGraphData.setDataCount(vNameValues); - } - histogramData.add(vBarGraphData); - } - } - } - } - } - - @SuppressWarnings("unchecked") - protected boolean extractNonRangeStackValuesFromBucket(SimpleOrderedMap<Object> jsonFacetResponse, String level, - Collection<BarGraphData> vGraphDatas, String typeXAxis) { - boolean zeroFlag = true; - if (jsonFacetResponse == null || jsonFacetResponse.get(level) == null - || jsonFacetResponse.get(level).toString().equals("{count=0}")) { - return false; - } - NamedList<Object> levelList = (NamedList<Object>) jsonFacetResponse.get(level); - if (levelList != null) { - ArrayList<Object> bucketList = (ArrayList<Object>) levelList.get(BUCKETS); - if (bucketList != null) { - for (int index = 0; index < bucketList.size(); index++) { - SimpleOrderedMap<Object> valueCount = (SimpleOrderedMap<Object>) bucketList.get(index); - if (valueCount != null && valueCount.size() > 2) { - BarGraphData vGraphData = new BarGraphData(); - Collection<NameValueData> levelCounts = new ArrayList<NameValueData>(); - String name = valueCount.getVal(0) != null ? valueCount.getVal(0).toString().trim() : ""; - if (isTypeNumber(typeXAxis)) { - NameValueData nameValue = new NameValueData(); - Double sumValue = (Double) valueCount.getVal(2); - String value = "0";// default is zero - if (sumValue != null) { - value = "" + sumValue.longValue(); - } - nameValue.setName(name); - nameValue.setValue(value); - levelCounts.add(nameValue); - } else { - SimpleOrderedMap<Object> valueCountMap = (SimpleOrderedMap<Object>) valueCount.getVal(2); - if (valueCountMap != null) { - ArrayList<Object> buckets = (ArrayList<Object>) valueCountMap.get(BUCKETS); - if (buckets != null) { - for (int innerIndex = 0; innerIndex < buckets.size(); innerIndex++) { - SimpleOrderedMap<Object> innerValueCount = (SimpleOrderedMap<Object>) buckets.get(innerIndex); - if (innerValueCount != null) { - String innerName = innerValueCount.getVal(0) != null ? innerValueCount.getVal(0).toString().trim() : ""; - String innerValue = innerValueCount.getVal(1) != null ? innerValueCount.getVal(1).toString().trim() : ""; - NameValueData nameValue = new NameValueData(); - nameValue.setValue(innerValue); - nameValue.setName(innerName); - levelCounts.add(nameValue); - } - } - } - } - } - vGraphData.setName(name); - vGraphData.setDataCount(levelCounts); - vGraphDatas.add(vGraphData); - } - } - } - } - return zeroFlag; - } - - protected boolean isTypeNumber(String typeXAxis) { - if (StringUtils.isBlank(typeXAxis)) { - return false; - } else { - return typeXAxis.contains(DataType.LONG.getType()) || typeXAxis.contains(DataType.INT.getType()) - || typeXAxis.contains(DataType.FLOAT.getType()) || typeXAxis.contains(DataType.DOUBLE.getType()); - } - } -}
http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java index 52482c2..56ba2d7 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java @@ -20,398 +20,151 @@ package org.apache.ambari.logsearch.manager; import javax.inject.Inject; +import javax.inject.Named; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; -import java.text.ParseException; +import java.io.StringWriter; import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; -import org.apache.ambari.logsearch.common.LogSearchConstants; -import org.apache.ambari.logsearch.common.ManageStartEndTime; +import freemarker.template.Configuration; +import freemarker.template.Template; +import freemarker.template.TemplateException; import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; import org.apache.ambari.logsearch.dao.AuditSolrDao; import org.apache.ambari.logsearch.graph.GraphDataGenerator; +import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest; +import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest; +import org.apache.ambari.logsearch.model.request.impl.AuditLogRequest; +import org.apache.ambari.logsearch.model.request.impl.AuditServiceLoadRequest; +import org.apache.ambari.logsearch.model.request.impl.FieldAuditLogRequest; +import org.apache.ambari.logsearch.model.request.impl.UserExportRequest; import org.apache.ambari.logsearch.model.response.AuditLogResponse; import org.apache.ambari.logsearch.model.response.BarGraphData; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; import org.apache.ambari.logsearch.model.response.GroupListResponse; import org.apache.ambari.logsearch.model.response.LogData; -import org.apache.ambari.logsearch.model.response.LogSearchResponse; import org.apache.ambari.logsearch.model.response.NameValueData; -import org.apache.ambari.logsearch.model.response.NameValueDataListResponse; -import org.apache.ambari.logsearch.query.model.AnyGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.AuditBarGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.CommonSearchCriteria; -import org.apache.ambari.logsearch.query.model.FieldAuditBarGraphSearchCriteria; -import org.apache.ambari.logsearch.query.model.FieldAuditLogSearchCriteria; -import org.apache.ambari.logsearch.query.model.UserExportSearchCriteria; +import org.apache.ambari.logsearch.model.response.TemplateData; +import org.apache.ambari.logsearch.solr.SolrConstants; import org.apache.ambari.logsearch.solr.model.SolrAuditLogData; import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData; -import org.apache.ambari.logsearch.util.BizUtil; -import org.apache.ambari.logsearch.util.DateUtil; +import org.apache.ambari.logsearch.util.DownloadUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; -import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.ambari.logsearch.view.VResponse; -import org.apache.ambari.logsearch.query.model.AuditLogSearchCriteria; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.RangeFacet; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.springframework.stereotype.Component; +import org.springframework.core.convert.ConversionService; +import org.springframework.data.solr.core.query.SimpleFacetQuery; +import org.springframework.data.solr.core.query.SimpleQuery; -@Component +import static org.apache.ambari.logsearch.solr.SolrConstants.AuditLogConstants.AUDIT_COMPONENT; + +@Named public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResponse> { private static final Logger logger = Logger.getLogger(AuditLogsManager.class); + private static final String AUDIT_LOG_TEMPLATE = "audit_log_txt.ftl"; + @Inject private AuditSolrDao auditSolrDao; @Inject private GraphDataGenerator graphDataGenerator; @Inject - private SolrAuditLogPropsConfig solrAuditLogPropsConfig; + private ConversionService conversionService; + @Inject + private Configuration freemarkerConfiguration; - public AuditLogResponse getLogs(AuditLogSearchCriteria searchCriteria) { - Boolean isLastPage = searchCriteria.isLastPage(); - if (isLastPage) { - SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - LogSearchResponse logResponse = getLastPage(searchCriteria, LogSearchConstants.AUDIT_EVTTIME, auditSolrDao, lastPageQuery); - if (logResponse == null) { - logResponse = new AuditLogResponse(); - } - return (AuditLogResponse) logResponse; - } - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - return getLogAsPaginationProvided(solrQuery, auditSolrDao); + public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) { + return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs"); } - private List<LogData> getComponents(CommonSearchCriteria searchCriteria) { - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + private List<LogData> getComponents(AuditComponentRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); List<LogData> docList = new ArrayList<>(); - try { - SolrUtil.setFacetField(solrQuery, LogSearchConstants.AUDIT_COMPONENT); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - List<FacetField> facetFields = null; - List<Count> componentsCount = new ArrayList<Count>(); - FacetField facetField = null; + QueryResponse queryResponse = auditSolrDao.process(facetQuery); + List<Count> componentsCount = graphDataGenerator.generateCount(queryResponse); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return docList; - } - - facetFields = queryResponse.getFacetFields(); - if (facetFields == null) { - return docList; - } - if (!facetFields.isEmpty()) { - facetField = facetFields.get(0); - } - if (facetField != null) { - componentsCount = facetField.getValues(); - } - - for (Count component : componentsCount) { - SolrComponentTypeLogData logData = new SolrComponentTypeLogData(); - logData.setType(component.getName()); - docList.add(logData); - } - return docList; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + for (Count component : componentsCount) { + SolrComponentTypeLogData logData = new SolrComponentTypeLogData(); + logData.setType(component.getName()); + docList.add(logData); } + return docList; } - public GroupListResponse getAuditComponents(CommonSearchCriteria searchCriteria) { + public GroupListResponse getAuditComponents(AuditComponentRequest request) { GroupListResponse componentResponse = new GroupListResponse(); - List<LogData> docList = getComponents(searchCriteria); + List<LogData> docList = getComponents(request); componentResponse.setGroupList(docList); return componentResponse; } - @SuppressWarnings("unchecked") - public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphSearchCriteria searchCriteria) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - String from = getFrom(searchCriteria.getStartTime()); - String to = getTo(searchCriteria.getEndTime()); - String unit = getUnit(searchCriteria.getUnit()); - - List<BarGraphData> histogramData = new ArrayList<BarGraphData>(); - String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_COMPONENT, - LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); - - return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery); - } - - @SuppressWarnings({"unchecked", "rawtypes"}) - public NameValueDataListResponse getLiveLogCounts() { - NameValueDataListResponse nameValueList = new NameValueDataListResponse(); - SolrQuery solrQuery = new SolrQuery(); - solrQuery.setParam("event", "/audit/logs/live/count"); - try { - Date[] timeRange = ManageStartEndTime.getStartEndTime(); - String startDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[0]); - String endDate = DateUtil.convertGivenDateFormatToSolrDateFormat(timeRange[1]); - - SolrUtil.setMainQuery(solrQuery, null); - SolrUtil.setFacetRange(solrQuery, LogSearchConstants.AUDIT_EVTTIME, startDate, endDate, "+2MINUTE"); - List<RangeFacet.Count> listCount; - - QueryResponse response = auditSolrDao.process(solrQuery); - - List<RangeFacet> rangeFacet = response.getFacetRanges(); - if (rangeFacet == null) { - return nameValueList; - } - RangeFacet range = rangeFacet.get(0); - - if (range == null) { - return nameValueList; - } - - listCount = range.getCounts(); - - List<NameValueData> nameValues = new ArrayList<>(); - int count = 0; - for (RangeFacet.Count cnt : listCount) { - NameValueData nameValue = new NameValueData(); - nameValue.setName("" + count); - nameValue.setValue("" + cnt.getCount()); - nameValues.add(nameValue); - count++; - } - nameValueList.setvNameValues(nameValues); - return nameValueList; - - } catch (SolrException | SolrServerException | ParseException - | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public BarGraphDataListResponse topTenUsers(FieldAuditBarGraphSearchCriteria searchCriteria) { - - String jsonUserQuery = - "{Users:{type:terms, field:reqUser, facet:{ Repo:{ type:terms, field:repo, facet:{eventCount:\"sum(event_count)\"}}}}}"; - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - try { - BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return barGraphDataListResponse; - } - - NamedList<Object> namedList = queryResponse.getResponse(); - - if (namedList == null) { - return barGraphDataListResponse; - } - - @SuppressWarnings("unchecked") - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList.get("facets"); - if (jsonFacetResponse == null) { - return barGraphDataListResponse; - } - if (jsonFacetResponse.toString().equals("{count=0}")) { - return barGraphDataListResponse; - } - barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); - return barGraphDataListResponse; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - - public BarGraphDataListResponse topTenResources(FieldAuditLogSearchCriteria searchCriteria) { - - String jsonUserQuery = - "{Users:{type:terms,field:resource,facet:{Repo:{type:terms,field:repo,facet:{eventCount:\"sum(event_count)\"}}}}}"; - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - try { - BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); - QueryResponse queryResponse = auditSolrDao.process(solrQuery); - if (queryResponse == null) { - return barGraphDataListResponse; - } - - NamedList<Object> namedList = queryResponse.getResponse(); - if (namedList == null) { - return barGraphDataListResponse; - } - - @SuppressWarnings("unchecked") - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList.get("facets"); - - barGraphDataListResponse = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); - return barGraphDataListResponse; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } + public BarGraphDataListResponse getAuditBarGraphData(AuditBarGraphRequest request) { + SolrQuery solrQuery = conversionService.convert(request, SolrQuery.class); + QueryResponse response = auditSolrDao.process(solrQuery); + return graphDataGenerator.generateBarGraphDataResponseWithRanges(response, SolrConstants.AuditLogConstants.AUDIT_COMPONENT, true); } - @SuppressWarnings("unchecked") - public BarGraphDataListResponse getRequestUserLineGraph(FieldAuditBarGraphSearchCriteria searchCriteria) { - - String from = getFrom(searchCriteria.getStartTime()); - String to = getTo(searchCriteria.getEndTime()); - String unit = getUnit(searchCriteria.getUnit()); - - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - List<BarGraphData> histogramData = new ArrayList<BarGraphData>(); - - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - - String jsonHistogramQuery = queryGenerator.buildJSONFacetTermTimeRangeQuery(LogSearchConstants.AUDIT_REQUEST_USER, - LogSearchConstants.AUDIT_EVTTIME, from, to, unit).replace("\\", ""); - - return getGraphResponse(dataList, solrQuery, histogramData, jsonHistogramQuery); + public BarGraphDataListResponse topResources(FieldAuditLogRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + QueryResponse queryResponse = auditSolrDao.process(facetQuery); + return graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0); } public String getAuditLogsSchemaFieldsName() { - return convertObjToString(auditSolrDao.schemaFieldNameMap); + return convertObjToString(auditSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap()); } - public BarGraphDataListResponse getAnyGraphData(AnyGraphSearchCriteria searchCriteria) { - searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME); - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - BarGraphDataListResponse result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery); - if (result == null) { - result = new BarGraphDataListResponse(); - } - return result; - + public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) { + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); + QueryResponse response = auditSolrDao.process(facetQuery); + return graphDataGenerator.generateBarGraphFromFieldFacet(response, AUDIT_COMPONENT); } - @SuppressWarnings({"unchecked"}) - public Response exportUserTableToTextFile(UserExportSearchCriteria searchCriteria) { - String jsonUserQuery = - "{ Users: { type: terms, field: reqUser, facet: {Repo: { type: terms, field: repo, facet: { eventCount: \"sum(event_count)\"}}}},x:{ type: terms,field: resource, facet: {y: { type: terms, field: repo,facet: { eventCount: \"sum(event_count)\"}}}}}"; - - SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - String startTime = searchCriteria.getStartTime(); - String endTime = searchCriteria.getEndTime(); + public Response export(UserExportRequest request) throws TemplateException { + String startTime = request.getFrom(); + String endTime = request.getTo(); + SimpleFacetQuery facetQuery = conversionService.convert(request, SimpleFacetQuery.class); startTime = startTime == null ? "" : startTime; endTime = endTime == null ? "" : "_" + endTime; - SolrUtil.setJSONFacet(solrQuery, jsonUserQuery); - SolrUtil.setRowCount(solrQuery, 0); - - String dataFormat = searchCriteria.getFormat(); + String dataFormat = request.getFormat(); FileOutputStream fis = null; try { - QueryResponse queryResponse = auditSolrDao.process(solrQuery); + QueryResponse queryResponse = auditSolrDao.process(facetQuery); if (queryResponse == null) { VResponse response = new VResponse(); - response.setMsgDesc("Query was not able to execute " + solrQuery); + response.setMsgDesc("Query was not able to execute " + facetQuery); throw RESTErrorUtil.createRESTException(response); } - - NamedList<Object> namedList = queryResponse.getResponse(); - if (namedList == null) { - VResponse response = new VResponse(); - response.setMsgDesc("Query was not able to execute " + solrQuery); - throw RESTErrorUtil.createRESTException(response); - } - BarGraphDataListResponse vBarUserDataList = new BarGraphDataListResponse(); - BarGraphDataListResponse vBarResourceDataList = new BarGraphDataListResponse(); - - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) namedList.get("facets"); - vBarUserDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "Repo", "Users"); - vBarResourceDataList = BizUtil.buildSummaryForTopCounts(jsonFacetResponse, "y", "x"); + BarGraphDataListResponse vBarUserDataList = graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 0); + BarGraphDataListResponse vBarResourceDataList = graphDataGenerator.generateSecondLevelBarGraphDataResponse(queryResponse, 1); String data = ""; - String summary = ""; if ("text".equals(dataFormat)) { - int users = 0; - int resources = 0; - summary += "\n\n\n\n"; - data += addBlank("Users") + "Components/Access" + "\n"; - data += "--------------------------------------------------------------------------\n"; - Collection<BarGraphData> tableUserData = vBarUserDataList.getGraphData(); - for (BarGraphData graphData : tableUserData) { - String userName = graphData.getName(); - String largeUserName = ""; - - if (userName.length() > 45) { - largeUserName = userName.substring(0, 45); - data += addBlank(largeUserName); - } else - data += addBlank(userName); - - Collection<NameValueData> vnameValueList = graphData.getDataCount(); - data = appendNameValueData(data, vnameValueList); - while (largeUserName.length() > 0) { - data += largeUserName.substring(0, 45) + "\n"; - } - - users += 1; - } - data += "\n\n\n\n\n\n"; - data += addBlank("Resources") + "Components/Access" + "\n"; - data += "--------------------------------------------------------------------------\n"; - Collection<BarGraphData> tableResourceData = vBarResourceDataList.getGraphData(); - for (BarGraphData graphData : tableResourceData) { - String resourceName = graphData.getName(); - String largeResourceName = resourceName; - if (largeResourceName.length() > 45) { - resourceName = largeResourceName.substring(0, 45); - largeResourceName = largeResourceName.substring(45, largeResourceName.length()); - } else { - largeResourceName = ""; - } + StringWriter stringWriter = new StringWriter(); + Template template = freemarkerConfiguration.getTemplate(AUDIT_LOG_TEMPLATE); + Map<String, Object> models = new HashMap<>(); + DownloadUtil.fillUserResourcesModel(models, vBarUserDataList, vBarResourceDataList); + template.process(models, stringWriter); + data = stringWriter.toString(); - //resourceName = resourceName.replaceAll("(.{45})", resourceName.substring(0, 45)+"\n"); - data += addBlank(resourceName); - Collection<NameValueData> vnameValueList = graphData.getDataCount(); - data = appendNameValueData(data, vnameValueList); - String tempLargeResourceName = largeResourceName; - while (largeResourceName.length() > 45) { - largeResourceName = tempLargeResourceName.substring(0, 45); - tempLargeResourceName = tempLargeResourceName.substring(45, tempLargeResourceName.length()); - data += largeResourceName + "\n"; - } - if (largeResourceName.length() < 45 && !largeResourceName.isEmpty()) { - data += largeResourceName + "\n"; - } - resources += 1; - } - String header = "--------------------------------SUMMARY-----------------------------------\n"; - summary = header + "Users = " + users + "\nResources = " + resources + "\n" + summary; - data = summary + data; } else { data = "{" + convertObjToString(vBarUserDataList) + "," + convertObjToString(vBarResourceDataList) + "}"; dataFormat = "json"; } String fileName = "Users_Resource" + startTime + endTime + "."; File file = File.createTempFile(fileName, dataFormat); - fis = new FileOutputStream(file); fis.write(data.getBytes()); return Response @@ -419,8 +172,8 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp .header("Content-Disposition", "attachment;filename=" + fileName + dataFormat) .build(); - } catch (SolrServerException | SolrException | IOException e) { - logger.error("Error during solrQuery=" + e); + } catch (IOException e) { + logger.error("Error during download file (audit log) " + e); throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } finally { if (fis != null) { @@ -432,97 +185,6 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp } } - private BarGraphDataListResponse getGraphResponse(BarGraphDataListResponse dataList, SolrQuery solrQuery, - List<BarGraphData> histogramData, String jsonHistogramQuery) { - try { - SolrUtil.setJSONFacet(solrQuery, jsonHistogramQuery); - SolrUtil.setRowCount(solrQuery, 0); - QueryResponse response = auditSolrDao.process(solrQuery); - if (response == null) { - return dataList; - } - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets"); - - if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { - return dataList; - } - - auditSolrDao.extractValuesFromBuckets(jsonFacetResponse, "x", "y", histogramData); - - dataList.setGraphData(histogramData); - return dataList; - - } catch (SolrServerException | SolrException | IOException e) { - logger.error(e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - - } - } - - private String appendNameValueData(String data, Collection<NameValueData> vnameValueList) { - int count = 0; - String blank = ""; - for (NameValueData vNameValue : vnameValueList) { - data += blank + vNameValue.getName() + " " + vNameValue.getValue() + "\n"; - if (count == 0) - blank = addBlank(blank); - count++; - } - return data; - } - - private String addBlank(String field) { - int blanks = 50; - int strSize = field.length(); - String fieldWithBlank = field; - for (int i = 0; i < blanks - strSize; i++) { - fieldWithBlank += " "; - } - return fieldWithBlank; - } - - public BarGraphDataListResponse getServiceLoad(CommonSearchCriteria searchCriteria) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - Collection<BarGraphData> vaDatas = new ArrayList<BarGraphData>(); - dataList.setGraphData(vaDatas); - - SolrQuery serivceLoadQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); - - try { - SolrUtil.setFacetField(serivceLoadQuery, LogSearchConstants.AUDIT_COMPONENT); - QueryResponse serviceLoadResponse = auditSolrDao.process(serivceLoadQuery); - if (serviceLoadResponse == null) { - return dataList; - } - FacetField serviceFacetField = serviceLoadResponse.getFacetField(LogSearchConstants.AUDIT_COMPONENT); - if (serviceFacetField == null) { - return dataList; - } - - List<Count> serviceLoadFacets = serviceFacetField.getValues(); - if (serviceLoadFacets == null) { - return dataList; - } - for (Count cnt : serviceLoadFacets) { - List<NameValueData> valueList = new ArrayList<NameValueData>(); - BarGraphData vBarGraphData = new BarGraphData(); - vaDatas.add(vBarGraphData); - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(cnt.getName()); - vBarGraphData.setName(cnt.getName().toUpperCase()); - vNameValue.setValue("" + cnt.getCount()); - valueList.add(vNameValue); - vBarGraphData.setDataCount(valueList); - } - - return dataList; - - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); - } - } - @Override protected List<SolrAuditLogData> convertToSolrBeans(QueryResponse response) { return response.getBeans(SolrAuditLogData.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java index 13df470..a7b9cb5 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ManagerBase.java @@ -28,17 +28,13 @@ import java.util.Date; import java.util.List; import java.util.Scanner; -import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.model.response.LogData; import org.apache.ambari.logsearch.model.response.LogSearchResponse; -import org.apache.ambari.logsearch.query.model.SearchCriteria; import org.apache.ambari.logsearch.dao.SolrDaoBase; -import org.apache.ambari.logsearch.query.QueryGeneration; import org.apache.ambari.logsearch.util.DateUtil; import org.apache.ambari.logsearch.util.JSONUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; -import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; @@ -46,30 +42,13 @@ import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; - -import javax.inject.Inject; +import org.springframework.data.solr.core.DefaultQueryParser; +import org.springframework.data.solr.core.query.SimpleQuery; +import org.springframework.data.solr.core.query.SolrDataQuery; public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE extends LogSearchResponse> extends JsonManagerBase { private static final Logger logger = Logger.getLogger(ManagerBase.class); - @Inject - protected QueryGeneration queryGenerator; - - public enum LogType { - SERVICE("Service"), - AUDIT("Audit"); - - private String label; - - private LogType(String label) { - this.label = label; - } - - public String getLabel() { - return label; - } - } - public ManagerBase() { super(); } @@ -103,77 +82,52 @@ public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE } - protected SEARCH_RESPONSE getLastPage(SearchCriteria searchCriteria, String logTimeField, SolrDaoBase solrDoaBase, - SolrQuery lastPageQuery) { - - Integer maxRows = searchCriteria.getMaxRows(); - String givenSortType = searchCriteria.getSortType(); - searchCriteria = new SearchCriteria(); - searchCriteria.setSortBy(logTimeField); - if (givenSortType == null || givenSortType.equals(LogSearchConstants.DESCENDING_ORDER)) { - lastPageQuery.removeSort(LogSearchConstants.LOGTIME); - searchCriteria.setSortType(LogSearchConstants.ASCENDING_ORDER); - } else { - searchCriteria.setSortType(LogSearchConstants.DESCENDING_ORDER); - } - queryGenerator.setSingleSortOrder(lastPageQuery, searchCriteria); - - - Long totalLogs = 0l; - int startIndex = 0; - int numberOfLogsOnLastPage = 0; - SEARCH_RESPONSE logResponse = null; - try { - SolrUtil.setStart(lastPageQuery, 0); - SolrUtil.setRowCount(lastPageQuery, maxRows); - logResponse = getLogAsPaginationProvided(lastPageQuery, solrDoaBase); - totalLogs = countQuery(lastPageQuery,solrDoaBase); - startIndex = Integer.parseInt("" + ((totalLogs / maxRows) * maxRows)); - numberOfLogsOnLastPage = Integer.parseInt("" + (totalLogs - startIndex)); - logResponse.setStartIndex(startIndex); - logResponse.setTotalCount(totalLogs); - logResponse.setPageSize(maxRows); - List<LOG_DATA_TYPE> docList = logResponse.getLogList(); - List<LOG_DATA_TYPE> lastPageDocList = new ArrayList<>(); - logResponse.setLogList(lastPageDocList); - int cnt = 0; - for(LOG_DATA_TYPE doc:docList){ - if(cnt<numberOfLogsOnLastPage){ - lastPageDocList.add(doc); - } - cnt++; + protected SEARCH_RESPONSE getLastPage(SolrDaoBase solrDoaBase, SimpleQuery lastPageQuery, String event) { + int maxRows = lastPageQuery.getRows(); + SEARCH_RESPONSE logResponse = getLogAsPaginationProvided(lastPageQuery, solrDoaBase, event); + Long totalLogs = solrDoaBase.count(lastPageQuery); + int startIndex = Integer.parseInt("" + ((totalLogs / maxRows) * maxRows)); + int numberOfLogsOnLastPage = Integer.parseInt("" + (totalLogs - startIndex)); + logResponse.setStartIndex(startIndex); + logResponse.setTotalCount(totalLogs); + logResponse.setPageSize(maxRows); + List<LOG_DATA_TYPE> docList = logResponse.getLogList(); + List<LOG_DATA_TYPE> lastPageDocList = new ArrayList<>(); + logResponse.setLogList(lastPageDocList); + int cnt = 0; + for (LOG_DATA_TYPE doc : docList) { + if (cnt < numberOfLogsOnLastPage) { + lastPageDocList.add(doc); } - Collections.reverse(lastPageDocList); - - } catch (SolrException | SolrServerException | IOException | NumberFormatException e) { - logger.error("Count Query was not executed successfully",e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + cnt++; } + Collections.reverse(lastPageDocList); return logResponse; } - protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase) { - try { - QueryResponse response = solrDaoBase.process(solrQuery); - SEARCH_RESPONSE logResponse = createLogSearchResponse(); - SolrDocumentList docList = response.getResults(); - List<LOG_DATA_TYPE> serviceLogDataList = convertToSolrBeans(response); - if (docList != null && !docList.isEmpty()) { - logResponse.setLogList(serviceLogDataList); - logResponse.setStartIndex((int) docList.getStart()); - logResponse.setTotalCount(docList.getNumFound()); - Integer rowNumber = solrQuery.getRows(); - if (rowNumber == null) { - logger.error("No RowNumber was set in solrQuery"); - return createLogSearchResponse(); - } - logResponse.setPageSize(rowNumber); + protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrDataQuery solrQuery, SolrDaoBase solrDaoBase, String event) { + SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(solrQuery); + return getLogAsPaginationProvided(query, solrDaoBase, event); + } + + + protected SEARCH_RESPONSE getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase, String event) { + QueryResponse response = solrDaoBase.process(solrQuery, event); + SEARCH_RESPONSE logResponse = createLogSearchResponse(); + SolrDocumentList docList = response.getResults(); + List<LOG_DATA_TYPE> serviceLogDataList = convertToSolrBeans(response); + if (docList != null && !docList.isEmpty()) { + logResponse.setLogList(serviceLogDataList); + logResponse.setStartIndex((int) docList.getStart()); + logResponse.setTotalCount(docList.getNumFound()); + Integer rowNumber = solrQuery.getRows(); + if (rowNumber == null) { + logger.error("No RowNumber was set in solrQuery"); + return createLogSearchResponse(); } - return logResponse; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Error during solrQuery=" + solrQuery, e); - throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + logResponse.setPageSize(rowNumber); } + return logResponse; } protected Long countQuery(SolrQuery query,SolrDaoBase solrDaoBase) throws SolrException, SolrServerException, IOException { @@ -189,13 +143,6 @@ public abstract class ManagerBase<LOG_DATA_TYPE extends LogData, SEARCH_RESPONSE return docList.getNumFound(); } - protected String getUnit(String unit) { - if (StringUtils.isBlank(unit)) { - unit = "+1HOUR"; - } - return unit; - } - protected String getFrom(String from) { if (StringUtils.isBlank(from)) { Date date = DateUtil.getTodayFromDate(); http://git-wip-us.apache.org/repos/asf/ambari/blob/44644cd6/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java index 82817d6..cbab651 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/PublicManager.java @@ -25,11 +25,11 @@ import java.util.List; import org.apache.ambari.logsearch.conf.AuthPropsConfig; import org.apache.ambari.logsearch.model.response.NameValueData; import org.apache.ambari.logsearch.model.response.NameValueDataListResponse; -import org.springframework.stereotype.Component; import javax.inject.Inject; +import javax.inject.Named; -@Component +@Named public class PublicManager extends JsonManagerBase { @Inject
