Repository: ambari Updated Branches: refs/heads/trunk 7c404df3f -> 962976585
AMBARI-17140. Last button in the log search pagination panel does not take the user to the last page (Dharmesh Makwana via oleewere) Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/96297658 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/96297658 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/96297658 Branch: refs/heads/trunk Commit: 96297658593f6dd283239198cfa0ef4c2f0f6dc5 Parents: 7c404df Author: oleewere <[email protected]> Authored: Tue Jun 14 14:10:11 2016 +0200 Committer: oleewere <[email protected]> Committed: Tue Jun 14 14:12:09 2016 +0200 ---------------------------------------------------------------------- .../logsearch/common/LogSearchConstants.java | 2 + .../ambari/logsearch/dao/SolrDaoBase.java | 5 +- .../ambari/logsearch/manager/AuditMgr.java | 16 ++++- .../ambari/logsearch/manager/LogsMgr.java | 76 ++++++++++++++------ .../ambari/logsearch/manager/MgrBase.java | 70 ++++++++++++++++++ .../ambari/logsearch/manager/UserConfigMgr.java | 13 ++-- .../ambari/logsearch/query/QueryGeneration.java | 4 +- .../apache/ambari/logsearch/rest/AuditREST.java | 1 + .../ambari/logsearch/rest/DashboardREST.java | 1 + .../ambari/logsearch/util/ConfigUtil.java | 32 ++------- .../ambari/logsearch/view/VUserConfig.java | 11 ++- .../src/main/resources/default.properties | 5 +- .../main/webapp/META-INF/applicationContext.xml | 1 + .../scripts/collections/BaseCollection.js | 12 +++- .../src/main/webapp/scripts/utils/Intro.js | 32 ++++----- .../scripts/views/audit/AuditTabLayoutView.js | 3 +- .../views/dialog/ApplySearchFilterView.js | 2 +- .../views/tabs/EventHistoryLayoutView.js | 2 +- 18 files changed, 199 insertions(+), 89 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java index 917956f..593be65 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java @@ -88,6 +88,8 @@ public class LogSearchConstants { public static final String UI_SUFFIX = "@UI@"; public static final String SOLR_SUFFIX = "@Solr@"; public static final String NGRAM_SUFFIX = "ngram_"; + public static final String DEFAULT_SERVICE_COLUMN_SUFFIX = "service"; + public static final String DEFAULT_AUDIT_COLUMN_SUFFIX = "audit"; //Date Format for SOLR public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS"; http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java index 147e148..cd4ef93 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -31,6 +31,7 @@ import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE; import org.apache.ambari.logsearch.util.ConfigUtil; import org.apache.ambari.logsearch.util.JSONUtil; +import org.apache.ambari.logsearch.util.PropertiesUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; import org.apache.ambari.logsearch.util.StringUtil; import org.apache.log4j.Logger; @@ -534,13 +535,15 @@ public abstract class SolrDaoBase { SolrRequest<SchemaResponse> request = new SchemaRequest(); request.setMethod(METHOD.GET); request.setPath("/schema/fields"); - if (solrClient != null) { + String historyCollection = PropertiesUtil.getProperty("logsearch.solr.collection.history","history"); + if (solrClient != null && !collectionName.equals(historyCollection)) { NamedList<Object> namedList = null; try { namedList = solrClient.request(request); logger.info("populateSchemaFields() collection=" + collectionName + ", fields=" + namedList); } catch (SolrException | SolrServerException | IOException e) { + logger.error( "Error occured while popuplating field. collection=" + collectionName, e); http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java index 53e386e..3dd8146 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java @@ -69,7 +69,7 @@ import org.springframework.stereotype.Component; @Component public class AuditMgr extends MgrBase { - static Logger logger = Logger.getLogger(AuditMgr.class); + static Logger logger = Logger.getLogger(AuditMgr.class); @Autowired AuditSolrDao auditSolrDao; @@ -93,6 +93,16 @@ public class AuditMgr extends MgrBase { GraphDataGenerator graphDataGenerator; public String getLogs(SearchCriteria searchCriteria) { + String lastPage = (String) searchCriteria.getParamValue("isLastPage"); + Boolean isLastPage = Boolean.parseBoolean(lastPage); + if (isLastPage) { + SolrQuery lastPageQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); + VSolrLogList collection = getLastPage(searchCriteria,LogSearchConstants.AUDIT_EVTTIME,auditSolrDao,lastPageQuery); + if(collection == null){ + collection = new VSolrLogList(); + } + return convertObjToString(collection); + } SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); VSolrLogList collection = getLogAsPaginationProvided(solrQuery, auditSolrDao); @@ -458,7 +468,7 @@ public class AuditMgr extends MgrBase { } public String getAuditLogsSchemaFieldsName() { - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs"); + String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX); String excludeArray[] = PropertiesUtil .getPropertyStringList("logsearch.solr.audit.logs.exclude.columnlist"); List<String> fieldNames = new ArrayList<String>(); @@ -482,7 +492,7 @@ public class AuditMgr extends MgrBase { public String getAnyGraphData(SearchCriteria searchCriteria) { searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME); - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs"); + String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX); searchCriteria.addParam("suffix", suffix); SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria, http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java index ca63671..b03a643 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java @@ -21,8 +21,10 @@ package org.apache.ambari.logsearch.manager; import java.io.IOException; import java.text.ParseException; import java.util.ArrayList; +import java.util.Calendar; import java.util.Collection; import java.util.Date; +import java.util.GregorianCalendar; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; @@ -30,6 +32,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; +import java.util.TimeZone; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; @@ -41,6 +44,7 @@ import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.common.SearchCriteria; import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; import org.apache.ambari.logsearch.graph.GraphDataGenerator; +import org.apache.ambari.logsearch.query.QueryGenerationBase; import org.apache.ambari.logsearch.util.BizUtil; import org.apache.ambari.logsearch.util.ConfigUtil; import org.apache.ambari.logsearch.util.FileUtil; @@ -72,6 +76,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import com.google.common.collect.Lists; @@ -105,6 +110,9 @@ public class LogsMgr extends MgrBase { public String searchLogs(SearchCriteria searchCriteria) { String keyword = (String) searchCriteria.getParamValue("keyword"); String logId = (String) searchCriteria.getParamValue("sourceLogId"); + String lastPage = (String) searchCriteria.getParamValue("isLastPage"); + Boolean isLastPage = Boolean.parseBoolean(lastPage); + if (!stringUtil.isEmpty(keyword)) { try { return getPageByKeyword(searchCriteria); @@ -121,8 +129,16 @@ public class LogsMgr extends MgrBase { throw restErrorUtil.createRESTException(MessageEnums.SOLR_ERROR .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } + } else if (isLastPage) { + SolrQuery lastPageQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); + VSolrLogList collection = getLastPage(searchCriteria,LogSearchConstants.LOGTIME,serviceLogsSolrDao,lastPageQuery); + if(collection == null){ + collection = new VSolrLogList(); + } + return convertObjToString(collection); } else { - SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); + SolrQuery solrQuery = queryGenerator + .commonServiceFilterQuery(searchCriteria); solrQuery.setParam("event", "/solr/logs_search"); @@ -820,7 +836,7 @@ public class LogsMgr extends MgrBase { } - long countNumberLogs = countQuery(rangeLogQuery) - 1; + long countNumberLogs = countQuery(rangeLogQuery,serviceLogsSolrDao) - 1; //Adding numbers on @@ -1037,7 +1053,7 @@ public class LogsMgr extends MgrBase { } - long countNumberLogs = countQuery(rangeLogQuery) - 1; + long countNumberLogs = countQuery(rangeLogQuery,serviceLogsSolrDao) - 1; //Adding numbers on try { @@ -1136,7 +1152,7 @@ public class LogsMgr extends MgrBase { queryGenerator.setSingleRangeFilter(solrQuery, LogSearchConstants.LOGTIME, endTimeMinusOneMilli, endLogTime); queryGenerator.setRowCount(solrQuery, 0); - startIndex = countQuery(solrQuery); + startIndex = countQuery(solrQuery,serviceLogsSolrDao); } catch (SolrException | SolrServerException | IOException e) { logger.error(e); } @@ -1645,20 +1661,6 @@ public class LogsMgr extends MgrBase { return collection; } - public Long countQuery(SolrQuery query) throws SolrException, - SolrServerException, IOException { - query.setRows(0); - QueryResponse response = serviceLogsSolrDao.process(query); - if(response == null){ - return 0l; - } - SolrDocumentList docList = response.getResults(); - if(docList == null){ - return 0l; - } - return docList.getNumFound(); - } - public String getServiceLogsFieldsName() { String fieldsNameStrArry[] = PropertiesUtil .getPropertyStringList("logsearch.service.logs.fields"); @@ -1687,7 +1689,7 @@ public class LogsMgr extends MgrBase { public String getServiceLogsSchemaFieldsName() { List<String> fieldNames = new ArrayList<String>(); - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs"); + String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX); String excludeArray[] = PropertiesUtil .getPropertyStringList("logsearch.solr.service.logs.exclude.columnlist"); @@ -1705,7 +1707,7 @@ public class LogsMgr extends MgrBase { } HashMap<String, String> uiFieldColumnMappingSorted = new LinkedHashMap<String, String>(); - uiFieldColumnMappingSorted.put(LogSearchConstants.SOLR_LOG_MESSAGE, ""); + uiFieldColumnMappingSorted.put(LogSearchConstants.SOLR_LOG_MESSAGE, LogSearchConstants.SOLR_LOG_MESSAGE); Iterator<Entry<String, String>> it = bizUtil .sortHashMapByValues(uiFieldColumnMapping).entrySet().iterator(); @@ -1756,7 +1758,7 @@ public class LogsMgr extends MgrBase { public String getAnyGraphData(SearchCriteria searchCriteria) { searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME); - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs"); + String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX); searchCriteria.addParam("suffix", suffix); SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria, @@ -1930,4 +1932,36 @@ public class LogsMgr extends MgrBase { return getLogAsPaginationProvided(solrQuery, serviceLogsSolrDao); } + + @Scheduled(cron = "${logsearch.solr.warming.cron}") + public void warmingSolrServer(){ + logger.info("solr warming triggered."); + SolrQuery solrQuery = new SolrQuery(); + TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT"); + GregorianCalendar utc = new GregorianCalendar(gmtTimeZone); + utc.setTimeInMillis(new Date().getTime()); + utc.set(GregorianCalendar.HOUR, 0); + utc.set(GregorianCalendar.MINUTE, 0); + utc.set(GregorianCalendar.MILLISECOND, 001); + utc.set(GregorianCalendar.SECOND, 0); + dateUtil.convertDateWithMillisecondsToSolrDate(utc.getTime()); + String from = dateUtil.convertDateWithMillisecondsToSolrDate(utc.getTime()); + utc.set(Calendar.MILLISECOND, 999); + utc.set(Calendar.SECOND, 59); + utc.set(Calendar.MINUTE, 59); + utc.set(Calendar.HOUR, 23); + String to = dateUtil.convertDateWithMillisecondsToSolrDate(utc.getTime()); + queryGenerator.setSingleRangeFilter(solrQuery, + LogSearchConstants.LOGTIME, from,to); + String level = LogSearchConstants.FATAL+","+LogSearchConstants.ERROR+","+LogSearchConstants.WARN; + queryGenerator.setFilterClauseWithFieldName(solrQuery, level, + LogSearchConstants.SOLR_LEVEL, "", QueryGenerationBase.CONDITION.OR); + try { + serviceLogsSolrDao.process(solrQuery); + } catch (SolrServerException | IOException e) { + logger.error("Error while warming solr server",e); + } + } + + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/MgrBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/MgrBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/MgrBase.java index 1d069d3..99eb9f2 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/MgrBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/MgrBase.java @@ -22,10 +22,13 @@ package org.apache.ambari.logsearch.manager; import java.io.File; import java.io.IOException; import java.text.ParseException; +import java.util.Collections; import java.util.Date; import java.util.Scanner; +import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.common.SearchCriteria; import org.apache.ambari.logsearch.dao.SolrDaoBase; import org.apache.ambari.logsearch.query.QueryGeneration; import org.apache.ambari.logsearch.util.DateUtil; @@ -38,6 +41,7 @@ import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.SolrException; import org.springframework.beans.factory.annotation.Autowired; @@ -158,6 +162,58 @@ public class MgrBase { MessageEnums.ERROR_SYSTEM); } + + protected VSolrLogList getLastPage(SearchCriteria searchCriteria,String logTimeField,SolrDaoBase solrDoaBase,SolrQuery lastPageQuery){ + + Integer maxRows = searchCriteria.getMaxRows(); + String givenSortType = searchCriteria.getSortType(); + searchCriteria = new SearchCriteria(); + searchCriteria.setSortBy(logTimeField); + if(givenSortType == null || givenSortType.equals(LogSearchConstants.DESCENDING_ORDER)){ + lastPageQuery.removeSort(LogSearchConstants.LOGTIME); + searchCriteria.setSortType(LogSearchConstants.ASCENDING_ORDER); + } else { + searchCriteria.setSortType(LogSearchConstants.DESCENDING_ORDER); + } + queryGenerator.setSingleSortOrder(lastPageQuery, searchCriteria); + + + Long totalLogs = 0l; + int startIndex = 0; + int numberOfLogsOnLastPage = 0; + VSolrLogList collection = null; + try { + queryGenerator.setStart(lastPageQuery, 0); + queryGenerator.setRowCount(lastPageQuery, maxRows); + collection = getLogAsPaginationProvided(lastPageQuery, + solrDoaBase); + totalLogs = countQuery(lastPageQuery,solrDoaBase); + if(maxRows != null){ + startIndex = Integer.parseInt("" + ((totalLogs/maxRows) * maxRows)); + numberOfLogsOnLastPage = Integer.parseInt("" + (totalLogs-startIndex)); + } + collection.setStartIndex(startIndex); + collection.setTotalCount(totalLogs); + collection.setPageSize(maxRows); + SolrDocumentList docList = collection.getList(); + SolrDocumentList lastPageDocList = new SolrDocumentList(); + collection.setSolrDocuments(lastPageDocList); + int cnt = 0; + for(SolrDocument doc:docList){ + if(cnt<numberOfLogsOnLastPage){ + lastPageDocList.add(doc); + } + cnt++; + } + Collections.reverse(lastPageDocList); + + } catch (SolrException | SolrServerException | IOException | NumberFormatException e) { + logger.error("Count Query was not executed successfully",e); + throw restErrorUtil.createRESTException(MessageEnums.SOLR_ERROR + .getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); + } + return collection; + } public VSolrLogList getLogAsPaginationProvided(SolrQuery solrQuery, SolrDaoBase solrDaoBase) { @@ -184,6 +240,20 @@ public class MgrBase { } } + + public Long countQuery(SolrQuery query,SolrDaoBase solrDaoBase) throws SolrException, + SolrServerException, IOException { + query.setRows(0); + QueryResponse response = solrDaoBase.process(query); + if (response == null) { + return 0l; + } + SolrDocumentList docList = response.getResults(); + if (docList == null) { + return 0l; + } + return docList.getNumFound(); + } protected String getUnit(String unit) { if (stringUtil.isEmpty(unit)) { http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/UserConfigMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/UserConfigMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/UserConfigMgr.java index 3b23f2a..f033689 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/UserConfigMgr.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/UserConfigMgr.java @@ -77,11 +77,11 @@ public class UserConfigMgr extends MgrBase { if (isNotUnique(vHistory) && !vHistory.isOverwrite()) { throw restErrorUtil.createRESTException( - "Name '" + vHistory.getFilterName() + "' already exists", + "Name '" + vHistory.getFiltername() + "' already exists", MessageEnums.INVALID_INPUT_DATA); } String loggedInUserName = vHistory.getUserName(); - String filterName = vHistory.getFilterName(); + String filterName = vHistory.getFiltername(); solrInputDoc.addField(LogSearchConstants.ID, vHistory.getId()); solrInputDoc.addField(LogSearchConstants.USER_NAME, loggedInUserName); @@ -127,7 +127,7 @@ public class UserConfigMgr extends MgrBase { } private boolean isNotUnique(VUserConfig vHistory) { - String filterName = vHistory.getFilterName(); + String filterName = vHistory.getFiltername(); String rowType = vHistory.getRowType(); if (filterName != null && rowType != null) { @@ -150,7 +150,7 @@ public class UserConfigMgr extends MgrBase { } private boolean isValid(VUserConfig vHistory) { - return !stringUtil.isEmpty(vHistory.getFilterName()) + return !stringUtil.isEmpty(vHistory.getFiltername()) && !stringUtil.isEmpty(vHistory.getRowType()) && !stringUtil.isEmpty(vHistory.getUserName()) && !stringUtil.isEmpty(vHistory.getValues()); @@ -201,8 +201,7 @@ public class UserConfigMgr extends MgrBase { queryGenerator.setSingleIncludeFilter(userConfigQuery, LogSearchConstants.FILTER_NAME, filterName); - if (stringUtil.isEmpty(searchCriteria.getSortBy()) - || searchCriteria.getSortBy().equals("historyName")) { + if (stringUtil.isEmpty(searchCriteria.getSortBy())) { searchCriteria.setSortBy(LogSearchConstants.FILTER_NAME); } if (stringUtil.isEmpty(searchCriteria.getSortType())) { @@ -216,7 +215,7 @@ public class UserConfigMgr extends MgrBase { for (SolrDocument solrDoc : solrList) { VUserConfig userConfig = new VUserConfig(); - userConfig.setFilterName("" + userConfig.setFiltername("" + solrDoc.get(LogSearchConstants.FILTER_NAME)); userConfig.setId("" + solrDoc.get(LogSearchConstants.ID)); userConfig.setValues("" + solrDoc.get(LogSearchConstants.VALUES)); http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java index 4647c7b..93105b0 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java @@ -341,11 +341,11 @@ public class QueryGeneration extends QueryGenerationBase { switch (logType) { case AUDIT: String auditSuffix = PropertiesUtil - .getProperty("logsearch.solr.collection.audit.logs"); + .getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX); type = ConfigUtil.schemaFieldsName.get(key + auditSuffix); break; case SERVICE: - String serviceLogs = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs"); + String serviceLogs = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX); type = ConfigUtil.schemaFieldsName.get(key + serviceLogs); break; default: http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java index 92bfb01..6d18b4a 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/AuditREST.java @@ -53,6 +53,7 @@ public class AuditREST { public String getAuditLogs(@Context HttpServletRequest request) { SearchCriteria searchCriteria = new SearchCriteria(request); searchCriteria.addRequiredAuditLogsParams(request); + searchCriteria.addParam("isLastPage", request.getParameter("isLastPage")); return auditMgr.getLogs(searchCriteria); } http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java index 1e107ed..cb87f6f 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/DashboardREST.java @@ -63,6 +63,7 @@ public class DashboardREST { request.getParameter("keywordType")); searchCriteria.addParam("token", request.getParameter("token")); + searchCriteria.addParam("isLastPage",request.getParameter("isLastPage")); return logMgr.searchLogs(searchCriteria); } http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java index 44d184d..2bf3b51 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java @@ -58,39 +58,16 @@ public class ConfigUtil { if (mapping.length > 1) { String solrField = mapping[0]; String uiField = mapping[1]; - - String modifiedUIField = getModifiedUIField(uiField); + columnMappingMap.put(solrField + LogSearchConstants.SOLR_SUFFIX, - modifiedUIField); - columnMappingMap.put(modifiedUIField + LogSearchConstants.UI_SUFFIX, + uiField); + columnMappingMap.put(uiField + LogSearchConstants.UI_SUFFIX, solrField); } } } } - private static String getModifiedUIField(String uiField) { - String modifiedUIField = ""; - String temp = serviceLogsColumnMapping.get(uiField - + LogSearchConstants.UI_SUFFIX); - if (temp == null){ - return uiField; - }else { - String lastChar = uiField.substring(uiField.length() - 1, - uiField.length()); - int k = 1; - try { - k = Integer.parseInt(lastChar); - k = k + 1; - modifiedUIField = uiField.substring(0, uiField.length() - 2); - } catch (Exception e) { - - } - modifiedUIField = uiField + "_" + k; - } - return getModifiedUIField(modifiedUIField); - } - private static void intializeLogLevels() { logLevels.add(LogSearchConstants.TRACE); logLevels.add(LogSearchConstants.DEBUG); @@ -132,7 +109,8 @@ public class ConfigUtil { if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") - && !name.contains(LogSearchConstants.NGRAM_SUFFIX)) { + && !name.contains(LogSearchConstants.NGRAM_SUFFIX) + && !name.contains("tags") && !name.contains("_str")) { schemaFieldsName.put(name + suffix, type); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/view/VUserConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/view/VUserConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/view/VUserConfig.java index 55ec1c0..6487f91 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/view/VUserConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/view/VUserConfig.java @@ -32,7 +32,7 @@ public class VUserConfig { protected String id; protected String userName; - protected String filterName; + protected String filtername; protected String values; List<String> shareNameList; @@ -61,15 +61,14 @@ public class VUserConfig { this.userName = userName; } - public String getFilterName() { - return filterName; + public String getFiltername() { + return filtername; } - public void setFilterName(String filterName) { - this.filterName = filterName; + public void setFiltername(String filtername) { + this.filtername = filtername; } - public String getValues() { return values; } http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties index 25b4f1a..d89a6c9 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/resources/default.properties @@ -20,7 +20,7 @@ logsearch.service.logs.fields=logtime,level,event_count,ip,type,path,file,line_n logsearch.solr.service.logs.exclude.columnlist=tags,text,message,seq_num #Exclude Column List for Ranger Audits -logsearch.solr.audit.logs.exclude.columnlist=tags,tags_str +logsearch.solr.audit.logs.exclude.columnlist=tags,tags_str,seq_num #Value Mapping for Audit Fields #Example FieldName=ValueOfUI:ValueOfSolr @@ -32,7 +32,7 @@ result=Allowed:1,Denied:0 logsearch.solr.service.logs.column.mapping=bundle_id:Bundle Id,thread_name:Thread,log_message:Message,case_id:Case Id,cluster:Cluster,event_count:Event Count,file:File,host:Host,id:Id,ip:Ip,level:Level,line_number:Line Number,logfile_line_number:Logfile Line Number,logger_name:Logger Name,logtype:Log Type,method:Method,path:Path,rowtype:Row Type,type:Type #For Audit Logs -logsearch.solr.audit.logs.column.mapping=access:Access Type,reqUser:User,enforcer:Access Enfocer,ip:Client IP +logsearch.solr.audit.logs.column.mapping=access:Access Type,reqUser:User,enforcer:Access Enforcer,ip:IP,action:Action,agent:Agent,agentHost:Agent Host,authType:Auth Type,bundle_id:Bundle Id,case_id:Case Id,cliIP:Client Ip,cliType:Client Type,cluster:Cluster,dst:DST,event_count:Event Count,file:File,host:Host,id:Id,level:Level,log_message:Log Message,logfile_line_number:Logfile Line Number,logger_name:Logger Name,logType:Log Type,path:Path,perm:Perm,policy:Policy,proxyAuthType:Proxy Auth Type,proxyUsers:Proxy Users,reason:Reason,repo:Repo,repoType:Repo Type,req_caller_id:Req Caller Id,req_self_id:Req Self Id,reqContext:Req Context,reqData:Req Data,resource:Resource,resType:Res Type,result:Result,seq_num:Seq Num,sess:Sess,text:Text,type:Type,ugi:UGI #login method #Note: Simple will be supported only if both file and ldap is disabled. @@ -45,3 +45,4 @@ logsearch.logfeeder.include.default.level=FATAL,ERROR,WARN,INFO,DEBUG,TRACE #login config logsearch.login.credentials.file=user_pass.json logsearch.login.ldap.config=logsearch-admin-site.xml +logsearch.solr.warming.cron=0 0/10 * * * ? http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml index 6a73d60..5e24d88 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/META-INF/applicationContext.xml @@ -36,6 +36,7 @@ http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd";> <context:component-scan base-package="org.apache.ambari.logsearch" /> + <task:annotation-driven /> <bean id="xmlPropertyConfigurer" class="org.apache.ambari.logsearch.util.XMLPropertiesUtil" /> <bean id="propertyConfigurer" class="org.apache.ambari.logsearch.util.PropertiesUtil"> http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js index 24b6974..ac82708 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/collections/BaseCollection.js @@ -126,7 +126,17 @@ define(['require', getLastPage: function (options) { return this.getPage("last", _.extend({reset:true}, options)); - } + }, + getPage : function(index, options){ + if(index === "last"){ + this.queryParams.isLastPage = true; + }else{ + delete this.queryParams.isLastPage; + } + var fn = Backbone.PageableCollection.prototype.getPage; + fn.apply(this,arguments); + + }, ///////////////////////////// // End overriding methods // ///////////////////////////// http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Intro.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Intro.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Intro.js index 869a1d4..74a232d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Intro.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/utils/Intro.js @@ -30,23 +30,23 @@ define(['require', 'intro'], function(require, IntroJs) { steps: [{ element: $('[data-id="troubleshoot"]').get(0), - intro: "Welcome, This is the landing page of the LogSearch app, its displays high-level info about services and its components along with graph.", + intro: "Welcome to Log Search! Choose Troubleshooting to display high-level log and usage aggregates for all services and components in your cluster.", position: "right" }, { element: $('#troubleShootHeader').get(0), - intro: "In this section you can choose specific service to focus the issue along with their dependency.", + intro: "Choose a Service to display high-level information about that service. Optionally, filter by components and apply a specific diagnostic time frame.", position: "bottom" }, { element: $('div[data-id="dateRange"] .selectDateRange').get(0), - intro: "This is date time picker by clicking on it you can select time frame and also there are pre-defined time slots.", + intro: "Click the Date-Time picker and choose a pre-defined time slot or specify date and time values.", position: "top" }, { element: $("#showServicelog").get(0), - intro: " After selecting the service and time frame you can navigate to the logs detail page [Service log tab].", + intro: "Choose Go To Logs to further refine your examination of Service Logs for a selected service.", position: "top" }, { element: $('[data-id="hierarchy"]').get(0), - intro: " This tab holds the logs of all services and its components with different views and aggregation and you can also apply filters for getting into the issues in detail.", + intro: "Service Logs displays a high-level history of logs generated by a selected service and allows you to filter logs by component. You control your view of log data per component and host, Choose to view aggregate Service Logs or drill into details of each log message.", position: "right" }, { element: $("#searchIncludeExclude").get(0), @@ -58,51 +58,51 @@ define(['require', 'intro'], function(require, IntroJs) { position: "top" }, { element: $('#r_EventHistory').get(0), - intro: "This section will track the ongoing filters applied to the query, you can navigate back and forth.", + intro: "Event History displays the current, filtered query results.", position: "top" }, { element: $('#r_Histogram').get(0), - intro: "This section is basically time filter required for particular time related results.", + intro: "Histogram displays comparative ratios of log severity during the currently defined time filter.", position: "top" }, { element: document.querySelectorAll('#r_BubbleTable')[1], - intro: "This is consolidated view for all host and components.", + intro: "The Log Data default view displays consolidated for all hosts.", position: "top", child: 'li[data-parent="true"]' }, { element: document.querySelectorAll('#r_BubbleTable')[1], - intro: "By clicking on host node the tree will expand to shows it underlying components.", + intro: "Expand the Log Data tree view and choose components to further refine your diagnostics.", position: "top", child: 'li[data-parent="true"]' }, { element: $(document.querySelectorAll('#r_BubbleTable')[1]).find('.box-content')[1], - intro: "Blue icon will appear near every components to view its logs on new tab. You can simultaneously view components log file in adjacent tabs.", + intro: "Choose the blue arrow icon to view logs for a specific component on a new tab.", position: "right", child: 'li[data-parent="true"]' }, { element: document.querySelectorAll('#r_BubbleTable')[1], - intro: " This view shows the actual log entries with segregated columns.", + intro: "Choose Service Logs to show a column-separated view of actual log entries.", position: "top", child: 'li[data-parent="true"]' }, { element: $(document.querySelectorAll('#r_BubbleTable')[1]).find('td.logTime:first').get(0), - intro: "This is quick menu for every log entry which has options for getting into the detail of that log.", + intro: "The Log Time column lists a consolidated view of filtered log entries for the selected service.", position: "right" }, { element: $('li[data-id="audit"]').get(0), - intro: "This tab holds the access information across services and its components with different views and aggregation and you can also apply filters for viewing access info in detail.", + intro: "The Access Logs tab displays access information across services and their components. You can create different views and aggregations and apply filters for viewing access log details.", position: "right" }, { element: $('#r_AuditLine').get(0), - intro: "This section shows the component which are accessed related to time.", + intro: "The Line section shows component access during a specific time frame.", position: "bottom" }, { element: $('#AuditSearch').get(0), - intro: "This filter allows you to query the data column wise(Access Enforcer, Access type etc). Include Search is basically \"or\" condition and Exclude Search is \"and\" condition between multiple input.", + intro: "This filter allows you to query Access log data column- wise (Access Enforcer, Access type etc). Include Search applies the \"or\" condition and Exclude Search the \"and\" condition between multiple inputs.", position: "top" }, { element: $('li[data-id="createFilters"]').get(0), - intro: "This is the feature to filter out the data that is handled by LogSearch. For ex if you are only interested in logs with level ERROR,FATAL etc to be tracked by LogSearch.", + intro: "The Log Feeder Filter feature supports limiting data handled by LogSearch. For example, you can limit logs tracked by LogSearch to only those logs with level ERROR or FATAL.", position: "left" }].filter(function(obj) { if (obj.child) http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js index 5f93c5a..1462080 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js @@ -363,7 +363,8 @@ define(['require', onAuditTabRefresh : function(e){ ViewUtils.setLatestTimeParams(this.defaultParams); //this.fetchAuditColumns(); - this.fetchAuditLogs(this.defaultParams); + $.extend(this.collection.queryParams,this.defaultParams); + this.collection.fetch({reset:true}); this.vent.trigger("tab:refresh",this.defaultParams); }, onViewTypeChange: function(e){ http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/ApplySearchFilterView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/ApplySearchFilterView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/ApplySearchFilterView.js index 4babebc..91f10ae 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/ApplySearchFilterView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/dialog/ApplySearchFilterView.js @@ -110,7 +110,7 @@ define(['require', getColumns: function() { var that = this, cols = { - filterName: { + filtername: { label: "Name", cell: "String", editable: false, http://git-wip-us.apache.org/repos/asf/ambari/blob/96297658/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/EventHistoryLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/EventHistoryLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/EventHistoryLayoutView.js index 0f0ce58..b30dd50 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/EventHistoryLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/EventHistoryLayoutView.js @@ -439,7 +439,7 @@ define(['require', content.selectedCollectionObject.get("params")["time"] = timeType; var postObject = { userName: "admin", - filterName: content.ui.filterName.val(), + filtername: content.ui.filterName.val(), rowType: "history", values: JSON.stringify(content.selectedCollectionObject.get('params')) }
