Repository: ambari Updated Branches: refs/heads/branch-dev-logsearch 4aa6595b7 -> 80094e91b
AMBARI-18310. Fix regression issues for logsearch refactor (escaping) (oleewere) Change-Id: I3bc878f2e4ccb2068cb12b8e479c5936edb49d11 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2da803c4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2da803c4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2da803c4 Branch: refs/heads/branch-dev-logsearch Commit: 2da803c4b78f369268404537409bce0897e75a55 Parents: 4aa6595 Author: oleewere <oleew...@gmail.com> Authored: Wed Oct 5 16:09:17 2016 +0200 Committer: oleewere <oleew...@gmail.com> Committed: Fri Oct 7 11:56:37 2016 +0200 ---------------------------------------------------------------------- .../AbstractAuditLogRequestQueryConverter.java | 4 ++-- .../converter/BaseServiceLogRequestQueryConverter.java | 9 ++++++--- .../org/apache/ambari/logsearch/dao/SolrDaoBase.java | 2 ++ .../ambari/logsearch/manager/ServiceLogsManager.java | 10 ++-------- .../ambari/logsearch/rest/ServiceLogsResource.java | 4 ---- .../org/apache/ambari/logsearch/util/SolrUtil.java | 13 +++++++++++++ .../webapp/scripts/views/audit/AuditTabLayoutView.js | 4 ++-- .../scripts/views/tabs/HierarchyTabLayoutView.js | 4 ++-- .../src/main/webapp/scripts/views/tabs/LogFileView.js | 4 ++-- .../BaseServiceLogRequestQueryConverterTest.java | 7 ++++--- .../controller/logging/LogLevelQueryResponse.java | 2 ++ .../server/controller/logging/LogQueryResponse.java | 2 ++ 12 files changed, 39 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractAuditLogRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractAuditLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractAuditLogRequestQueryConverter.java index e4bce01..5656c88 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractAuditLogRequestQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractAuditLogRequestQueryConverter.java @@ -44,8 +44,8 @@ public abstract class AbstractAuditLogRequestQueryConverter<SOURCE extends BaseL } else { defaultSortOrder = new Sort.Order(Sort.Direction.DESC, AUDIT_EVTTIME); } - Sort.Order secuqnceIdOrder = new Sort.Order(Sort.Direction.DESC, SEQUENCE_ID); - return new Sort(defaultSortOrder, secuqnceIdOrder); + Sort.Order sequenceIdOrder = new Sort.Order(Sort.Direction.DESC, SEQUENCE_ID); + return new Sort(defaultSortOrder, sequenceIdOrder); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java index 4894bfb..8270f2d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverter.java @@ -22,6 +22,7 @@ import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.model.request.impl.BaseServiceLogRequest; import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.data.domain.Sort; import org.springframework.data.solr.core.query.SimpleQuery; @@ -49,7 +50,9 @@ public class BaseServiceLogRequestQueryConverter extends AbstractServiceLogReque addEqualsFilterQuery(query, PATH, SolrUtil.escapeQueryChars(request.getFileName())); addEqualsFilterQuery(query, COMPONENT, SolrUtil.escapeQueryChars(request.getComponentName())); addEqualsFilterQuery(query, BUNDLE_ID, request.getBundleId()); - addInFilterQuery(query, LEVEL, levels); + if (CollectionUtils.isNotEmpty(levels)){ + addInFilterQuery(query, LEVEL, levels); + } addRangeFilter(query, LOGTIME, request.getFrom(), request.getTo()); return query; } @@ -65,8 +68,8 @@ public class BaseServiceLogRequestQueryConverter extends AbstractServiceLogReque } else { defaultSortOrder = new Sort.Order(Sort.Direction.DESC, LOGTIME); } - Sort.Order secuqnceIdOrder = new Sort.Order(Sort.Direction.DESC, SEQUENCE_ID); - return new Sort(defaultSortOrder, secuqnceIdOrder); + Sort.Order sequenceIdOrder = new Sort.Order(Sort.Direction.DESC, SEQUENCE_ID); + return new Sort(defaultSortOrder, sequenceIdOrder); } @Override http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java index d9a9464..77eb51f 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -23,6 +23,7 @@ import org.apache.ambari.logsearch.common.LogSearchContext; import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; @@ -49,6 +50,7 @@ public abstract class SolrDaoBase { } public QueryResponse process(SolrQuery solrQuery, String event) { + SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); if (getSolrClient() != null) { event = event == null ? solrQuery.get("event") : event; solrQuery.remove("event"); http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java index 6e319da..d276769 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java @@ -216,16 +216,10 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL public LogListResponse getPageByKeyword(ServiceLogRequest request, String event) throws SolrServerException { String defaultChoice = "0"; - String key = request.getKeyWord(); - if (StringUtils.isBlank(key)) { + String keyword = request.getKeyWord(); + if (StringUtils.isBlank(keyword)) { throw RESTErrorUtil.createRESTException("Keyword was not given", MessageEnums.DATA_NOT_FOUND); } - String keyword = SolrUtil.escapeForStandardTokenizer(key); - - if (keyword.startsWith("\"") && keyword.endsWith("\"")) { - keyword = keyword.substring(1); - keyword = keyword.substring(0, keyword.length() - 1); - } boolean isNext = !defaultChoice.equals(request.getKeywordType()); // 1 is next, 0 is previous return getPageForKeywordByType(request, keyword, isNext, event); http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java index d6ceca5..492a5b2 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java @@ -47,7 +47,6 @@ import org.apache.ambari.logsearch.model.response.NodeListResponse; import org.apache.ambari.logsearch.model.response.ServiceLogResponse; import org.apache.ambari.logsearch.manager.ServiceLogsManager; import org.springframework.context.annotation.Scope; -import org.springframework.core.convert.ConversionService; import static org.apache.ambari.logsearch.doc.DocConstants.ServiceOperationDescriptions.*; @@ -60,9 +59,6 @@ public class ServiceLogsResource { @Inject private ServiceLogsManager serviceLogsManager; - @Inject - private ConversionService conversionService; - @GET @Produces({"application/json"}) @ApiOperation(SEARCH_LOGS_OD) http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java index 51a0dda..6369107 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java @@ -19,7 +19,9 @@ package org.apache.ambari.logsearch.util; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.ambari.logsearch.common.LogSearchConstants; @@ -146,6 +148,17 @@ public class SolrUtil { return newSearch.replace(" ", "\\ "); } + + public static void removeDoubleOrTripleEscapeFromFilters(SolrQuery solrQuery) { + String[] filterQueries = solrQuery.getFilterQueries(); + List<String> newArray = new ArrayList<>(); + if (filterQueries != null && filterQueries.length > 0) { + for (String filterQuery : filterQueries) { + newArray.add(filterQuery.replaceAll("\\\\\\\\\\\\|\\\\\\\\", "\\\\")); + } + } + solrQuery.setFilterQueries(newArray.toArray(new String[0])); + } public static boolean isSolrFieldNumber(String fieldType, Map<String, String> schemaFieldsMap) { http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js index 371edd2..4e09e88 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/audit/AuditTabLayoutView.js @@ -193,7 +193,7 @@ define(['require', var obj = ViewUtils.replaceColumnNamesWithKeys(searchCollection, Globals.invertedAuditLogMappings, true); return { includeQuery: JSON.stringify(obj), - iMessage: query + query: query } } })); @@ -207,7 +207,7 @@ define(['require', var obj = ViewUtils.replaceColumnNamesWithKeys(searchCollection, Globals.invertedAuditLogMappings, true); return { excludeQuery: JSON.stringify(obj), - eMessage: query + query: query } } })); http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js index c504afa..43ee5db 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/HierarchyTabLayoutView.js @@ -287,7 +287,7 @@ define(['require', var obj = ViewUtils.replaceColumnNamesWithKeys(searchCollection, Globals.invertedServiceLogMappings, false); return { includeQuery: JSON.stringify(obj), - iMessage: query + query: query } } })); @@ -302,7 +302,7 @@ define(['require', var obj = ViewUtils.replaceColumnNamesWithKeys(searchCollection, Globals.invertedServiceLogMappings, false); return { excludeQuery: JSON.stringify(obj), - eMessage: query + query: query } } })); http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js index 2d39e36..a4e3a1a 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js @@ -318,7 +318,7 @@ define(['require', var obj = ViewUtils.replaceColumnNamesWithKeys(searchCollection, Globals.invertedServiceLogMappings, false); return { includeQuery: JSON.stringify(obj), - iMessage: query + query: query } } })); @@ -334,7 +334,7 @@ define(['require', var obj = ViewUtils.replaceColumnNamesWithKeys(searchCollection, Globals.invertedServiceLogMappings, false); return { excludeQuery: JSON.stringify(obj), - eMessage: query + query: query } } })); http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java index 1ade3c0..a75037b 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java @@ -19,6 +19,7 @@ package org.apache.ambari.logsearch.converter; import org.apache.ambari.logsearch.model.request.impl.ServiceLogRequest; +import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.solr.client.solrj.SolrQuery; import org.junit.Before; import org.junit.Test; @@ -42,17 +43,17 @@ public class BaseServiceLogRequestQueryConverterTest extends AbstractRequestConv ServiceLogRequest logRequest = new ServiceLogRequest(); fillBaseLogRequestWithTestData(logRequest); logRequest.setLevel("FATAL,ERROR,WARN,UNKNOWN"); - logRequest.setFileName("myfile"); + logRequest.setFileName("/var/log/myfile-*-hdfs.log"); logRequest.setComponentName("component"); logRequest.setHostName("logsearch.com"); // WHEN SimpleQuery query = underTest.convert(logRequest); DefaultQueryParser defaultQueryParser = new DefaultQueryParser(); SolrQuery solrQuery = defaultQueryParser.doConstructSolrQuery(query); - + SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); // THEN assertEquals("?q=*%3A*&start=0&rows=25&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*" + - "&fq=host%3Alogsearch.com&fq=path%3Amyfile&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" + + "&fq=host%3Alogsearch.com&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" + "&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc", solrQuery.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLevelQueryResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLevelQueryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLevelQueryResponse.java index 1280877..27e4c9c 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLevelQueryResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogLevelQueryResponse.java @@ -17,10 +17,12 @@ */ package org.apache.ambari.server.controller.logging; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.annotate.JsonProperty; import java.util.List; +@JsonIgnoreProperties(ignoreUnknown = true) public class LogLevelQueryResponse { private String startIndex; http://git-wip-us.apache.org/repos/asf/ambari/blob/2da803c4/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java ---------------------------------------------------------------------- diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java index 9bde7a5..feba8c2 100644 --- a/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java +++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/logging/LogQueryResponse.java @@ -18,6 +18,7 @@ package org.apache.ambari.server.controller.logging; +import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.annotate.JsonProperty; import org.codehaus.jackson.map.util.JSONPObject; @@ -29,6 +30,7 @@ import java.util.List; * a call to the LogSearch service. * */ +@JsonIgnoreProperties(ignoreUnknown = true) public class LogQueryResponse {
