AMBARI-18660. Log Search schema meta data loader fix (Miklos Gergely via oleewere)
Change-Id: Ibbf3f15984b3439e25cb2b7f06be1830aa389240 Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/3bb4ec10 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/3bb4ec10 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/3bb4ec10 Branch: refs/heads/branch-dev-patch-upgrade Commit: 3bb4ec107054aefecc7471d87521bb8efdd923a7 Parents: e734a7b Author: Miklos Gergely <[email protected]> Authored: Sat Nov 5 14:42:36 2016 +0100 Committer: oleewere <[email protected]> Committed: Sat Nov 5 16:08:57 2016 +0100 ---------------------------------------------------------------------- .../ambari/logsearch/conf/SolrConfig.java | 19 +- .../ambari/logsearch/conf/SolrPropsConfig.java | 1 - .../logsearch/conf/SolrUserPropsConfig.java | 12 ++ .../AbstractOperationHolderConverter.java | 50 +++-- .../AbstractSearchRequestQueryConverter.java | 4 - .../ambari/logsearch/dao/AuditSolrDao.java | 6 +- .../logsearch/dao/ServiceLogsSolrDao.java | 3 +- .../ambari/logsearch/dao/SolrAliasDao.java | 10 +- .../ambari/logsearch/dao/SolrCollectionDao.java | 8 +- .../logsearch/dao/SolrSchemaFieldDao.java | 201 ++++++++++++------- .../ambari/logsearch/dao/UserConfigSolrDao.java | 7 +- .../ambari/logsearch/doc/DocConstants.java | 1 + .../logsearch/manager/AuditLogsManager.java | 7 +- .../logsearch/manager/ServiceLogsManager.java | 8 +- .../model/request/impl/CommonSearchRequest.java | 7 + .../logsearch/rest/ServiceLogsResource.java | 9 + .../apache/ambari/logsearch/util/SolrUtil.java | 93 +++------ .../webapp/scripts/views/tabs/LogFileView.js | 9 +- .../src/main/webapp/static/schema_fields.json | 4 +- .../converter/AuditLogRequestConverterTest.java | 2 +- ...ditServiceLoadRequestQueryConverterTest.java | 4 +- ...BaseServiceLogRequestQueryConverterTest.java | 2 +- .../FieldAuditLogRequestQueryConverterTest.java | 2 +- .../ServiceLogAnyGraphRequestConverterTest.java | 4 +- ...ComponentLevelRequestQueryConverterTest.java | 4 +- ...ComponentRequestFacetQueryConverterTest.java | 2 +- ...eLogLevelCountRequestQueryConverterTest.java | 2 +- ...ceLogTreeRequestFacetQueryConverterTest.java | 4 +- ...ceLogTruncatedRequestQueryConverterTest.java | 2 +- .../UserExportRequestQueryConverterTest.java | 2 +- .../shipper-conf/input.config-ambari.json | 20 +- 31 files changed, 272 insertions(+), 237 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java index 79df3c2..fb25a63 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java @@ -19,8 +19,6 @@ package org.apache.ambari.logsearch.conf; import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; -import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; -import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder; import org.apache.commons.lang.StringUtils; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.impl.HttpClientUtil; @@ -32,11 +30,13 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.DependsOn; import org.springframework.data.solr.core.SolrTemplate; import org.springframework.data.solr.repository.config.EnableSolrRepositories; +import org.springframework.scheduling.annotation.EnableScheduling; import javax.inject.Inject; @Configuration @EnableSolrRepositories +@EnableScheduling public class SolrConfig { private static final Logger LOG = LoggerFactory.getLogger(SolrConfig.class); @@ -80,18 +80,9 @@ public class SolrConfig { solrUserConfigPropsConfig.getCollection())); } - @Bean(name = "serviceSolrFieldDao") - public SolrSchemaFieldDao serviceSolrFieldDao() { - return new SolrSchemaFieldDao(); - } - - @Bean(name = "auditSolrFieldDao") - public SolrSchemaFieldDao auditSolrFieldDao() { - return new SolrSchemaFieldDao(); - } - - @Bean(name = "userConfigSolrFieldDao") - public SolrSchemaFieldDao userConfigSolrFieldDao() { + @Bean + @DependsOn({"serviceSolrTemplate", "auditSolrTemplate"}) + public SolrSchemaFieldDao solrSchemaFieldDao() { return new SolrSchemaFieldDao(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java index 78012b0..f80d6e5 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrPropsConfig.java @@ -46,5 +46,4 @@ public interface SolrPropsConfig { String getSplitInterval(); void setSplitInterval(String splitInterval); - } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java index 38366d5..c8c96c8 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrUserPropsConfig.java @@ -44,6 +44,9 @@ public class SolrUserPropsConfig extends SolrConnectionPropsConfig { @Value("#{'${logsearch.logfeeder.include.default.level:FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN}'.split(',')}") private List<String> logLevels; + @Value("${logsearch.schema.fields.populate.interval.mins:1}") + private Integer populateIntervalMins; + @Override public String getCollection() { return collection; @@ -101,4 +104,13 @@ public class SolrUserPropsConfig extends SolrConnectionPropsConfig { public void setLogLevels(List<String> logLevels) { this.logLevels = logLevels; } + + + public Integer getPopulateIntervalMins() { + return populateIntervalMins; + } + + void setPopulateIntervalMins(Integer populateIntervalMins) { + this.populateIntervalMins = populateIntervalMins; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java index 5945d72..3c91f51 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractOperationHolderConverter.java @@ -26,13 +26,13 @@ import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.text.StrTokenizer; import org.springframework.data.solr.core.query.Criteria; import org.springframework.data.solr.core.query.Query; import org.springframework.data.solr.core.query.SimpleFilterQuery; import org.springframework.data.solr.core.query.SimpleStringCriteria; import javax.inject.Inject; -import javax.inject.Named; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -43,12 +43,7 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE extends AbstractConverterAware<REQUEST_TYPE, QUERY_TYPE> { @Inject - @Named("serviceSolrFieldDao") - private SolrSchemaFieldDao serviceSolrSchemaFieldDao; - - @Inject - @Named("auditSolrFieldDao") - private SolrSchemaFieldDao auditSolrSchemaFieldDao; + private SolrSchemaFieldDao solrSchemaFieldDao; public List<String> splitValueAsList(String value, String separator) { return StringUtils.isNotEmpty(value) ? Splitter.on(separator).omitEmptyStrings().splitToList(value) : null; @@ -111,11 +106,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE new TypeToken<List<HashMap<String, String>>>(){}.getType()); for (Map<String, String> criteriaMap : criterias) { for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) { - escapeFieldValueByType(fieldEntry); if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) { - addFilterQuery(query, new Criteria(fieldEntry.getKey()).contains(escapeFieldValueByType(fieldEntry)), false); + addLogMessageFilter(query, fieldEntry.getValue(), false); } else { - addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeFieldValueByType(fieldEntry)), false); + addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeNonLogMessageField(fieldEntry)), false); } } } @@ -129,11 +123,10 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE new TypeToken<List<HashMap<String, String>>>(){}.getType()); for (Map<String, String> criteriaMap : criterias) { for (Map.Entry<String, String> fieldEntry : criteriaMap.entrySet()) { - escapeFieldValueByType(fieldEntry); if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) { - addFilterQuery(query, new Criteria(fieldEntry.getKey()).contains(escapeFieldValueByType(fieldEntry)), true); + addLogMessageFilter(query, fieldEntry.getValue(), true); } else { - addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeFieldValueByType(fieldEntry)), true); + addFilterQuery(query, new Criteria(fieldEntry.getKey()).is(escapeNonLogMessageField(fieldEntry)), true); } } } @@ -143,18 +136,31 @@ public abstract class AbstractOperationHolderConverter <REQUEST_TYPE, QUERY_TYPE public abstract LogType getLogType(); - private String escapeFieldValueByType(Map.Entry<String, String> fieldEntry) { - String escapedFieldValue; - if (fieldEntry.getKey().equalsIgnoreCase(LOG_MESSAGE)) { - escapedFieldValue = SolrUtil.escapeForLogMessage(fieldEntry.getValue()); - } else { - escapedFieldValue = SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldEntry.getKey(), getSchemaFieldsTypeMapByLogType(getLogType())); + private void addLogMessageFilter(Query query, String value, boolean negate) { + StrTokenizer tokenizer = new StrTokenizer(value, ' ', '"'); + for (String token : tokenizer.getTokenArray()) { + token = token.trim(); + if (token.contains(" ") || !token.startsWith("*") && !token.endsWith("*")) { + addFilterQuery(query, new Criteria(LOG_MESSAGE).is(SolrUtil.escapeQueryChars(token)), negate); + } else if (token.startsWith("*") && token.endsWith("*")) { + String plainToken = StringUtils.substring(token, 1, -1); + addFilterQuery(query, new Criteria(LOG_MESSAGE).contains(SolrUtil.escapeQueryChars(plainToken)), negate); + } else if (token.startsWith("*") && !token.endsWith("*")) { + String plainToken = StringUtils.substring(token, 1); + addFilterQuery(query, new Criteria(LOG_MESSAGE).endsWith(SolrUtil.escapeQueryChars(plainToken)), negate); + } else if (!token.startsWith("*") && token.endsWith("*")) { + String plainToken = StringUtils.substring(token, 0, -1); + addFilterQuery(query, new Criteria(LOG_MESSAGE).startsWith(SolrUtil.escapeQueryChars(plainToken)), negate); + } } - return escapedFieldValue; } - private Map<String, String> getSchemaFieldsTypeMapByLogType(LogType logType) { - return LogType.AUDIT.equals(logType) ? auditSolrSchemaFieldDao.getSchemaFieldTypeMap() : serviceSolrSchemaFieldDao.getSchemaFieldTypeMap(); + private String escapeNonLogMessageField(Map.Entry<String, String> fieldEntry) { + Map<String, String> schemaFieldNameMap = solrSchemaFieldDao.getSchemaFieldNameMap(getLogType()); + Map<String, String> schemaFieldTypeMap = solrSchemaFieldDao.getSchemaFieldTypeMap(getLogType()); + String fieldType = schemaFieldNameMap.get(fieldEntry.getKey()); + String fieldTypeMetaData = schemaFieldTypeMap.get(fieldType); + return SolrUtil.putWildCardByType(fieldEntry.getValue(), fieldType, fieldTypeMetaData); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java index 2604f13..d44b866 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/converter/AbstractSearchRequestQueryConverter.java @@ -18,7 +18,6 @@ */ package org.apache.ambari.logsearch.converter; -import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; import org.apache.ambari.logsearch.model.request.impl.CommonSearchRequest; import org.apache.commons.lang.StringUtils; import org.springframework.data.domain.PageRequest; @@ -27,9 +26,6 @@ import org.springframework.data.solr.core.query.Criteria; import org.springframework.data.solr.core.query.Query; import org.springframework.data.solr.core.query.SimpleStringCriteria; -import javax.inject.Inject; -import javax.inject.Named; - public abstract class AbstractSearchRequestQueryConverter<REQUEST_TYPE extends CommonSearchRequest, QUERY_TYPE extends Query> extends AbstractOperationHolderConverter<REQUEST_TYPE, QUERY_TYPE> { http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java index 2fbdb46..309687d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -48,7 +48,6 @@ public class AuditSolrDao extends SolrDaoBase { private SolrCollectionDao solrCollectionDao; @Inject - @Named("auditSolrFieldDao") private SolrSchemaFieldDao solrSchemaFieldDao; public AuditSolrDao() { @@ -70,10 +69,9 @@ public class AuditSolrDao extends SolrDaoBase { boolean createAlias = (aliasNameIn != null && StringUtils.isNotBlank(rangerAuditCollection)); solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig); if (createAlias) { - solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig); - } else { - solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig); + solrAliasDao.setupAlias(getSolrClient(), solrAuditLogPropsConfig); } + solrSchemaFieldDao.auditCollectionSetUp(); } catch (Exception e) { LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" + solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e); http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java index 0e1d57c..cf61163 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java @@ -44,7 +44,6 @@ public class ServiceLogsSolrDao extends SolrDaoBase { private SolrTemplate serviceSolrTemplate; @Inject - @Named("serviceSolrFieldDao") private SolrSchemaFieldDao solrSchemaFieldDao; public ServiceLogsSolrDao() { @@ -62,7 +61,7 @@ public class ServiceLogsSolrDao extends SolrDaoBase { try { solrCollectionDao.checkSolrStatus(getSolrClient()); solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig); - solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig); + solrSchemaFieldDao.serviceCollectionSetUp(); } catch (Exception e) { LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl() + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString() http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java index 6389446..d78ff0da 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java @@ -32,11 +32,10 @@ import javax.inject.Named; import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.List; @Named -public class SolrAliasDao { +class SolrAliasDao { private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class); @@ -45,9 +44,9 @@ public class SolrAliasDao { @Inject private SolrCollectionDao solrCollectionDao; - protected void setupAlias(final SolrSchemaFieldDao solrSchemaFieldDao, final CloudSolrClient solrClient, - final SolrAuditLogPropsConfig solrPropsConfig) throws Exception { - final Collection<String> collectionListIn = Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim()); + void setupAlias(final CloudSolrClient solrClient, final SolrAuditLogPropsConfig solrPropsConfig) throws Exception { + final Collection<String> collectionListIn = + Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim()); if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) { LOG.info("Will not create alias " + solrPropsConfig.getAliasNameIn() + " for " + @@ -71,7 +70,6 @@ public class SolrAliasDao { if (count == collectionListIn.size()) { LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " + "Collections=" + collectionListIn); - solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig); break; } } else { http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java index c83cd99..834ba38 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java @@ -40,7 +40,7 @@ import java.util.HashSet; import java.util.List; @Named -public class SolrCollectionDao { +class SolrCollectionDao { private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class); @@ -50,7 +50,7 @@ public class SolrCollectionDao { * This will try to get the collections from the Solr. Ping doesn't work if * collection is not given */ - public boolean checkSolrStatus(CloudSolrClient cloudSolrClient) { + boolean checkSolrStatus(CloudSolrClient cloudSolrClient) { int waitDurationMS = 3 * 60 * 1000; boolean status = false; try { @@ -86,7 +86,7 @@ public class SolrCollectionDao { return status; } - public void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception { + void setupCollections(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) throws Exception { boolean setupStatus = createCollectionsIfNeeded(solrClient, solrPropsConfig); LOG.info("Setup status for " + solrPropsConfig.getCollection() + " is " + setupStatus); if (!setupStatus) { @@ -136,7 +136,7 @@ public class SolrCollectionDao { } @SuppressWarnings("unchecked") - public List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException, + List<String> getCollections(CloudSolrClient solrClient) throws SolrServerException, IOException { try { CollectionAdminRequest.List colListReq = new CollectionAdminRequest.List(); http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java index 35bc2dc..e6a562e 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -19,123 +19,178 @@ package org.apache.ambari.logsearch.dao; import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.conf.SolrPropsConfig; import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; +import org.apache.http.HttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.LukeRequest; +import org.apache.solr.client.solrj.impl.HttpClientUtil; import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition; import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.client.solrj.response.LukeResponse; import org.apache.solr.client.solrj.response.LukeResponse.FieldInfo; import org.apache.solr.client.solrj.response.schema.SchemaResponse; import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.JavaBinCodec; +import org.apache.solr.common.util.NamedList; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.scheduling.annotation.Scheduled; -import javax.inject.Inject; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import javax.annotation.PostConstruct; +import javax.inject.Inject; +import javax.inject.Named; + public class SolrSchemaFieldDao { private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class); - private static final int SETUP_RETRY_SECOND = 30; - private static final int SETUP_UPDATE_SECOND = 1 * 60; // 1 min + private static final int RETRY_SECOND = 30; - private boolean populateFieldsThreadActive = false; - - private Map<String, String> schemaFieldNameMap = new HashMap<>(); - private Map<String, String> schemaFieldTypeMap = new HashMap<>(); + @Inject + @Named("serviceSolrTemplate") + private SolrTemplate serviceSolrTemplate; @Inject - private SolrUserPropsConfig solrUserPropsConfig; + @Named("auditSolrTemplate") + private SolrTemplate auditSolrTemplate; + + @Inject + private SolrUserPropsConfig solrUserConfigPropsConfig; + + private CloudSolrClient serviceSolrClient; + private CloudSolrClient auditSolrClient; + + private int retryCount; + private int skipCount; - public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) { - if (!populateFieldsThreadActive) { - populateFieldsThreadActive = true; - LOG.info("Creating thread to populated fields for collection=" + solrPropsConfig.getCollection()); - Thread fieldPopulationThread = new Thread("populated_fields_" + solrPropsConfig.getCollection()) { - @Override - public void run() { - LOG.info("Started thread to get fields for collection=" + solrPropsConfig.getCollection()); - int retryCount = 0; - while (true) { - try { - Thread.sleep(SETUP_RETRY_SECOND * 1000); - retryCount++; - boolean _result = _populateSchemaFields(solrClient, solrPropsConfig); - if (_result) { - LOG.info("Populate fields for collection " + solrPropsConfig.getCollection() + " is success, Update it after " + - SETUP_UPDATE_SECOND + " sec"); - Thread.sleep(SETUP_UPDATE_SECOND * 1000); - } - } catch (InterruptedException sleepInterrupted) { - LOG.info("Sleep interrupted while populating fields for collection " + solrPropsConfig.getCollection()); - break; - } catch (Exception ex) { - LOG.error("Error while populating fields for collection " + solrPropsConfig.getCollection() + ", retryCount=" + retryCount); - } - } - populateFieldsThreadActive = false; - LOG.info("Exiting thread for populating fields. collection=" + solrPropsConfig.getCollection()); - } - }; - fieldPopulationThread.setDaemon(true); - fieldPopulationThread.start(); + private boolean serviceCollectionSetUp = false; + private boolean auditCollectionSetUp = false; + + private Map<String, String> serviceSchemaFieldNameMap = new HashMap<>(); + private Map<String, String> serviceSchemaFieldTypeMap = new HashMap<>(); + private Map<String, String> auditSchemaFieldNameMap = new HashMap<>(); + private Map<String, String> auditSchemaFieldTypeMap = new HashMap<>(); + + @PostConstruct + public void init() { + this.serviceSolrClient = (CloudSolrClient) serviceSolrTemplate.getSolrClient(); + this.auditSolrClient = (CloudSolrClient) auditSolrTemplate.getSolrClient(); + } + + void serviceCollectionSetUp() { + this.serviceCollectionSetUp = true; + } + + void auditCollectionSetUp() { + this.auditCollectionSetUp = true; + } + + @Scheduled(fixedDelay = RETRY_SECOND * 1000) + public void populateAllSchemaFields() { + if (skipCount > 0) { + skipCount--; + return; + } + + if (serviceCollectionSetUp) { + populateSchemaFields(serviceSolrClient, serviceSchemaFieldNameMap, serviceSchemaFieldTypeMap); + } + if (auditCollectionSetUp) { + populateSchemaFields(auditSolrClient, auditSchemaFieldNameMap, auditSchemaFieldTypeMap); } } - /** - * Called from the thread. Don't call this directly - */ - private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { - String historyCollection = solrUserPropsConfig.getCollection(); - if (solrClient != null && !solrPropsConfig.getCollection().equals(historyCollection)) { - LukeResponse lukeResponse = null; + private void populateSchemaFields(CloudSolrClient solrClient, Map<String, String> schemaFieldNameMap, + Map<String, String> schemaFieldTypeMap) { + if (solrClient != null) { + LOG.debug("Started thread to get fields for collection=" + solrClient.getDefaultCollection()); + List<LukeResponse> lukeResponses = null; SchemaResponse schemaResponse = null; try { - LukeRequest lukeRequest = new LukeRequest(); - lukeRequest.setNumTerms(0); - lukeResponse = lukeRequest.process(solrClient); - + lukeResponses = getLukeResponsesForCores(solrClient); + SolrRequest<SchemaResponse> schemaRequest = new SchemaRequest(); schemaRequest.setMethod(SolrRequest.METHOD.GET); schemaRequest.setPath("/schema"); schemaResponse = schemaRequest.process(solrClient); - LOG.debug("populateSchemaFields() collection=" + solrPropsConfig.getCollection() + ", luke=" + lukeResponse + + LOG.debug("populateSchemaFields() collection=" + solrClient.getDefaultCollection() + ", luke=" + lukeResponses + ", schema= " + schemaResponse); } catch (SolrException | SolrServerException | IOException e) { - LOG.error("Error occured while popuplating field. collection=" + solrPropsConfig.getCollection(), e); + LOG.error("Error occured while popuplating field. collection=" + solrClient.getDefaultCollection(), e); } - if (lukeResponse != null && schemaResponse != null) { - extractSchemaFieldsName(lukeResponse, schemaResponse); - return true; + if (schemaResponse != null) { + extractSchemaFieldsName(lukeResponses, schemaResponse, schemaFieldNameMap, schemaFieldTypeMap); + LOG.debug("Populate fields for collection " + solrClient.getDefaultCollection()+ " was successful, next update it after " + + solrUserConfigPropsConfig.getPopulateIntervalMins() + " minutes"); + retryCount = 0; + skipCount = (solrUserConfigPropsConfig.getPopulateIntervalMins() * 60) / RETRY_SECOND - 1; + } + else { + retryCount++; + LOG.error("Error while populating fields for collection " + solrClient.getDefaultCollection() + ", retryCount=" + retryCount); } } - return false; + } + + private static final String LUKE_REQUEST_URL_SUFFIX = "admin/luke?numTerms=0&wt=javabin&version=2"; + + @SuppressWarnings("unchecked") + private List<LukeResponse> getLukeResponsesForCores(CloudSolrClient solrClient) { + ZkStateReader zkStateReader = solrClient.getZkStateReader(); + Collection<Slice> activeSlices = zkStateReader.getClusterState().getActiveSlices(solrClient.getDefaultCollection()); + + List<LukeResponse> lukeResponses = new ArrayList<>(); + for (Slice slice : activeSlices) { + for (Replica replica : slice.getReplicas()) { + try (CloseableHttpClient httpClient = HttpClientUtil.createClient(null)) { + HttpGet request = new HttpGet(replica.getCoreUrl() + LUKE_REQUEST_URL_SUFFIX); + HttpResponse response = httpClient.execute(request); + NamedList<Object> lukeData = (NamedList<Object>) new JavaBinCodec(null, null).unmarshal(response.getEntity().getContent()); + LukeResponse lukeResponse = new LukeResponse(); + lukeResponse.setResponse(lukeData); + lukeResponses.add(lukeResponse); + } catch (IOException e) { + LOG.error("Exception during getting luke responses", e); + } + } + } + return lukeResponses; } - private void extractSchemaFieldsName(LukeResponse lukeResponse, SchemaResponse schemaResponse) { + private void extractSchemaFieldsName(List<LukeResponse> lukeResponses, SchemaResponse schemaResponse, + Map<String, String> schemaFieldNameMap, Map<String, String> schemaFieldTypeMap) { try { HashMap<String, String> _schemaFieldNameMap = new HashMap<>(); HashMap<String, String> _schemaFieldTypeMap = new HashMap<>(); - for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) { - String name = e.getKey(); - String type = e.getValue().getType(); - if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && - !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) { - _schemaFieldNameMap.put(name, type); + for (LukeResponse lukeResponse : lukeResponses) { + for (Entry<String, FieldInfo> e : lukeResponse.getFieldInfo().entrySet()) { + String name = e.getKey(); + String type = e.getValue().getType(); + if (!name.contains("@") && !name.startsWith("_") && !name.contains("_md5") && !name.contains("_ms") && + !name.contains(LogSearchConstants.NGRAM_PREFIX) && !name.contains("tags") && !name.contains("_str")) { + _schemaFieldNameMap.put(name, type); + } } } @@ -162,19 +217,21 @@ public class SolrSchemaFieldDao { } synchronized (this) { - schemaFieldNameMap = _schemaFieldNameMap; - schemaFieldTypeMap = _schemaFieldTypeMap; + schemaFieldNameMap.clear(); + schemaFieldNameMap.putAll(_schemaFieldNameMap); + schemaFieldTypeMap.clear(); + schemaFieldTypeMap.putAll(_schemaFieldTypeMap); } } catch (Exception e) { LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM); } } - public synchronized Map<String, String> getSchemaFieldNameMap() { - return schemaFieldNameMap; + public Map<String, String> getSchemaFieldNameMap(LogType logType) { + return LogType.AUDIT == logType ? auditSchemaFieldNameMap : serviceSchemaFieldNameMap; } - public synchronized Map<String, String> getSchemaFieldTypeMap() { - return schemaFieldTypeMap; + public Map<String, String> getSchemaFieldTypeMap(LogType logType) { + return LogType.AUDIT == logType ? auditSchemaFieldTypeMap : serviceSchemaFieldTypeMap; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java index 418a405..106bedc 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java @@ -69,10 +69,6 @@ public class UserConfigSolrDao extends SolrDaoBase { private SolrCollectionDao solrCollectionDao; @Inject - @Named("userConfigSolrFieldDao") - private SolrSchemaFieldDao solrSchemaFieldDao; - - @Inject @Named("userConfigSolrTemplate") private SolrTemplate userConfigSolrTemplate; @@ -94,7 +90,6 @@ public class UserConfigSolrDao extends SolrDaoBase { try { solrCollectionDao.checkSolrStatus(getSolrClient()); solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig); - solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig); intializeLogFeederFilter(); } catch (Exception e) { @@ -185,6 +180,6 @@ public class UserConfigSolrDao extends SolrDaoBase { @Override public SolrSchemaFieldDao getSolrSchemaFieldDao() { - return solrSchemaFieldDao; + throw new UnsupportedOperationException(); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java index e6f5103..3639b88 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java @@ -90,6 +90,7 @@ public class DocConstants { public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "Get service logs schema fields"; public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch"; public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data"; + public static final String REQUEST_CANCEL = "Cancel an ongoing solr request"; } public class PublicOperationDescriptions { http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java index aded1d4..3ee1d17 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditLogsManager.java @@ -36,8 +36,11 @@ import java.util.Map; import freemarker.template.Configuration; import freemarker.template.Template; import freemarker.template.TemplateException; + +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; import org.apache.ambari.logsearch.graph.GraphDataGenerator; import org.apache.ambari.logsearch.model.request.impl.AuditBarGraphRequest; import org.apache.ambari.logsearch.model.request.impl.AuditComponentRequest; @@ -79,6 +82,8 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp private ConversionService conversionService; @Inject private Configuration freemarkerConfiguration; + @Inject + private SolrSchemaFieldDao solrSchemaFieldDao; public AuditLogResponse getLogs(AuditLogRequest auditLogRequest) { return getLogAsPaginationProvided(conversionService.convert(auditLogRequest, SimpleQuery.class), auditSolrDao, "/audit/logs"); @@ -118,7 +123,7 @@ public class AuditLogsManager extends ManagerBase<SolrAuditLogData, AuditLogResp } public String getAuditLogsSchemaFieldsName() { - return convertObjToString(auditSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap()); + return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.AUDIT)); } public BarGraphDataListResponse getServiceLoad(AuditServiceLoadRequest request) { http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java index 44d0c00..8986544 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/ServiceLogsManager.java @@ -28,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Scanner; import javax.inject.Inject; import javax.inject.Named; @@ -42,8 +41,10 @@ import freemarker.template.TemplateException; import org.apache.ambari.logsearch.common.HadoopServiceConfigHelper; import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; +import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; import org.apache.ambari.logsearch.graph.GraphDataGenerator; import org.apache.ambari.logsearch.model.request.impl.ServiceAnyGraphRequest; import org.apache.ambari.logsearch.model.request.impl.ServiceGraphRequest; @@ -71,7 +72,6 @@ import org.apache.ambari.logsearch.solr.model.SolrComponentTypeLogData; import org.apache.ambari.logsearch.solr.model.SolrHostLogData; import org.apache.ambari.logsearch.solr.model.SolrServiceLogData; import org.apache.ambari.logsearch.util.DownloadUtil; -import org.apache.ambari.logsearch.util.JSONUtil; import org.apache.ambari.logsearch.util.DateUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; import org.apache.ambari.logsearch.util.SolrUtil; @@ -116,6 +116,8 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL private ConversionService conversionService; @Inject private Configuration freemarkerConfiguration; + @Inject + private SolrSchemaFieldDao solrSchemaFieldDao; public ServiceLogResponse searchLogs(ServiceLogRequest request) { String event = "/service/logs"; @@ -418,7 +420,7 @@ public class ServiceLogsManager extends ManagerBase<SolrServiceLogData, ServiceL } public String getServiceLogsSchemaFieldsName() { - return convertObjToString(serviceLogsSolrDao.getSolrSchemaFieldDao().getSchemaFieldNameMap()); + return convertObjToString(solrSchemaFieldDao.getSchemaFieldNameMap(LogType.SERVICE)); } public BarGraphDataListResponse getAnyGraphCountData(ServiceAnyGraphRequest request) { http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java index 7001cf3..2617c51 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/model/request/impl/CommonSearchRequest.java @@ -21,6 +21,8 @@ package org.apache.ambari.logsearch.model.request.impl; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.model.request.CommonSearchParamDefinition; import org.apache.ambari.logsearch.model.request.SearchRequest; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.apache.commons.lang3.builder.ToStringStyle; import javax.ws.rs.QueryParam; @@ -116,4 +118,9 @@ public class CommonSearchRequest implements SearchRequest, CommonSearchParamDefi public void setEndTime(String endTime) { this.endTime = endTime; } + + @Override + public String toString() { + return ToStringBuilder.reflectionToString(this, ToStringStyle.MULTI_LINE_STYLE); + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java index 492a5b2..3e1e120 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/rest/ServiceLogsResource.java @@ -184,4 +184,13 @@ public class ServiceLogsResource { public String getHadoopServiceConfigJSON() { return serviceLogsManager.getHadoopServiceConfigJSON(); } + + @GET + @Path("/request/cancel") + @Produces({"application/json"}) + @ApiOperation(REQUEST_CANCEL) + public String cancelRequest() { + // TODO: create function that cancels an ongoing solr request + return "{\"endpoint status\": \"not supported yet\"}"; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java index 7a856d2..a3824de 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java @@ -35,7 +35,7 @@ import org.apache.solr.schema.TrieFloatField; import org.apache.solr.schema.TrieIntField; import org.apache.solr.schema.TrieLongField; import org.apache.commons.collections.MapUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; public class SolrUtil { private SolrUtil() { @@ -74,50 +74,14 @@ public class SolrUtil { if (search == null) { return null; } - String newString = search.trim(); - String newSearch = escapeQueryChars(newString); - boolean isSingleWord = true; - for (int i = 0; i < search.length(); i++) { - if (Character.isWhitespace(search.charAt(i))) { - isSingleWord = false; - } - } - if (!isSingleWord) { + String newSearch = escapeQueryChars(search.trim()); + if (StringUtils.containsWhitespace(newSearch)) { newSearch = "\"" + newSearch + "\""; } return newSearch; } - private static String escapeForKeyTokenizer(String search) { - if (search.startsWith("*") && search.endsWith("*") && StringUtils.isNotBlank(search)) { - // Remove the * from both the sides - if (search.length() > 1) { - search = search.substring(1, search.length() - 1); - } else { - //search string have only * - search=""; - } - } - search = escapeQueryChars(search); - - return "*" + search + "*"; - } - - /** - * This is a special case scenario to handle log_message for wild card - * scenarios - */ - public static String escapeForLogMessage(String search) { - if (search.startsWith("*") && search.endsWith("*")) { - search = escapeForKeyTokenizer(search); - } else { - // Use whitespace index - search = escapeForStandardTokenizer(search); - } - return search; - } - private static String makeSolrSearchStringWithoutAsterisk(String search) { String newString = search.trim(); String newSearch = newString.replaceAll("(?=[]\\[+&|!(){}^\"~=/$@%?:.\\\\])", "\\\\"); @@ -152,48 +116,45 @@ public class SolrUtil { } - private static boolean isSolrFieldNumber(String fieldType, Map<String, String> schemaFieldsMap) { - if (StringUtils.isBlank(fieldType)) { + private static boolean isSolrFieldNumber(Map<String, Object> fieldTypeInfoMap) { + if (MapUtils.isEmpty(fieldTypeInfoMap)) { return false; - } else { - HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType, schemaFieldsMap); - if (MapUtils.isEmpty(typeInfoMap)) { - return false; - } - String fieldTypeClassName = (String) typeInfoMap.get("class"); - return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) || - fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) || - fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) || - fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName()); } + String fieldTypeClassName = (String) fieldTypeInfoMap.get("class"); + return fieldTypeClassName.equalsIgnoreCase(TrieIntField.class.getSimpleName()) || + fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class.getSimpleName()) || + fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class.getSimpleName()) || + fieldTypeClassName.equalsIgnoreCase(TrieLongField.class.getSimpleName()); } - public static String putWildCardByType(String str, String key, Map<String, String> schemaFieldsMap) { - String fieldType = schemaFieldsMap.get(key); + public static String putWildCardByType(String str, String fieldType, String fieldTypeMetaData) { + Map<String, Object> fieldTypeInfoMap = getFieldTypeInfoMap(fieldTypeMetaData); if (StringUtils.isNotBlank(fieldType)) { - if (isSolrFieldNumber(fieldType, schemaFieldsMap)) { - String value = putEscapeCharacterForNumber(str, fieldType, schemaFieldsMap); + if (isSolrFieldNumber(fieldTypeInfoMap)) { + String value = putEscapeCharacterForNumber(str, fieldTypeInfoMap); if (StringUtils.isNotBlank(value)) { return value; } else { return null; } - } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class, schemaFieldsMap)) { + } else if (checkTokenizer(StandardTokenizerFactory.class, fieldTypeInfoMap)) { return escapeForStandardTokenizer(str); - } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class, schemaFieldsMap)|| "string".equalsIgnoreCase(fieldType)) { + } else if (checkTokenizer(KeywordTokenizerFactory.class, fieldTypeInfoMap) || "string".equalsIgnoreCase(fieldType)) { return makeSolrSearchStringWithoutAsterisk(str); - } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class, schemaFieldsMap)) { + } else if (checkTokenizer(PathHierarchyTokenizerFactory.class, fieldTypeInfoMap)) { return str; + } else { + return escapeQueryChars(str); } } return str; } - private static String putEscapeCharacterForNumber(String str,String fieldType, Map<String, String> schemaFieldsMap) { + private static String putEscapeCharacterForNumber(String str, Map<String, Object> fieldTypeInfoMap) { if (StringUtils.isNotEmpty(str)) { str = str.replace("*", ""); } - String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType, schemaFieldsMap); + String escapeCharSting = parseInputValueAsPerFieldType(str, fieldTypeInfoMap); if (escapeCharSting == null || escapeCharSting.isEmpty()) { return null; } @@ -201,9 +162,8 @@ public class SolrUtil { return escapeCharSting; } - private static String parseInputValueAsPerFieldType(String str,String fieldType, Map<String, String> schemaFieldsMap) { + private static String parseInputValueAsPerFieldType(String str, Map<String, Object> fieldTypeInfoMap) { try { - HashMap<String, Object> fieldTypeInfoMap = SolrUtil.getFieldTypeInfoMap(fieldType, schemaFieldsMap); String className = (String) fieldTypeInfoMap.get("class"); if (className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())) { return "" + Double.parseDouble(str); @@ -219,8 +179,7 @@ public class SolrUtil { } } - private static HashMap<String, Object> getFieldTypeInfoMap(String fieldType, Map<String, String> schemaFieldsTypeMap) { - String fieldTypeMetaData = schemaFieldsTypeMap.get(fieldType); + private static Map<String, Object> getFieldTypeInfoMap(String fieldTypeMetaData) { HashMap<String, Object> fieldTypeMap = JSONUtil.jsonToMapObject(fieldTypeMetaData); if (fieldTypeMap == null) { return new HashMap<>(); @@ -269,10 +228,8 @@ public class SolrUtil { } @SuppressWarnings("unchecked") - private static boolean checkTokenizer(String fieldType, Class<? extends TokenizerFactory> tokenizerFactoryClass, - Map<String, String> schemaFieldsMap) { - HashMap<String, Object> fieldTypeMap = SolrUtil.getFieldTypeInfoMap(fieldType ,schemaFieldsMap); - HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap.get("analyzer"); + private static boolean checkTokenizer(Class<? extends TokenizerFactory> tokenizerFactoryClass, Map<String, Object> fieldTypeInfoMap) { + HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeInfoMap.get("analyzer"); HashMap<String, Object> tokenizerMap = (HashMap<String, Object>)MapUtils.getObject(analyzer, "tokenizer"); if (tokenizerMap != null) { String tokenizerClass = (String) tokenizerMap.get("class"); http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js index a4e3a1a..57d4acd 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/scripts/views/tabs/LogFileView.js @@ -596,12 +596,13 @@ define(['require', this.ui.find.trigger("keyup"); this.ui.find.focus(); }else if(type === "IA" || type === "EA"){ - this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns", - key:Globals.serviceLogsColumns["log_message"],value:"*"+this.selectionText+"*"}); - } + this.vent.trigger("toggle:facet",{viewName:((type === "IA") ? "include" : "exclude") +"ServiceColumns", + key:Globals.defaultServiceLogMappings["log_message"],value:"*"+this.selectionText+"*"}); + } else { //this.vent.trigger("add:include:exclude",{type:type,value:this.selectionText}); - this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns", key: Globals.serviceLogsColumns["log_message"], value: this.selectionText }); + this.vent.trigger("toggle:facet", { viewName: ((type === "I") ? "include" : "exclude") + "ServiceColumns", + key: Globals.defaultServiceLogMappings["log_message"], value: this.selectionText }); } this.ui.contextMenu.hide(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json index 55f1d3c..d145141 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/webapp/static/schema_fields.json @@ -46,7 +46,7 @@ "logType": "Log Type", "logfile_line_number": "Logfile Line Number", "logger_name": "Logger Name", - "logtime": "logtime", + "logtime": "Log Time", "path": "Path", "perm": "Perm", "policy": "Policy", @@ -76,11 +76,13 @@ "ws_repo_id": "Repo Id", "ws_repo_version": "Repo Version", "ws_repositories": "Repositories", + "ws_request_id": "Request Id", "ws_result_status": "Result Status", "ws_roles": "Roles", "ws_stack_version": "Stack Version", "ws_stack": "Stack", "ws_status": "Status", + "ws_task_id": "Task Id", "ws_version_note": "Version Note", "ws_version_number": "Version Number" }, http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java index ec6aad7..1aae82a 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditLogRequestConverterTest.java @@ -46,7 +46,7 @@ public class AuditLogRequestConverterTest extends AbstractRequestConverterTest { SolrQuery queryResult = new DefaultQueryParser().doConstructSolrQuery(simpleQuery); // THEN assertEquals("?q=*%3A*&start=0&rows=25&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" + - "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&sort=evtTime+desc%2Cseq_num+desc", + "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=evtTime+desc%2Cseq_num+desc", queryResult.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java index 71e91e1..2b629e5 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/AuditServiceLoadRequestQueryConverterTest.java @@ -43,8 +43,8 @@ public class AuditServiceLoadRequestQueryConverterTest extends AbstractRequestCo // WHEN SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*&" + - "fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" + + assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage&" + + "fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&fq=-repo%3A%28hst_agent+system_message%29" + "&facet=true&facet.mincount=1&facet.limit=10&facet.field=repo", solrQuery.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java index 3775a2d..89f8eba 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/BaseServiceLogRequestQueryConverterTest.java @@ -53,7 +53,7 @@ public class BaseServiceLogRequestQueryConverterTest extends AbstractRequestConv SolrUtil.removeDoubleOrTripleEscapeFromFilters(solrQuery); // THEN assertEquals("?q=*%3A*&start=0&rows=25&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + - "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=host%3Alogsearch.com" + + "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=host%3Alogsearch.com" + "&fq=path%3A%5C%2Fvar%5C%2Flog%5C%2Fmyfile%5C-%5C*%5C-hdfs.log&fq=type%3Acomponent&fq=level%3A%28FATAL+ERROR+WARN+UNKNOWN%29" + "&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&sort=logtime+desc%2Cseq_num+desc", solrQuery.toQueryString()); http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java index 1864af3..1428276 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/FieldAuditLogRequestQueryConverterTest.java @@ -46,7 +46,7 @@ public class FieldAuditLogRequestQueryConverterTest extends AbstractRequestConve SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&" + - "fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29&" + + "fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29&" + "fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=10&facet.pivot=myfield%2Crepo", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java index 63dc733..5d15166 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogAnyGraphRequestConverterTest.java @@ -44,8 +44,8 @@ public class ServiceLogAnyGraphRequestConverterTest extends AbstractRequestConve // WHEN SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" + - "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + + assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" + + "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + "&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java index 2a7cc16..b1ae332 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentLevelRequestQueryConverterTest.java @@ -45,8 +45,8 @@ public class ServiceLogComponentLevelRequestQueryConverterTest extends AbstractR // WHEN SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" + - "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + + assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" + + "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + "&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=type%2Clevel", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java index a5ea683..3c7f242 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogComponentRequestFacetQueryConverterTest.java @@ -46,7 +46,7 @@ public class ServiceLogComponentRequestFacetQueryConverterTest extends AbstractR SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" + - "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" + + "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" + "&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29&facet=true&facet.mincount=1&facet.limit=-1" + "&facet.sort=index&facet.pivot=type%2Chost%2Clevel&facet.pivot=type%2Clevel", query.toQueryString()); http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java index b79ff91..f6beb3d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogLevelCountRequestQueryConverterTest.java @@ -44,7 +44,7 @@ public class ServiceLogLevelCountRequestQueryConverterTest extends AbstractReque SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" + - "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29" + + "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29" + "&fq=-type%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.field=level", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java index 634ac19..107a99d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTreeRequestFacetQueryConverterTest.java @@ -44,8 +44,8 @@ public class ServiceLogTreeRequestFacetQueryConverterTest extends AbstractReques // WHEN SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN - assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3A*myincludemessage*" + - "&fq=-log_message%3A*myexcludemessage*&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" + + assertEquals("?q=*%3A*&rows=0&fq=logtime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D&fq=log_message%3Amyincludemessage" + + "&fq=-log_message%3Amyexcludemessage&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29&fq=level%3A%28WARN+ERROR+FATAL%29" + "&facet=true&facet.mincount=1&facet.limit=-1&facet.sort=index&facet.pivot=host%2Ctype%2Clevel&facet.pivot=host%2Clevel", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java index 41761bc..0151c6c 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/ServiceLogTruncatedRequestQueryConverterTest.java @@ -47,7 +47,7 @@ public class ServiceLogTruncatedRequestQueryConverterTest extends AbstractReques SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&start=0&rows=10&fq=type%3A%28logsearch_app+secure_log%29&fq=-type%3A%28hst_agent+system_message%29" + - "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&sort=logtime+desc%2Cseq_num+desc", + "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&sort=logtime+desc%2Cseq_num+desc", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java index 6644ca1..1608bbf 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/test/java/org/apache/ambari/logsearch/converter/UserExportRequestQueryConverterTest.java @@ -44,7 +44,7 @@ public class UserExportRequestQueryConverterTest extends AbstractRequestConverte SolrQuery query = new DefaultQueryParser().doConstructSolrQuery(underTest.convert(request)); // THEN assertEquals("?q=*%3A*&rows=0&fq=evtTime%3A%5B2016-09-13T22%3A00%3A01.000Z+TO+2016-09-14T22%3A00%3A01.000Z%5D" + - "&fq=log_message%3A*myincludemessage*&fq=-log_message%3A*myexcludemessage*&fq=repo%3A%28logsearch_app+secure_log%29" + + "&fq=log_message%3Amyincludemessage&fq=-log_message%3Amyexcludemessage&fq=repo%3A%28logsearch_app+secure_log%29" + "&fq=-repo%3A%28hst_agent+system_message%29&facet=true&facet.mincount=1&facet.limit=-1&facet.pivot=reqUser%2Crepo&facet.pivot=resource%2Crepo", query.toQueryString()); } http://git-wip-us.apache.org/repos/asf/ambari/blob/3bb4ec10/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json ---------------------------------------------------------------------- diff --git a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json index 576bfa8..2dfe6d3 100644 --- a/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json +++ b/ambari-logsearch/docker/test-config/logfeeder/shipper-conf/input.config-ambari.json @@ -282,16 +282,6 @@ "new_fieldname":"cliType" } }, - "RequestId":{ - "map_fieldname":{ - "new_fieldname":"request_id" - } - }, - "TaskId":{ - "map_fieldname":{ - "new_fieldname":"task_id" - } - }, "Operation":{ "map_fieldname":{ "new_fieldname":"action" @@ -374,6 +364,11 @@ "new_fieldname":"ws_repositories" } }, + "RequestId":{ + "map_fieldname":{ + "new_fieldname":"ws_request_id" + } + }, "Roles":{ "map_fieldname":{ "new_fieldname":"ws_roles" @@ -389,6 +384,11 @@ "new_fieldname":"ws_stack_version" } }, + "TaskId":{ + "map_fieldname":{ + "new_fieldname":"ws_task_id" + } + }, "VersionNote":{ "map_fieldvalue":{ "pre_value":"null",
