AMBARI-18310. Logsearch - Refactor solr query layer (oleewere) Change-Id: I0e99217fea1de03135b86e007fcb2a3ff6c2e257
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ea644cc4 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ea644cc4 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ea644cc4 Branch: refs/heads/branch-dev-logsearch Commit: ea644cc434be39b026826589020ffa12ad3417fb Parents: 3013589 Author: oleewere <[email protected]> Authored: Mon Sep 12 21:35:08 2016 +0200 Committer: oleewere <[email protected]> Committed: Tue Sep 27 21:50:35 2016 +0200 ---------------------------------------------------------------------- .../ambari-logsearch-portal/pom.xml | 10 + .../logsearch/common/ExternalServerClient.java | 4 +- .../logsearch/common/LogSearchConstants.java | 51 +- .../apache/ambari/logsearch/common/LogType.java | 34 + .../logsearch/conf/ApplicationConfig.java | 13 + .../ambari/logsearch/conf/SolrConfig.java | 11 + .../ambari/logsearch/dao/AuditSolrDao.java | 23 +- .../logsearch/dao/ServiceLogsSolrDao.java | 21 +- .../ambari/logsearch/dao/SolrAliasDao.java | 8 +- .../ambari/logsearch/dao/SolrCollectionDao.java | 4 +- .../ambari/logsearch/dao/SolrDaoBase.java | 99 +- .../logsearch/dao/SolrSchemaFieldDao.java | 24 +- .../ambari/logsearch/dao/UserConfigSolrDao.java | 23 +- .../ambari/logsearch/doc/DocConstants.java | 18 +- .../logsearch/graph/GraphDataGenerator.java | 625 +++--- .../logsearch/graph/GraphDataGeneratorBase.java | 169 -- .../logsearch/manager/AuditLogsManager.java | 478 +---- .../ambari/logsearch/manager/ManagerBase.java | 137 +- .../ambari/logsearch/manager/PublicManager.java | 4 +- .../logsearch/manager/ServiceLogsManager.java | 1771 +++--------------- .../logsearch/manager/SessionManager.java | 5 +- .../logsearch/manager/UserConfigManager.java | 70 +- .../model/request/LogParamDefinition.java | 7 + .../request/LogTruncatedParamDefinition.java | 4 +- .../model/request/QueryParamDefinition.java | 32 - .../request/ServiceLogParamDefinition.java | 13 - .../model/request/TopParamDefinition.java | 31 + .../model/request/impl/AnyGraphRequest.java | 109 -- .../request/impl/AuditBarGraphRequest.java | 2 +- .../request/impl/AuditComponentRequest.java | 25 + .../model/request/impl/AuditLogRequest.java | 2 +- .../request/impl/AuditServiceLoadRequest.java | 25 + .../model/request/impl/BaseAuditLogRequest.java | 53 - .../model/request/impl/BaseLogRequest.java | 43 +- .../request/impl/BaseServiceLogRequest.java | 54 +- .../request/impl/FieldAuditLogRequest.java | 18 +- .../model/request/impl/QueryRequest.java | 40 - .../impl/ServiceExtremeDatesRequest.java | 41 - .../impl/ServiceLogAggregatedInfoRequest.java | 25 + .../impl/ServiceLogComponentHostRequest.java | 39 + .../impl/ServiceLogComponentLevelRequest.java | 25 + .../impl/ServiceLogHostComponentRequest.java | 39 + .../impl/ServiceLogLevelCountRequest.java | 25 + .../impl/ServiceLogTruncatedRequest.java | 6 +- .../model/request/impl/SimpleQueryRequest.java | 42 - .../logsearch/model/response/TemplateData.java | 36 + .../ambari/logsearch/query/QueryGeneration.java | 340 ---- .../logsearch/query/QueryGenerationBase.java | 282 --- .../AbstractAuditLogRequestQueryConverter.java | 47 + .../AbstractCommonAuditLogRequestConverter.java | 44 - .../AbstractCommonSearchRequestConverter.java | 55 - ...bstractCommonServiceLogRequestConverter.java | 51 - .../query/converter/AbstractConverterAware.java | 4 +- .../AbstractDateRangeFacetQueryConverter.java | 55 + .../AbstractLogRequestFacetQueryConverter.java | 61 + .../AbstractLogRequestQueryConverter.java | 44 + .../AbstractSearchRequestQueryConverter.java | 173 ++ .../converter/AnyGraphRequestConverter.java | 39 - .../AuditBarGraphRequestConverter.java | 34 - .../AuditBarGraphRequestQueryConverter.java | 40 + .../AuditComponentsRequestQueryConverter.java | 62 + .../converter/AuditLogRequestConverter.java | 34 - .../AuditLogRequestQueryConverter.java | 44 + .../AuditServiceLoadRequestQueryConverter.java | 47 + .../converter/BaseAuditLogRequestConverter.java | 33 - .../BaseServiceLogRequestConverter.java | 33 - .../BaseServiceLogRequestQueryConverter.java | 82 + .../FieldAuditLogRequestConverter.java | 34 - .../FieldAuditLogRequestQueryConverter.java | 48 + .../FieldBarGraphRequestConverter.java | 35 - .../ServiceAnyGraphRequestConverter.java | 39 - .../ServiceExtremeDatesRequestConverter.java | 34 - .../converter/ServiceGraphRequestConverter.java | 34 - ...ServiceLogAnyGraphRequestQueryConverter.java | 46 + ...eLogComponentLevelRequestQueryConverter.java | 61 + ...eLogComponentRequestFacetQueryConverter.java | 62 + .../ServiceLogExportRequestConverter.java | 35 - ...rviceLogLevelCountRequestQueryConverter.java | 46 + ...eLogLevelDateRangeRequestQueryConverter.java | 55 + .../converter/ServiceLogRequestConverter.java | 39 - ...erviceLogTreeRequestFacetQueryConverter.java | 63 + .../ServiceLogTruncatedRequestConverter.java | 36 - ...erviceLogTruncatedRequestQueryConverter.java | 95 + .../converter/SimpleQueryRequestConverter.java | 35 - .../StringFieldFacetQueryConverter.java | 44 + .../converter/UserExportRequestConverter.java | 36 - .../UserExportRequestQueryConverter.java | 49 + .../query/model/AnyGraphSearchCriteria.java | 77 - .../model/AuditBarGraphSearchCriteria.java | 33 - .../query/model/AuditLogSearchCriteria.java | 33 - .../model/CommonServiceLogSearchCriteria.java | 88 - .../model/FieldAuditBarGraphSearchCriteria.java | 32 - .../model/FieldAuditLogSearchCriteria.java | 32 - .../model/ServiceAnyGraphSearchCriteria.java | 60 - .../model/ServiceExtremeDatesCriteria.java | 32 - .../query/model/ServiceGraphSearchCriteria.java | 32 - .../model/ServiceLogExportSearchCriteria.java | 40 - .../query/model/ServiceLogSearchCriteria.java | 65 - .../ServiceLogTruncatedSearchCriteria.java | 48 - .../query/model/UserExportSearchCriteria.java | 31 - .../logsearch/rest/AuditLogsResource.java | 79 +- .../ambari/logsearch/rest/PublicResource.java | 4 +- .../logsearch/rest/ServiceLogsResource.java | 88 +- .../logsearch/rest/UserConfigResource.java | 4 +- .../ambari/logsearch/solr/SolrConstants.java | 62 + .../apache/ambari/logsearch/util/BizUtil.java | 261 --- .../apache/ambari/logsearch/util/DateUtil.java | 9 + .../ambari/logsearch/util/DownloadUtil.java | 176 ++ .../apache/ambari/logsearch/util/FileUtil.java | 84 - .../apache/ambari/logsearch/util/SolrUtil.java | 292 +-- .../LogsearchKRBAuthenticationFilter.java | 1 - .../LogsearchAuthenticationProvider.java | 4 +- ...rchExternalServerAuthenticationProvider.java | 9 +- .../LogsearchFileAuthenticationProvider.java | 4 +- .../LogsearchLdapAuthenticationProvider.java | 4 +- .../LogsearchSimpleAuthenticationProvider.java | 4 +- .../src/main/resources/default.properties | 1 - .../main/resources/templates/audit_log_txt.ftl | 42 + .../resources/templates/service_log_txt.ftl | 36 + .../scripts/collection_bases/VLogListBase.js | 2 +- .../src/main/webapp/scripts/utils/ViewUtils.js | 2 +- .../scripts/views/audit/AuditAggregatedView.js | 6 +- .../scripts/views/dashboard/LogLevelBoxView.js | 2 +- .../views/filter/CreateLogfeederFilterView.js | 6 +- .../views/tabs/HierarchyTabLayoutView.js | 1 + .../troubleshoot/TroubleShootLayoutView.js | 6 +- .../webapp/templates/tabs/LogFileView_tmpl.html | 50 +- ...BaseServiceLogRequestQueryConverterTest.java | 78 + .../logging/LoggingRequestHelperImpl.java | 2 +- 129 files changed, 3143 insertions(+), 5647 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index a1d1bfc..5485ca1 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -757,5 +757,15 @@ <artifactId>spring-data-solr</artifactId> <version>${spring-data-solr.version}</version> </dependency> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-context-support</artifactId> + <version>${spring.version}</version> + </dependency> + <dependency> + <groupId>org.freemarker</groupId> + <artifactId>freemarker</artifactId> + <version>2.3.20</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java index 9682a3d..c476b9d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/ExternalServerClient.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import javax.inject.Inject; +import javax.inject.Named; import javax.ws.rs.client.Invocation; import javax.ws.rs.client.WebTarget; import javax.ws.rs.core.MediaType; @@ -33,12 +34,11 @@ import org.apache.log4j.Logger; import org.glassfish.jersey.client.JerseyClient; import org.glassfish.jersey.client.JerseyClientBuilder; import org.glassfish.jersey.client.authentication.HttpAuthenticationFeature; -import org.springframework.stereotype.Component; /** * Layer to send REST request to External server using jersey client */ -@Component +@Named public class ExternalServerClient { private static Logger LOG = Logger.getLogger(ExternalServerClient.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java index d9d3b86..4e23146 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java @@ -29,7 +29,7 @@ public class LogSearchConstants { public static final String FATAL = "FATAL"; public static final String UNKNOWN = "UNKNOWN"; - public static final String[] SUPPORTED_LOG_LEVEL ={FATAL,ERROR,WARN,INFO,DEBUG,TRACE,UNKNOWN}; + public static final String[] SUPPORTED_LOG_LEVELS = {FATAL, ERROR, WARN, INFO, DEBUG, TRACE, UNKNOWN}; // Application Constants public static final String HOST = "H"; @@ -46,40 +46,10 @@ public class LogSearchConstants { public static final String COMPOSITE_KEY = "composite_filtername-username"; public static final String SHARE_NAME_LIST = "share_username_list"; - // SOLR Document Constants for ServiceLogs - public static final String BUNDLE_ID = "bundle_id"; - public static final String LOGTIME = "logtime"; - public static final String SEQUNCE_ID = "seq_num"; - public static final String SOLR_COMPONENT = "type"; - public static final String SOLR_LOG_MESSAGE = "log_message"; - public static final String SOLR_KEY_LOG_MESSAGE = "key_log_message"; - public static final String SOLR_HOST = "host"; - public static final String SOLR_LEVEL = "level"; - public static final String SOLR_THREAD_NAME = "thread_name"; - public static final String SOLR_LOGGER_NAME = "logger_name"; - public static final String SOLR_FILE = "file"; - public static final String SOLR_LINE_NUMBER = "line_number"; - public static final String SOLR_PATH = "path"; - - //SOLR Document Constant for audit log - public static final String AUDIT_COMPONENT = "repo"; - public static final String AUDIT_EVTTIME = "evtTime"; - public static final String AUDIT_REQUEST_USER = "reqUser"; - - // Operator's - public static final String MINUS_OPERATOR = "-"; - public static final String NO_OPERATOR = ""; - - //operation - public static final String EXCLUDE_QUERY = "excludeQuery"; - public static final String INCLUDE_QUERY = "includeQuery"; - // Seprator's public static final String I_E_SEPRATOR = "\\|i\\:\\:e\\|"; //SUFFIX - public static final String UI_SUFFIX = "@UI@"; - public static final String SOLR_SUFFIX = "@Solr@"; public static final String NGRAM_SUFFIX = "ngram_"; //Date Format for SOLR @@ -96,35 +66,21 @@ public class LogSearchConstants { // logfeeder public static final String LOGFEEDER_FILTER_NAME = "log_feeder_config"; - public static final String LIST_SEPARATOR = ","; - + public static final String SORT = "sort"; public static final String FL = "fl"; //Facet Constant public static final String FACET_FIELD = "facet.field"; - public static final String FACET_MINCOUNT = "facet.mincount"; - public static final String FACET_JSON_FIELD = "json.facet"; public static final String FACET_PIVOT = "facet.pivot"; public static final String FACET_PIVOT_MINCOUNT = "facet.pivot.mincount"; - public static final String FACET_DATE = "facet.date"; - public static final String FACET_DATE_START = "facet.date.start"; - public static final String FACET_DATE_END = "facet.date.end"; - public static final String FACET_DATE_GAP = "facet.date.gap"; - public static final String FACET_RANGE = "facet.range"; - public static final String FACET_RANGE_START = "facet.range.start"; - public static final String FACET_RANGE_END = "facet.range.end"; - public static final String FACET_RANGE_GAP = "facet.range.gap"; - public static final String FACET_GROUP = "group"; - public static final String FACET_GROUP_MAIN = "group.main"; - public static final String FACET_GROUP_FIELD = "group.field"; // Request params - public static final String REQUEST_PARAM_QUERY = "q"; public static final String REQUEST_PARAM_XAXIS = "xAxis"; public static final String REQUEST_PARAM_YAXIS = "yAxis"; public static final String REQUEST_PARAM_STACK_BY = "stackBy"; public static final String REQUEST_PARAM_UNIT = "unit"; + public static final String REQUEST_PARAM_TOP = "top"; public static final String REQUEST_PARAM_BUNDLE_ID = "bundle_id"; public static final String REQUEST_PARAM_START_INDEX = "startIndex"; public static final String REQUEST_PARAM_PAGE = "page"; @@ -151,7 +107,6 @@ public class LogSearchConstants { public static final String REQUEST_PARAM_HOST_NAME = "host_name"; public static final String REQUEST_PARAM_COMPONENT_NAME = "component_name"; public static final String REQUEST_PARAM_FILE_NAME = "file_name"; - public static final String REQUEST_PARAM_DATE_RANGE_LABEL = "dateRangeLabel"; public static final String REQUEST_PARAM_KEYWORD = "find"; public static final String REQUEST_PARAM_SOURCE_LOG_ID = "sourceLogId"; public static final String REQUEST_PARAM_KEYWORD_TYPE = "keywordType"; http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java new file mode 100644 index 0000000..2e6cddb --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogType.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.common; + +public enum LogType { + SERVICE("Service"), + AUDIT("Audit"); + + private String label; + + private LogType(String label) { + this.label = label; + } + + public String getLabel() { + return label; + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java index b279a83..82a09b2 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/ApplicationConfig.java @@ -18,12 +18,16 @@ */ package org.apache.ambari.logsearch.conf; +import freemarker.template.TemplateException; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; import org.springframework.context.support.ConversionServiceFactoryBean; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; +import org.springframework.ui.freemarker.FreeMarkerConfigurationFactoryBean; + +import java.io.IOException; @Configuration @ComponentScan("org.apache.ambari.logsearch") @@ -42,4 +46,13 @@ public class ApplicationConfig { return conversionServiceFactoryBean; } + @Bean + public freemarker.template.Configuration freemarkerConfiguration() throws IOException, TemplateException { + FreeMarkerConfigurationFactoryBean factoryBean = new FreeMarkerConfigurationFactoryBean(); + factoryBean.setPreferFileSystemAccess(false); + factoryBean.setTemplateLoaderPath("classpath:/templates"); + factoryBean.afterPropertiesSet(); + return factoryBean.getObject(); + } + } http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java index 7508fb1..4868409 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/conf/SolrConfig.java @@ -18,6 +18,7 @@ */ package org.apache.ambari.logsearch.conf; +import org.apache.ambari.logsearch.dao.SolrSchemaFieldDao; import org.apache.ambari.logsearch.solr.AmbariSolrCloudClient; import org.apache.ambari.logsearch.solr.AmbariSolrCloudClientBuilder; import org.apache.commons.lang.StringUtils; @@ -79,6 +80,16 @@ public class SolrConfig { solrUserConfigPropsConfig.getCollection())); } + @Bean(name = "serviceSolrFieldDao") + public SolrSchemaFieldDao serviceSolrFieldDao() { + return new SolrSchemaFieldDao(); + } + + @Bean(name = "auditSolrFieldDao") + public SolrSchemaFieldDao auditSolrFieldDao() { + return new SolrSchemaFieldDao(); + } + private CloudSolrClient createClient(String solrUrl, String zookeeperConnectString, String defaultCollection) { if (StringUtils.isNotEmpty(zookeeperConnectString)) { CloudSolrClient cloudSolrClient = new CloudSolrClient(zookeeperConnectString); http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java index 8d6a4da..959d6f4 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/AuditSolrDao.java @@ -21,17 +21,15 @@ package org.apache.ambari.logsearch.dao; import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.inject.Named; +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.conf.SolrAuditLogPropsConfig; -import org.apache.ambari.logsearch.manager.ManagerBase.LogType; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.solr.core.SolrTemplate; -import org.springframework.stereotype.Component; -@Component +@Named public class AuditSolrDao extends SolrDaoBase { private static final Logger LOG = Logger.getLogger(AuditSolrDao.class); @@ -40,7 +38,7 @@ public class AuditSolrDao extends SolrDaoBase { private SolrAuditLogPropsConfig solrAuditLogPropsConfig; @Inject - @Qualifier("auditSolrTemplate") + @Named("auditSolrTemplate") private SolrTemplate auditSolrTemplate; @Inject @@ -50,6 +48,7 @@ public class AuditSolrDao extends SolrDaoBase { private SolrCollectionDao solrCollectionDao; @Inject + @Named("auditSolrFieldDao") private SolrSchemaFieldDao solrSchemaFieldDao; public AuditSolrDao() { @@ -57,8 +56,8 @@ public class AuditSolrDao extends SolrDaoBase { } @Override - public CloudSolrClient getSolrClient() { - return (CloudSolrClient) auditSolrTemplate.getSolrClient(); + public SolrTemplate getSolrTemplate() { + return auditSolrTemplate; } @PostConstruct @@ -71,13 +70,17 @@ public class AuditSolrDao extends SolrDaoBase { boolean createAlias = (aliasNameIn != null && !StringUtils.isBlank(rangerAuditCollection)); solrCollectionDao.setupCollections(getSolrClient(), solrAuditLogPropsConfig); if (createAlias) { - solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig, this); + solrAliasDao.setupAlias(solrSchemaFieldDao, getSolrClient(), solrAuditLogPropsConfig); } else { - solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig, this); + solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrAuditLogPropsConfig); } } catch (Exception e) { LOG.error("Error while connecting to Solr for audit logs : solrUrl=" + solrAuditLogPropsConfig.getSolrUrl() + ", zkConnectString=" + solrAuditLogPropsConfig.getZkConnectString() + ", collection=" + solrAuditLogPropsConfig.getCollection(), e); } } + + public SolrSchemaFieldDao getSolrSchemaFieldDao() { + return solrSchemaFieldDao; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java index a6f5acf..0e1d57c 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/ServiceLogsSolrDao.java @@ -21,16 +21,14 @@ package org.apache.ambari.logsearch.dao; import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.inject.Named; +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.conf.SolrServiceLogPropsConfig; -import org.apache.ambari.logsearch.manager.ManagerBase.LogType; import org.apache.log4j.Logger; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.solr.core.SolrTemplate; -import org.springframework.stereotype.Component; -@Component +@Named public class ServiceLogsSolrDao extends SolrDaoBase { private static final Logger LOG = Logger.getLogger(ServiceLogsSolrDao.class); @@ -42,10 +40,11 @@ public class ServiceLogsSolrDao extends SolrDaoBase { private SolrServiceLogPropsConfig solrServiceLogPropsConfig; @Inject - @Qualifier("serviceSolrTemplate") + @Named("serviceSolrTemplate") private SolrTemplate serviceSolrTemplate; @Inject + @Named("serviceSolrFieldDao") private SolrSchemaFieldDao solrSchemaFieldDao; public ServiceLogsSolrDao() { @@ -53,8 +52,8 @@ public class ServiceLogsSolrDao extends SolrDaoBase { } @Override - public CloudSolrClient getSolrClient() { - return (CloudSolrClient) serviceSolrTemplate.getSolrClient(); + public SolrTemplate getSolrTemplate() { + return serviceSolrTemplate; } @PostConstruct @@ -63,11 +62,15 @@ public class ServiceLogsSolrDao extends SolrDaoBase { try { solrCollectionDao.checkSolrStatus(getSolrClient()); solrCollectionDao.setupCollections(getSolrClient(), solrServiceLogPropsConfig); - solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig, this); + solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrServiceLogPropsConfig); } catch (Exception e) { LOG.error("error while connecting to Solr for service logs : solrUrl=" + solrServiceLogPropsConfig.getSolrUrl() + ", zkConnectString=" + solrServiceLogPropsConfig.getZkConnectString() + ", collection=" + solrServiceLogPropsConfig.getCollection(), e); } } + + public SolrSchemaFieldDao getSolrSchemaFieldDao() { + return solrSchemaFieldDao; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java index 81471d3..6389446 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrAliasDao.java @@ -26,16 +26,16 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; import javax.inject.Inject; +import javax.inject.Named; import java.io.IOException; import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.List; -@Component +@Named public class SolrAliasDao { private static final Logger LOG = LoggerFactory.getLogger(SolrAliasDao.class); @@ -46,7 +46,7 @@ public class SolrAliasDao { private SolrCollectionDao solrCollectionDao; protected void setupAlias(final SolrSchemaFieldDao solrSchemaFieldDao, final CloudSolrClient solrClient, - final SolrAuditLogPropsConfig solrPropsConfig, final SolrDaoBase solrDaoBase) throws Exception { + final SolrAuditLogPropsConfig solrPropsConfig) throws Exception { final Collection<String> collectionListIn = Arrays.asList(solrPropsConfig.getCollection(), solrPropsConfig.getRangerCollection().trim()); if (solrPropsConfig.getAliasNameIn() == null || collectionListIn.size() == 0 || solrClient == null) { @@ -71,7 +71,7 @@ public class SolrAliasDao { if (count == collectionListIn.size()) { LOG.info("Setup for alias " + solrPropsConfig.getAliasNameIn() + " is successful. Exiting setup retry thread. " + "Collections=" + collectionListIn); - solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig, solrDaoBase); + solrSchemaFieldDao.populateSchemaFields(solrClient, solrPropsConfig); break; } } else { http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java index ed93b4d..1cbac31 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrCollectionDao.java @@ -30,15 +30,15 @@ import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; +import javax.inject.Named; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.List; -@Component +@Named public class SolrCollectionDao { private static final Logger LOG = LoggerFactory.getLogger(SolrCollectionDao.class); http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java index 8381948..d9a9464 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -19,88 +19,91 @@ package org.apache.ambari.logsearch.dao; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import org.apache.ambari.logsearch.common.LogSearchContext; +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.common.MessageEnums; -import org.apache.ambari.logsearch.manager.ManagerBase.LogType; -import org.apache.ambari.logsearch.model.response.BarGraphData; -import org.apache.ambari.logsearch.model.response.NameValueData; -import org.apache.ambari.logsearch.util.DateUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; import org.apache.log4j.Logger; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest.METHOD; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SimpleOrderedMap; +import org.springframework.data.solr.core.DefaultQueryParser; +import org.springframework.data.solr.core.SolrCallback; +import org.springframework.data.solr.core.SolrTemplate; +import org.springframework.data.solr.core.query.SolrDataQuery; + +import java.io.IOException; public abstract class SolrDaoBase { + private static final Logger LOG = Logger.getLogger(SolrDaoBase.class); private static final Logger LOG_PERFORMANCE = Logger.getLogger("org.apache.ambari.logsearch.performance"); - public Map<String, String> schemaFieldNameMap = new HashMap<>(); - public Map<String, String> schemaFieldTypeMap = new HashMap<>(); - private LogType logType; protected SolrDaoBase(LogType logType) { this.logType = logType; } - public QueryResponse process(SolrQuery solrQuery) throws SolrServerException, IOException { + public QueryResponse process(SolrQuery solrQuery, String event) { if (getSolrClient() != null) { - String event = solrQuery.get("event"); + event = event == null ? solrQuery.get("event") : event; solrQuery.remove("event"); - QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST); - if (event != null && !"/audit/logs/live/count".equalsIgnoreCase(event)) { - LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " + + try { + QueryResponse queryResponse = getSolrClient().query(solrQuery, METHOD.POST); + if (event != null) { + LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Event :- " + event + " SolrQuery :- " + solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " + queryResponse.getElapsedTime()); + } + return queryResponse; + } catch (Exception e){ + LOG.error("Error during solrQuery=" + e); + throw RESTErrorUtil.createRESTException(MessageEnums.SOLR_ERROR.getMessage().getMessage(), MessageEnums.ERROR_SYSTEM); } - return queryResponse; } else { throw RESTErrorUtil.createRESTException("Solr configuration improper for " + logType.getLabel() +" logs", MessageEnums.ERROR_SYSTEM); } } - @SuppressWarnings("unchecked") - public void extractValuesFromBuckets(SimpleOrderedMap<Object> jsonFacetResponse, String outerField, String innerField, - List<BarGraphData> histogramData) { - NamedList<Object> stack = (NamedList<Object>) jsonFacetResponse.get(outerField); - ArrayList<Object> stackBuckets = (ArrayList<Object>) stack.get("buckets"); - for (Object temp : stackBuckets) { - BarGraphData vBarGraphData = new BarGraphData(); - - SimpleOrderedMap<Object> level = (SimpleOrderedMap<Object>) temp; - String name = ((String) level.getVal(0)).toUpperCase(); - vBarGraphData.setName(name); + public QueryResponse process(SolrQuery solrQuery) { + return process(solrQuery, null); + } - Collection<NameValueData> vNameValues = new ArrayList<>(); - vBarGraphData.setDataCount(vNameValues); - ArrayList<Object> levelBuckets = (ArrayList<Object>) ((NamedList<Object>) level.get(innerField)).get("buckets"); - for (Object temp1 : levelBuckets) { - SimpleOrderedMap<Object> countValue = (SimpleOrderedMap<Object>) temp1; - String value = DateUtil.convertDateWithMillisecondsToSolrDate((Date) countValue.getVal(0)); + public QueryResponse process(SolrDataQuery solrDataQuery) { + return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery)); + } - String count = "" + countValue.getVal(1); - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(value); - vNameValue.setValue(count); - vNameValues.add(vNameValue); + public long count(final SolrDataQuery solrDataQuery) { + return getSolrTemplate().execute(new SolrCallback<Long>() { + @Override + public Long doInSolr(SolrClient solrClient) throws SolrServerException, IOException { + SolrQuery solrQuery = new DefaultQueryParser().doConstructSolrQuery(solrDataQuery); + solrQuery.setStart(0); + solrQuery.setRows(0); + QueryResponse queryResponse = solrClient.query(solrQuery); + long count = solrClient.query(solrQuery).getResults().getNumFound(); + LOG_PERFORMANCE.info("\n Username :- " + LogSearchContext.getCurrentUsername() + " Count SolrQuery :- " + + solrQuery + "\nQuery Time Execution :- " + queryResponse.getQTime() + " Total Time Elapsed is :- " + + queryResponse.getElapsedTime() + " Count result :- " + count); + return count; } - histogramData.add(vBarGraphData); - } + }); + } + + public QueryResponse process(SolrDataQuery solrDataQuery, String event) { + return process(new DefaultQueryParser().doConstructSolrQuery(solrDataQuery), event); } - public abstract CloudSolrClient getSolrClient(); + public CloudSolrClient getSolrClient() { + return (CloudSolrClient) getSolrTemplate().getSolrClient(); + } + + public abstract SolrTemplate getSolrTemplate(); + + public abstract SolrSchemaFieldDao getSolrSchemaFieldDao(); } http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java index b6764d0..f16dc41 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrSchemaFieldDao.java @@ -33,16 +33,12 @@ import org.codehaus.jettison.json.JSONArray; import org.codehaus.jettison.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.context.annotation.Scope; -import org.springframework.stereotype.Component; import javax.inject.Inject; import java.io.IOException; import java.util.HashMap; import java.util.Map; -@Component -@Scope(value = "prototype") public class SolrSchemaFieldDao { private static final Logger LOG = LoggerFactory.getLogger(SolrSchemaFieldDao.class); @@ -52,11 +48,13 @@ public class SolrSchemaFieldDao { private boolean populateFieldsThreadActive = false; + private Map<String, String> schemaFieldNameMap = new HashMap<>(); + private Map<String, String> schemaFieldTypeMap = new HashMap<>(); + @Inject private SolrUserPropsConfig solrUserPropsConfig; - public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig, - final SolrDaoBase solrDao) { + public void populateSchemaFields(final CloudSolrClient solrClient, final SolrPropsConfig solrPropsConfig) { if (!populateFieldsThreadActive) { populateFieldsThreadActive = true; LOG.info("Creating thread to populated fields for collection=" + solrPropsConfig.getCollection()); @@ -69,7 +67,7 @@ public class SolrSchemaFieldDao { try { Thread.sleep(SETUP_RETRY_SECOND * 1000); retryCount++; - boolean _result = _populateSchemaFields(solrClient, solrPropsConfig, solrDao); + boolean _result = _populateSchemaFields(solrClient, solrPropsConfig); if (_result) { LOG.info("Populate fields for collection " + solrPropsConfig.getCollection() + " is success, Update it after " + SETUP_UPDATE_SECOND + " sec"); @@ -94,7 +92,7 @@ public class SolrSchemaFieldDao { /** * Called from the thread. Don't call this directly */ - private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig, SolrDaoBase solrDao) { + private boolean _populateSchemaFields(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) { SolrRequest<SchemaResponse> request = new SchemaRequest(); request.setMethod(SolrRequest.METHOD.GET); request.setPath("/schema"); @@ -109,7 +107,7 @@ public class SolrSchemaFieldDao { } if (namedList != null) { - extractSchemaFieldsName(namedList.toString(), solrDao.schemaFieldNameMap, solrDao.schemaFieldTypeMap); + extractSchemaFieldsName(namedList.toString(), schemaFieldNameMap, schemaFieldTypeMap); return true; } } @@ -156,4 +154,12 @@ public class SolrSchemaFieldDao { LOG.error(e + "Credentials not specified in logsearch.properties " + MessageEnums.ERROR_SYSTEM); } } + + public Map<String, String> getSchemaFieldTypeMap() { + return schemaFieldTypeMap; + } + + public Map<String, String> getSchemaFieldNameMap() { + return schemaFieldNameMap; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java index 88c4577..d650bb7 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserConfigSolrDao.java @@ -28,14 +28,15 @@ import java.util.List; import java.util.Scanner; import javax.annotation.PostConstruct; import javax.inject.Inject; +import javax.inject.Named; import org.apache.ambari.logsearch.common.LogSearchContext; +import org.apache.ambari.logsearch.common.LogType; import org.apache.ambari.logsearch.conf.SolrUserPropsConfig; import org.apache.ambari.logsearch.view.VLogfeederFilterWrapper; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.common.SolrDocument; @@ -47,15 +48,12 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import com.google.gson.JsonParseException; -import org.apache.ambari.logsearch.manager.ManagerBase.LogType; import org.apache.ambari.logsearch.util.JSONUtil; import org.apache.log4j.Logger; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.solr.core.SolrTemplate; -import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; -@Component +@Named public class UserConfigSolrDao extends SolrDaoBase { private static final Logger LOG = Logger.getLogger(UserConfigSolrDao.class); @@ -69,10 +67,11 @@ public class UserConfigSolrDao extends SolrDaoBase { private SolrCollectionDao solrCollectionDao; @Inject + @Named("serviceSolrFieldDao") private SolrSchemaFieldDao solrSchemaFieldDao; @Inject - @Qualifier("userConfigSolrTemplate") + @Named("userConfigSolrTemplate") private SolrTemplate userConfigSolrTemplate; public UserConfigSolrDao() { @@ -80,11 +79,10 @@ public class UserConfigSolrDao extends SolrDaoBase { } @Override - public CloudSolrClient getSolrClient() { - return (CloudSolrClient) userConfigSolrTemplate.getSolrClient(); + public SolrTemplate getSolrTemplate() { + return userConfigSolrTemplate; } - @PostConstruct public void postConstructor() { String solrUrl = solrUserConfig.getSolrUrl(); @@ -94,7 +92,7 @@ public class UserConfigSolrDao extends SolrDaoBase { try { solrCollectionDao.checkSolrStatus(getSolrClient()); solrCollectionDao.setupCollections(getSolrClient(), solrUserConfig); - solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig, this); + solrSchemaFieldDao.populateSchemaFields(getSolrClient(), solrUserConfig); intializeLogFeederFilter(); } catch (Exception e) { @@ -201,6 +199,11 @@ public class UserConfigSolrDao extends SolrDaoBase { return logfeederFilterWrapper; } + @Override + public SolrSchemaFieldDao getSolrSchemaFieldDao() { + return solrSchemaFieldDao; + } + private String getHadoopServiceConfigJSON() { StringBuilder result = new StringBuilder(""); http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java index 0df2ca3..e6f5103 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/doc/DocConstants.java @@ -44,6 +44,7 @@ public class DocConstants { public static final String IS_LAST_PAGE_D = "Show last page (true/false)"; public static final String FIELD_D = "Get values for particular field"; public static final String FORMAT_D = "File Export format, can be 'txt' or 'json'"; + public static final String TOP = "Number that defines how many top element you would like to see."; } public class AuditOperationDescriptions { @@ -51,11 +52,7 @@ public class DocConstants { public static final String GET_AUDIT_LOGS_OD = "Get the list of logs details"; public static final String GET_AUDIT_COMPONENTS_OD = "Get the list of audit components currently active or having data in Solr"; public static final String GET_AUDIT_LINE_GRAPH_DATA_OD = "Get the data required for line graph"; - public static final String GET_TOP_AUDIT_USERS_OD = "Get the top audit users having maximum access"; - public static final String GET_TOP_AUDIT_RESOURCES_OD = "Get the top audit resources having maximum access"; - public static final String GET_LIVE_LOGS_COUNT_OD = "not required"; - public static final String GET_REQUEST_USER_LINE_GRAPH_OD = "not required"; - public static final String GET_ANY_GRAPH_DATA_OD = "Get the data generic enough to use for graph plots"; + public static final String GET_TOP_AUDIT_RESOURCES_OD = "Get the top audit resource count (grouped by type)"; public static final String EXPORT_USER_TALBE_TO_TEXT_FILE_OD = "Export the tables shown on Audit tab"; public static final String GET_SERVICE_LOAD_OD = "The graph for showing the top users accessing the services"; } @@ -64,7 +61,6 @@ public class DocConstants { public static final String LEVEL_D = "filter for log level"; public static final String BUNDLE_ID = "filter for host"; public static final String FILE_NAME_D = "File name filter which is supported from browser url"; - public static final String DATE_RANGE_LABEL_D = "Date range label (e.g.: Today)"; public static final String HOST_NAME_D = "Host name filter which is supported from browser url"; public static final String COMPONENT_NAME_D = "Component name filter which is supported from browser url"; public static final String FIND_D = "Finding particular text on subsequent pages in case of table view with pagination"; @@ -72,7 +68,6 @@ public class DocConstants { public static final String KEYWORD_TYPE_D = "Serching the find param value in previous or next in paginated table"; public static final String TOKEN_D = "unique number used along with FIND_D. The request can be canceled using this token"; public static final String SOURCE_LOG_ID_D = "fetch the record set having that log Id"; - public static final String G_MUST_NOT_D = "not required"; public static final String NUMBER_ROWS_D = "Getting rows after particular log entry - used in 'Preview' option"; public static final String SCROLL_TYPE_D = "Used in 'Preview' feature for getting records 'after' or 'before'"; public static final String UTC_OFFSET_D = "timezone offset"; @@ -86,18 +81,15 @@ public class DocConstants { public static final String GET_LOG_LEVELS_COUNT_OD = "Get Log levels with their counts"; public static final String GET_COMPONENTS_COUNT_OD = "Get components with their counts"; public static final String GET_HOSTS_COUNT_OD = "Get hosts with their counts"; - public static final String GET_TREE_EXTENSION_OD = "Get host and compoenets hierarchy"; + public static final String GET_TREE_EXTENSION_OD = "Get host and compoenets hierarchy with log counts"; public static final String GET_HISTOGRAM_DATA_OD = "Get data for histogram"; - public static final String CANCEL_FIND_REQUEST_OD = "Cancel the FIND_D param request using TOKEN_D"; public static final String EXPORT_TO_TEXT_FILE_OD = "Export the table data in file"; public static final String GET_COMPONENT_LIST_WITH_LEVEL_COUNT_OD = "Get components with log level distribution count"; - public static final String GET_EXTREME_DATES_FOR_BUNDLE_ID_OD = "Get the start and end time of particular bundle_id"; - public static final String GET_SERVICE_LOGS_FIELD_NAME_OD = "Get service logs schema fields name (Human readable)"; - public static final String GET_ANY_GRAPH_DATA_OD = "Get the data generic enough to use for graph plots"; - public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data"; + public static final String GET_ANY_GRAPH_COUNT_DATA_OD = "Get the data generic enough to use for graph plots (yAzis is always count)"; public static final String GET_HOST_LIST_BY_COMPONENT_OD = "Get host list of components"; public static final String GET_SERVICE_LOGS_SCHEMA_FIELD_NAME_OD = "Get service logs schema fields"; public static final String GET_HADOOP_SERVICE_CONFIG_JSON_OD = "Get the json having meta data of services supported by logsearch"; + public static final String GET_AFTER_BEFORE_LOGS_OD = "Preview feature data"; } public class PublicOperationDescriptions { http://git-wip-us.apache.org/repos/asf/ambari/blob/ea644cc4/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java index 287ff6c..85b5fda 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java @@ -19,324 +19,405 @@ package org.apache.ambari.logsearch.graph; -import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Map; -import org.apache.ambari.logsearch.common.LogSearchConstants; -import org.apache.ambari.logsearch.common.MessageEnums; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.ambari.logsearch.model.response.BarGraphData; import org.apache.ambari.logsearch.model.response.BarGraphDataListResponse; +import org.apache.ambari.logsearch.model.response.CountData; +import org.apache.ambari.logsearch.model.response.CountDataListResponse; +import org.apache.ambari.logsearch.model.response.GraphData; +import org.apache.ambari.logsearch.model.response.GraphDataListResponse; import org.apache.ambari.logsearch.model.response.NameValueData; -import org.apache.ambari.logsearch.query.model.SearchCriteria; -import org.apache.ambari.logsearch.dao.SolrDaoBase; -import org.apache.ambari.logsearch.query.QueryGeneration; -import org.apache.ambari.logsearch.util.RESTErrorUtil; -import org.apache.ambari.logsearch.util.SolrUtil; +import org.apache.ambari.logsearch.model.response.NameValueDataListResponse; +import org.apache.ambari.logsearch.model.response.NodeData; +import org.apache.ambari.logsearch.model.response.NodeListResponse; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.apache.log4j.Logger; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FacetField.Count; +import org.apache.solr.client.solrj.response.PivotField; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RangeFacet; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.SimpleOrderedMap; -import org.springframework.stereotype.Component; +import org.apache.solr.common.util.NamedList; -import javax.inject.Inject; +import javax.inject.Named; -@Component -public class GraphDataGenerator extends GraphDataGeneratorBase { +@Named +public class GraphDataGenerator { - private static final Logger logger = Logger.getLogger(GraphDataGenerator.class); + public BarGraphDataListResponse generateBarGraphDataResponseWithRanges(QueryResponse response, String typeField, boolean typeUppercase) { + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + if (response == null) { + return dataList; + } + NamedList<List<PivotField>> facetPivotResponse = response.getFacetPivot(); + if (response.getFacetPivot() == null) { + return dataList; + } + List<PivotField> pivotFields = facetPivotResponse.get(typeField); + for (int pivotIndex = 0; pivotIndex < pivotFields.size(); pivotIndex++) { + PivotField pivotField = facetPivotResponse.get(typeField).get(pivotIndex); + List<NameValueData> nameValues = generateNameValueDataList(pivotField.getFacetRanges()); + BarGraphData barGraphData = new BarGraphData(); + barGraphData.setDataCount(nameValues); + String typeValue = typeUppercase ? StringUtils.upperCase(pivotField.getValue().toString()) : pivotField.getValue().toString(); + barGraphData.setName(typeValue); + dataList.getGraphData().add(barGraphData); + } + return dataList; + } - @Inject - private QueryGeneration queryGenerator; + public BarGraphDataListResponse generateSecondLevelBarGraphDataResponse(QueryResponse response, int val) { + BarGraphDataListResponse barGraphDataListResponse = new BarGraphDataListResponse(); + NamedList<List<PivotField>> pivotFieldNameList = response.getFacetPivot(); + if (pivotFieldNameList == null) { + return barGraphDataListResponse; + } + List<PivotField> pivotFields = pivotFieldNameList.getVal(val); + List<BarGraphData> barGraphDataList = new ArrayList<>(); + for (PivotField pivotField : pivotFields) { + BarGraphData barGraphData = new BarGraphData(); + barGraphData.setName(String.valueOf(pivotField.getValue())); + List<PivotField> secondLevelPivotFields = pivotField.getPivot(); + List<NameValueData> nameValueDataList = new ArrayList<>(); + for (PivotField sPivotField : secondLevelPivotFields) { + NameValueData nvD = new NameValueData(); + nvD.setName(String.valueOf(sPivotField.getValue())); + nvD.setValue(String.valueOf(sPivotField.getCount())); + nameValueDataList.add(nvD); + } + barGraphData.setDataCount(nameValueDataList); + barGraphDataList.add(barGraphData); + } + barGraphDataListResponse.setGraphData(barGraphDataList); + return barGraphDataListResponse; + } - public BarGraphDataListResponse getAnyGraphData(SearchCriteria searchCriteria, SolrDaoBase solrDaoBase, SolrQuery solrQuery) { - // X axis credentials - String xAxisField = (String) searchCriteria.getParamValue("xAxis"); - String stackField = (String) searchCriteria.getParamValue("stackBy"); - String from = (String) searchCriteria.getParamValue("from"); - String to = (String) searchCriteria.getParamValue("to"); - String unit = (String) searchCriteria.getParamValue("unit"); - String typeXAxis = solrDaoBase.schemaFieldNameMap.get(xAxisField); - typeXAxis = (StringUtils.isBlank(typeXAxis)) ? "string" : typeXAxis; + public BarGraphDataListResponse generateBarGraphFromFieldFacet(QueryResponse response, String facetField) { + BarGraphDataListResponse dataList = new BarGraphDataListResponse(); + Collection<BarGraphData> vaDatas = new ArrayList<>(); + dataList.setGraphData(vaDatas); + if (response == null) { + return dataList; + } + FacetField facetFieldObj = response.getFacetField(facetField); + if (facetFieldObj == null) { + return dataList; + } + + List<Count> counts = facetFieldObj.getValues(); + if (counts == null) { + return dataList; + } + for (Count cnt : counts) { + List<NameValueData> valueList = new ArrayList<>(); + BarGraphData vBarGraphData = new BarGraphData(); + vaDatas.add(vBarGraphData); + NameValueData vNameValue = new NameValueData(); + vNameValue.setName(cnt.getName()); + vBarGraphData.setName(cnt.getName().toUpperCase()); + vNameValue.setValue("" + cnt.getCount()); + valueList.add(vNameValue); + vBarGraphData.setDataCount(valueList); + } + return dataList; + } + + public List<NameValueData> generateNameValueDataList(List<RangeFacet> rangeFacet) { + List<NameValueData> nameValues = new ArrayList<>(); + if (rangeFacet == null) { + return nameValues; + } + RangeFacet range = rangeFacet.get(0); - // Y axis credentials - String yAxisField = (String) searchCriteria.getParamValue("yAxis"); - // add updated typeXAxis as a type parameter - searchCriteria.addParam("type", typeXAxis); - String fieldTime = (String) searchCriteria.getParamValue("fieldTime"); - // decide graph type based on user request parameter - GraphType garphType = getGraphType(searchCriteria); - switch (garphType) { - case NORMAL_GRAPH: - return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery); - case RANGE_NON_STACK_GRAPH: - return rangeNonStackGraph(xAxisField, yAxisField, from, to, unit, solrDaoBase, typeXAxis, fieldTime, solrQuery); - case NON_RANGE_STACK_GRAPH: - return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery); - case RANGE_STACK_GRAPH: - return rangeStackGraph(xAxisField, stackField, from, to, unit, solrDaoBase, solrQuery); - default: - logger.warn("Invalid graph type :" + garphType.name()); - return null; + if (range == null) { + return nameValues; + } + List<RangeFacet.Count> listCount = range.getCounts(); + for (RangeFacet.Count cnt : listCount) { + NameValueData nameValue = new NameValueData(); + nameValue.setName(String.valueOf(cnt.getValue())); + nameValue.setValue(String.valueOf(cnt.getCount())); + nameValues.add(nameValue); } + return nameValues; } - private GraphType getGraphType(SearchCriteria searchCriteria) { - // default graph type is unknown - GraphType graphType = GraphType.UNKNOWN; - // X axis credentials - String xAxisField = (String) searchCriteria.getParamValue("xAxis"); - String stackField = (String) searchCriteria.getParamValue("stackBy"); - String from = (String) searchCriteria.getParamValue("from"); - String to = (String) searchCriteria.getParamValue("to"); - String xType = (String) searchCriteria.getParamValue("type"); - if (xType != null) { - // Y axis credentials - String yAxisField = (String) searchCriteria.getParamValue("yAxis"); - if (StringUtils.isBlank(xAxisField) || StringUtils.isBlank(yAxisField)) { - graphType = GraphType.UNKNOWN; - } else if (StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from) - && !(xType.contains("date") || xType.contains("time"))) { - graphType = GraphType.NORMAL_GRAPH; - } else if (StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from) - && (xType.contains("date") || xType.contains("time"))) { - graphType = GraphType.RANGE_NON_STACK_GRAPH; - } else if (!StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from) - && !(xType.contains("date") || xType.contains("time"))) { - graphType = GraphType.NON_RANGE_STACK_GRAPH; - } else if (!StringUtils.isBlank(stackField) && !StringUtils.isBlank(to) && !StringUtils.isBlank(from) - && (xType.contains("date") || xType.contains("time"))) { - graphType = GraphType.RANGE_STACK_GRAPH; + public List<Count> generateCount(QueryResponse response) { + List<Count> counts = new ArrayList<>(); + List<FacetField> facetFields = null; + FacetField facetField = null; + if (response == null) { + return counts; + } + + facetFields = response.getFacetFields(); + if (facetFields == null) { + return counts; + } + if (!facetFields.isEmpty()) { + facetField = facetFields.get(0); + } + if (facetField != null) { + counts = facetField.getValues(); + } + return counts; + } + + public BarGraphDataListResponse getGraphDataWithDefaults(QueryResponse queryResponse, String field, String[] defaults) { + BarGraphDataListResponse response = new BarGraphDataListResponse(); + BarGraphData barGraphData = new BarGraphData(); + List<NameValueData> nameValues = generateLevelCountData(queryResponse, defaults); + barGraphData.setName(field); + barGraphData.setDataCount(nameValues); + response.setGraphData(Lists.newArrayList(barGraphData)); + return response; + } + + public NameValueDataListResponse getNameValueDataListResponseWithDefaults(QueryResponse response, String[] defaults) { + NameValueDataListResponse result = new NameValueDataListResponse(); + result.setvNameValues(generateLevelCountData(response, defaults)); + return result; + } + + public NodeListResponse generateServiceNodeTreeFromFacetResponse(QueryResponse queryResponse, + String firstHierarchy, String secondHierarchy, + String firstType, String secondType) { + NodeListResponse response = new NodeListResponse(); + if (queryResponse == null) { + return response; + } + NamedList<List<PivotField>> namedPivotFieldList = queryResponse.getFacetPivot(); + List<PivotField> firstLevelPivots = namedPivotFieldList.get(firstHierarchy); + List<PivotField> secondLevelPivots = namedPivotFieldList.get(secondHierarchy); + if (!CollectionUtils.isNotEmpty(firstLevelPivots) || !CollectionUtils.isNotEmpty(secondLevelPivots)) { + return response; + } + List<NodeData> nodeDataList = buidTreeData(firstLevelPivots, secondLevelPivots, firstType, secondType); + response.setvNodeList(nodeDataList); + return response; + } + + public NodeListResponse generateOneLevelServiceNodeTree(QueryResponse queryResponse, String componentLevelHirachy) { + NodeListResponse response = new NodeListResponse(); + List<NodeData> datatList = new ArrayList<>(); + List<List<PivotField>> listPivotField = new ArrayList<>(); + NamedList<List<PivotField>> namedList = queryResponse.getFacetPivot(); + if (namedList != null) { + listPivotField = namedList.getAll(componentLevelHirachy); + } + List<PivotField> secondHirarchicalPivotFields = null; + if (listPivotField == null || listPivotField.isEmpty()) { + return response; + } else { + secondHirarchicalPivotFields = listPivotField.get(0); + } + for (PivotField singlePivotField : secondHirarchicalPivotFields) { + if (singlePivotField != null) { + NodeData comp = new NodeData(); + comp.setName("" + singlePivotField.getValue()); + List<PivotField> levelList = singlePivotField.getPivot(); + List<NameValueData> levelCountList = new ArrayList<>(); + comp.setLogLevelCount(levelCountList); + if (levelList != null) { + for (PivotField levelPivot : levelList) { + NameValueData level = new NameValueData(); + level.setName(("" + levelPivot.getValue()).toUpperCase()); + level.setValue("" + levelPivot.getCount()); + levelCountList.add(level); + } + } + datatList.add(comp); } } - return graphType; + response.setvNodeList(datatList); + return response; } - @SuppressWarnings("unchecked") - private BarGraphDataListResponse normalGraph(String xAxisField, String yAxisField, String from, String to, SolrDaoBase solrDaoBase, - String typeXAxis, String fieldTime, SolrQuery solrQuery) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - Collection<BarGraphData> vBarGraphDatas = new ArrayList<BarGraphData>(); - BarGraphData vBarGraphData = new BarGraphData(); - Collection<NameValueData> vNameValues = new ArrayList<NameValueData>(); - SolrUtil.setMainQuery(solrQuery, null); - queryGenerator.setSingleIncludeFilter(solrQuery, fieldTime, "[" + from + " TO " + to + "]"); - if (typeXAxis.contains("string") || typeXAxis.contains("key_lower_case") || typeXAxis.contains("text")) { - SolrUtil.setFacetField(solrQuery, xAxisField); - try { - QueryResponse response = solrDaoBase.process(solrQuery); - if (response != null && response.getResults() != null) { - long count = response.getResults().getNumFound(); - if (count > 0) { - FacetField facetField = response.getFacetField(xAxisField); - if (facetField != null) { - List<Count> countValues = facetField.getValues(); - if (countValues != null) { - for (Count countValue : countValues) { - if (countValue != null) { - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(countValue.getName()); - vNameValue.setValue("" + countValue.getCount()); - vNameValues.add(vNameValue); - } - } + private List<NodeData> buidTreeData(List<PivotField> firstHirarchicalPivotFields, + List<PivotField> secondHirarchicalPivotFields, + String firstPriority, String secondPriority) { + List<NodeData> extensionTree = new ArrayList<>(); + if (firstHirarchicalPivotFields != null) { + for (PivotField pivotHost : firstHirarchicalPivotFields) { + if (pivotHost != null) { + NodeData hostNode = new NodeData(); + String name = (pivotHost.getValue() == null ? "" : "" + pivotHost.getValue()); + String value = "" + pivotHost.getCount(); + if (!StringUtils.isBlank(name)) { + hostNode.setName(name); + } + if (!StringUtils.isBlank(value)) { + hostNode.setValue(value); + } + if (!StringUtils.isBlank(firstPriority)) { + hostNode.setType(firstPriority); + } + + hostNode.setParent(true); + hostNode.setRoot(true); + PivotField hostPivot = null; + for (PivotField searchHost : secondHirarchicalPivotFields) { + if (!StringUtils.isBlank(hostNode.getName()) + && hostNode.getName().equals(searchHost.getValue())) { + hostPivot = searchHost; + break; + } + } + List<PivotField> pivotLevelHost = hostPivot == null? null : hostPivot.getPivot(); + if (pivotLevelHost != null) { + Collection<NameValueData> logLevelCount = new ArrayList<>(); + for (PivotField pivotLevel : pivotLevelHost) { + if (pivotLevel != null) { + NameValueData vnameValue = new NameValueData(); + String levelName = (pivotLevel.getValue() == null ? "" : "" + pivotLevel.getValue()); + vnameValue.setName(levelName.toUpperCase()); + vnameValue.setValue("" + pivotLevel.getCount()); + logLevelCount.add(vnameValue); } - vBarGraphData.setName(xAxisField); - vBarGraphDatas.add(vBarGraphData); - dataList.setGraphData(vBarGraphDatas); } + hostNode.setLogLevelCount(logLevelCount); } - } - if (xAxisField.equalsIgnoreCase(LogSearchConstants.SOLR_LEVEL)) { - Collection<NameValueData> sortedVNameValues = new ArrayList<NameValueData>(); - for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) { - NameValueData value = new NameValueData(); - value.setName(level); - String val = "0"; - for (NameValueData valueLevel : vNameValues) { - if (valueLevel.getName().equalsIgnoreCase(level)) { - val = valueLevel.getValue(); - break; + List<PivotField> pivotComponents = pivotHost.getPivot(); + if (pivotComponents != null) { + Collection<NodeData> componentNodes = new ArrayList<>(); + for (PivotField pivotComp : pivotComponents) { + if (pivotComp != null) { + NodeData compNode = new NodeData(); + String compName = (pivotComp.getValue() == null ? "" : "" + pivotComp.getValue()); + compNode.setName(compName); + if (!StringUtils.isBlank(secondPriority)) { + compNode.setType(secondPriority); + } + compNode.setValue("" + pivotComp.getCount()); + compNode.setParent(false); + compNode.setRoot(false); + List<PivotField> pivotLevels = pivotComp.getPivot(); + if (pivotLevels != null) { + Collection<NameValueData> logLevelCount = new ArrayList<>(); + for (PivotField pivotLevel : pivotLevels) { + if (pivotLevel != null) { + NameValueData vnameValue = new NameValueData(); + String compLevel = pivotLevel.getValue() == null ? "" : "" + pivotLevel.getValue(); + vnameValue.setName((compLevel).toUpperCase()); + + vnameValue.setValue("" + pivotLevel.getCount()); + logLevelCount.add(vnameValue); + } + } + compNode.setLogLevelCount(logLevelCount); + } + componentNodes.add(compNode); } } - value.setValue(val); - sortedVNameValues.add(value); + hostNode.setChilds(componentNodes); } - vBarGraphData.setDataCount(sortedVNameValues); - } else { - vBarGraphData.setDataCount(vNameValues); - } - return dataList; - } catch (SolrException | SolrServerException | IOException e) { - String query = solrQuery != null ? solrQuery.toQueryString() : ""; - logger.error("Got exception for solr query :" + query, e.getCause()); - } - } else { - SolrUtil.setRowCount(solrQuery, 0); - String yAxis = yAxisField.contains("count") ? "sum" : yAxisField; - String jsonQuery = queryGenerator.buildJSONFacetAggregatedFuncitonQuery(yAxis, xAxisField); - SolrUtil.setJSONFacet(solrQuery, jsonQuery); - try { - QueryResponse response = solrDaoBase.process(solrQuery); - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets"); - if (jsonFacetResponse.toString().equals("{count=0}")) { - return dataList; + extensionTree.add(hostNode); } - NameValueData value = new NameValueData(); - String sum = (String) jsonFacetResponse.getVal(1); - value.setName(xAxisField); - value.setValue(sum != null ? sum.substring(0, sum.indexOf(".")) : ""); - vNameValues.add(value); - vBarGraphData.setDataCount(vNameValues); - vBarGraphData.setName(xAxisField); - vBarGraphDatas.add(vBarGraphData); - dataList.setGraphData(vBarGraphDatas); - return dataList; - } catch (SolrException | SolrServerException | IOException e) { - String query = solrQuery != null ? solrQuery.toQueryString() : ""; - logger.error("Got exception for solr query :" + query, e.getCause()); } } - return null; + + return extensionTree; } - @SuppressWarnings("unchecked") - private BarGraphDataListResponse nonRangeStackGraph(String xAxisField, String yAxisField, String stackField, String from, String to, - SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - Collection<BarGraphData> vGraphData = new ArrayList<BarGraphData>(); - String mainQuery = queryGenerator.buildInclusiveRangeFilterQuery(fieldTime, from, to); - SolrUtil.setMainQuery(solrQuery, mainQuery); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - String jsonQuery = ""; - if (SolrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) { - String function = (yAxisField.contains("count")) ? "sum" : yAxisField; - jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(stackField, xAxisField, function); - } else { - jsonQuery = queryGenerator.buildJsonFacetTermsRangeQuery(stackField, xAxisField); - } - try { - SolrUtil.setJSONFacet(solrQuery, jsonQuery); - dataList.setGraphData(vGraphData); - QueryResponse response = solrDaoBase.process(solrQuery); - if (response == null) { - response = new QueryResponse(); - } - Long count = response.getResults().getNumFound(); - if (count <= 0) { - return dataList; - } - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets"); - if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { - return dataList; - } - extractNonRangeStackValuesFromBucket(jsonFacetResponse, stackField, vGraphData, typeXAxis); - if (LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(stackField) && LogSearchConstants.SOLR_LEVEL.equalsIgnoreCase(xAxisField)) { - Collection<BarGraphData> levelVGraphData = dataList.getGraphData(); - for (BarGraphData graphData : levelVGraphData) { - Collection<NameValueData> valueList = graphData.getDataCount(); - Collection<NameValueData> valueListSorted = new ArrayList<NameValueData>(); - for (String level : LogSearchConstants.SUPPORTED_LOG_LEVEL) { - String val = "0"; - for (NameValueData value : valueList) { - if (value.getName().equalsIgnoreCase(level)) { - val = value.getValue(); - break; - } - } - NameValueData v1 = new NameValueData(); - v1.setName(level.toUpperCase()); - v1.setValue(val); - valueListSorted.add(v1); - } - graphData.setDataCount(valueListSorted); - } + private List<NameValueData> generateLevelCountData(QueryResponse queryResponse, String[] defaults) { + List<NameValueData> nameValues = Lists.newLinkedList(); + Map<String, NameValueData> linkedMap = Maps.newLinkedHashMap(); + List<Count> counts = generateCount(queryResponse); + if (!CollectionUtils.isNotEmpty(counts)) { + return nameValues; + } + for (String defaultValue : defaults) { + NameValueData nameValue = new NameValueData(); + nameValue.setName(defaultValue); + nameValue.setValue("0"); + linkedMap.put(defaultValue, nameValue); + } + for (Count count : counts) { + if (!linkedMap.containsKey(count.getName())) { + NameValueData nameValue = new NameValueData(); + String name = count.getName().toUpperCase(); + nameValue.setName(name); + nameValue.setValue(String.valueOf(count.getCount())); + linkedMap.put(name, nameValue); } - return dataList; - } catch (SolrException | IOException | SolrServerException e) { - String query = solrQuery != null ? solrQuery.toQueryString() : ""; - logger.error("Got exception for solr query :" + query, e.getCause()); - throw RESTErrorUtil.createRESTException(MessageEnums.DATA_NOT_FOUND.getMessage().getMessage(), MessageEnums.DATA_NOT_FOUND); } + + for (Map.Entry<String, NameValueData> nameValueDataEntry : linkedMap.entrySet()) { + nameValues.add(nameValueDataEntry.getValue()); + } + return nameValues; } - @SuppressWarnings("unchecked") - private BarGraphDataListResponse rangeNonStackGraph(String xAxisField, String yAxisField, String from, String to, String unit, - SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, SolrQuery solrQuery) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - Collection<BarGraphData> vBarGraphDatas = new ArrayList<BarGraphData>(); - BarGraphData vBarGraphData = new BarGraphData(); - Collection<NameValueData> vNameValues = new ArrayList<NameValueData>(); - SolrUtil.setMainQuery(solrQuery, null); - if (SolrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) { - queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to); - return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery); - } else { - try { - SolrUtil.setFacetRange(solrQuery, xAxisField, from, to, unit); - QueryResponse response = solrDaoBase.process(solrQuery); - if (response != null) { - Long count = response.getResults().getNumFound(); - if (count > 0) { - @SuppressWarnings("rawtypes") - List<RangeFacet> rangeFacet = response.getFacetRanges(); - if (rangeFacet != null && rangeFacet.size() > 0) { - List<RangeFacet.Count> listCount = rangeFacet.get(0).getCounts(); - if (listCount != null) { - for (RangeFacet.Count cnt : listCount) { - NameValueData vNameValue = new NameValueData(); - vNameValue.setName(cnt.getValue()); - vNameValue.setValue("" + cnt.getCount()); - vNameValues.add(vNameValue); - } - vBarGraphData.setDataCount(vNameValues); - vBarGraphDatas.add(vBarGraphData); - vBarGraphData.setName(xAxisField); - dataList.setGraphData(vBarGraphDatas); - } - } - } - } - return dataList; - } catch (SolrException | SolrServerException | IOException e) { - logger.error("Got exception for solr query :" + solrQuery, e.getCause()); + public CountDataListResponse generateCountResponseByField(QueryResponse response, String field) { + CountDataListResponse collection = new CountDataListResponse(); + List<CountData> vCounts = new ArrayList<>(); + if (response == null) { + return collection; + } + FacetField facetFields = response.getFacetField(field); + if (facetFields == null) { + return collection; + } + List<Count> fieldList = facetFields.getValues(); + + if (fieldList == null) { + return collection; + } + + for (Count cnt : fieldList) { + if (cnt != null) { + CountData vCount = new CountData(); + vCount.setName(cnt.getName()); + vCount.setCount(cnt.getCount()); + vCounts.add(vCount); } } - return null; + collection.setvCounts(vCounts); + return collection; } - @SuppressWarnings("unchecked") - private BarGraphDataListResponse rangeStackGraph(String xAxisField, String stackField, String from, String to, String unit, - SolrDaoBase solrDaoBase, SolrQuery solrQuery) { - BarGraphDataListResponse dataList = new BarGraphDataListResponse(); - List<BarGraphData> histogramData = new ArrayList<BarGraphData>(); - SolrUtil.setMainQuery(solrQuery, null); - SolrUtil.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); - String jsonHistogramQuery = - queryGenerator.buildJSONFacetTermTimeRangeQuery(stackField, xAxisField, from, to, unit).replace("\\", ""); - try { - solrQuery.set("json.facet", jsonHistogramQuery); - SolrUtil.setRowCount(solrQuery, 0); - QueryResponse response = solrDaoBase.process(solrQuery); - if (response != null) { - SimpleOrderedMap<Object> jsonFacetResponse = (SimpleOrderedMap<Object>) response.getResponse().get("facets"); - if (jsonFacetResponse == null || jsonFacetResponse.toString().equals("{count=0}")) { - return dataList; + public GraphDataListResponse generateSimpleGraphResponse(QueryResponse response, String hierarchy) { + GraphDataListResponse graphInfo = new GraphDataListResponse(); + if (response == null) { + return graphInfo; + } + List<List<PivotField>> hirarchicalPivotField = new ArrayList<List<PivotField>>(); + List<GraphData> dataList = new ArrayList<>(); + NamedList<List<PivotField>> namedList = response.getFacetPivot(); + if (namedList != null) { + hirarchicalPivotField = namedList.getAll(hierarchy); + } + if (!hirarchicalPivotField.isEmpty()) { + dataList = buidGraphData(hirarchicalPivotField.get(0)); + } + if (!dataList.isEmpty()) { + graphInfo.setGraphData(dataList); + } + + return graphInfo; + } + + private List<GraphData> buidGraphData(List<PivotField> pivotFields) { + List<GraphData> logList = new ArrayList<>(); + if (pivotFields != null) { + for (PivotField pivotField : pivotFields) { + if (pivotField != null) { + GraphData logLevel = new GraphData(); + logLevel.setName("" + pivotField.getValue()); + logLevel.setCount(Long.valueOf(pivotField.getCount())); + if (pivotField.getPivot() != null) { + logLevel.setDataList(buidGraphData(pivotField.getPivot())); + } + logList.add(logLevel); } - extractRangeStackValuesFromBucket(jsonFacetResponse, "x", "y", histogramData); - dataList.setGraphData(histogramData); } - return dataList; - } catch (SolrException | IOException | SolrServerException e) { - logger.error("Got exception for solr query :" + solrQuery, e.getCause()); } - return null; + return logList; } }
