AMBARI-18225. Fixed search issue for number field in Logsearch portal (Hayat Behlim via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5bfcf70e Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5bfcf70e Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5bfcf70e Branch: refs/heads/logsearch-ga Commit: 5bfcf70ece374777d21b4a81f7d2d418b8f73e2c Parents: 858332f Author: oleewere <[email protected]> Authored: Mon Aug 22 15:30:47 2016 +0200 Committer: oleewere <[email protected]> Committed: Tue Sep 13 12:19:23 2016 +0200 ---------------------------------------------------------------------- .../ambari-logsearch-portal/pom.xml | 21 ++++ .../logsearch/common/LogSearchConstants.java | 2 - .../ambari/logsearch/dao/SolrDaoBase.java | 44 ++++---- .../apache/ambari/logsearch/dao/UserDao.java | 3 + .../logsearch/graph/GraphDataGenerator.java | 22 ++-- .../ambari/logsearch/manager/AuditMgr.java | 6 +- .../ambari/logsearch/manager/LogsMgr.java | 5 +- .../ambari/logsearch/query/QueryGeneration.java | 103 ++++++++++++++----- .../logsearch/query/QueryGenerationBase.java | 11 ++ .../ambari/logsearch/util/ConfigUtil.java | 70 +++++++------ .../apache/ambari/logsearch/util/SolrUtil.java | 54 ++++++++++ 11 files changed, 243 insertions(+), 98 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index 05dc406..106c9b0 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -535,6 +535,27 @@ <artifactId>solr-solrj</artifactId> <version>${solr.version}</version> </dependency> + <dependency> + <groupId>org.apache.solr</groupId> + <artifactId>solr-core</artifactId> + <version>${solr.version}</version> + <exclusions> + <exclusion> + <groupId>*</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-core</artifactId> + <version>${solr.version}</version> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>${solr.version}</version> + </dependency> <!-- Hadoop --> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java index 43a7eb7..5dcdac1 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/common/LogSearchConstants.java @@ -91,8 +91,6 @@ public class LogSearchConstants { public static final String UI_SUFFIX = "@UI@"; public static final String SOLR_SUFFIX = "@Solr@"; public static final String NGRAM_SUFFIX = "ngram_"; - public static final String DEFAULT_SERVICE_COLUMN_SUFFIX = "service"; - public static final String DEFAULT_AUDIT_COLUMN_SUFFIX = "audit"; //Date Format for SOLR public static final String SOLR_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss,SSS"; http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java index 91c4a26..cda5e26 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/SolrDaoBase.java @@ -22,10 +22,10 @@ package org.apache.ambari.logsearch.dao; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; - import org.apache.ambari.logsearch.common.LogsearchContextUtil; import org.apache.ambari.logsearch.common.MessageEnums; import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE; @@ -60,7 +60,10 @@ import org.springframework.beans.factory.annotation.Autowired; public abstract class SolrDaoBase { static private Logger logger = Logger.getLogger(SolrDaoBase.class); - + + public HashMap<String, String> schemaFieldsNameMap = new HashMap<String, String>(); + public HashMap<String, String> schemaFieldTypeMap = new HashMap<String, String>(); + private static Logger logPerformance = Logger .getLogger("org.apache.ambari.logsearch.performance"); @@ -95,6 +98,7 @@ public abstract class SolrDaoBase { private boolean populateFieldsThreadActive = false; int SETUP_RETRY_SECOND = 30; + int SETUP_UPDATE_SECOND = 10*60; //10 min int ALIAS_SETUP_RETRY_SECOND = 30*60; //30 minutes private boolean isZkConnectString=false;//by default its false @@ -599,17 +603,16 @@ public abstract class SolrDaoBase { } private void populateSchemaFields() { - boolean result = _populateSchemaFields(); - if (!result && !populateFieldsThreadActive) { + if (!populateFieldsThreadActive) { populateFieldsThreadActive = true; logger.info("Creating thread to populated fields for collection=" - + collectionName); + + collectionName); Thread fieldPopulationThread = new Thread("populated_fields_" - + collectionName) { + + collectionName) { @Override public void run() { logger.info("Started thread to get fields for collection=" - + collectionName); + + collectionName); int retryCount = 0; while (true) { try { @@ -617,27 +620,25 @@ public abstract class SolrDaoBase { retryCount++; boolean _result = _populateSchemaFields(); if (_result) { - logger.info("Populate fields for collection " - + collectionName + " is success"); - break; + logger.info("Populate fields for collection " + collectionName + + " is success, Update it after " + SETUP_UPDATE_SECOND + + " sec"); + Thread.sleep(SETUP_UPDATE_SECOND * 1000); } } catch (InterruptedException sleepInterrupted) { - logger.info("Sleep interrupted while populating fields for collection " - + collectionName); + logger + .info("Sleep interrupted while populating fields for collection " + + collectionName); break; } catch (Exception ex) { logger.error("Error while populating fields for collection " - + collectionName - + ", retryCount=" - + retryCount); - } finally { - populateFieldsThreadActive = false; + + collectionName + ", retryCount=" + retryCount); } } + populateFieldsThreadActive = false; logger.info("Exiting thread for populating fields. collection=" - + collectionName); + + collectionName); } - }; fieldPopulationThread.setDaemon(true); fieldPopulationThread.start(); @@ -650,7 +651,7 @@ public abstract class SolrDaoBase { private boolean _populateSchemaFields() { SolrRequest<SchemaResponse> request = new SchemaRequest(); request.setMethod(METHOD.GET); - request.setPath("/schema/fields"); + request.setPath("/schema"); String historyCollection = PropertiesUtil.getProperty("logsearch.solr.collection.history","history"); if (solrClient != null && !collectionName.equals(historyCollection)) { NamedList<Object> namedList = null; @@ -659,14 +660,13 @@ public abstract class SolrDaoBase { logger.info("populateSchemaFields() collection=" + collectionName + ", fields=" + namedList); } catch (SolrException | SolrServerException | IOException e) { - logger.error( "Error occured while popuplating field. collection=" + collectionName, e); } if (namedList != null) { ConfigUtil.extractSchemaFieldsName(namedList.toString(), - collectionName); + schemaFieldsNameMap,schemaFieldTypeMap); return true; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java index 091c40b..6b2f049 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/dao/UserDao.java @@ -192,6 +192,9 @@ public class UserDao { * @return */ public String encryptPassword(String username, String password) { + if (!stringUtil.isEmpty(username)) { + username = username.toLowerCase(); + } String saltEncodedpasswd = md5Encoder .encodePassword(password, username); return saltEncodedpasswd; http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java index 361f8e3..d3975b3 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/graph/GraphDataGenerator.java @@ -32,6 +32,7 @@ import org.apache.ambari.logsearch.query.QueryGeneration; import org.apache.ambari.logsearch.util.ConfigUtil; import org.apache.ambari.logsearch.util.DateUtil; import org.apache.ambari.logsearch.util.RESTErrorUtil; +import org.apache.ambari.logsearch.util.SolrUtil; import org.apache.ambari.logsearch.util.StringUtil; import org.apache.ambari.logsearch.view.VBarDataList; import org.apache.ambari.logsearch.view.VBarGraphData; @@ -45,9 +46,12 @@ import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RangeFacet; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.TextField; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; + + @Component public class GraphDataGenerator extends GraphDataGeneratorBase { @@ -62,6 +66,9 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { @Autowired DateUtil dateUtil; + + @Autowired + SolrUtil solrUtil; private static Logger logger = Logger.getLogger(GraphDataGenerator.class); @@ -80,8 +87,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { String from = (String) searchCriteria.getParamValue("from"); String to = (String) searchCriteria.getParamValue("to"); String unit = (String) searchCriteria.getParamValue("unit"); - String suffix = (String) searchCriteria.getParamValue("suffix"); - String typeXAxis = ConfigUtil.schemaFieldsName.get(xAxisField + suffix); + String typeXAxis = solrDaoBase.schemaFieldsNameMap.get(xAxisField); typeXAxis = (stringUtil.isEmpty(typeXAxis)) ? "string" : typeXAxis; // Y axis credentials @@ -102,8 +108,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { return nonRangeStackGraph(xAxisField, yAxisField, stackField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery); case RANGE_STACK_GRAPH: - return rangeStackGraph(xAxisField, yAxisField, stackField, from, to, - unit, solrDaoBase, typeXAxis, fieldTime, solrQuery); + return rangeStackGraph(xAxisField, stackField, from, to, unit, solrDaoBase, solrQuery); default: logger.warn("Invalid graph type :" + garphType.name()); return null; @@ -255,7 +260,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { queryGenerator.setMainQuery(solrQuery, mainQuery); queryGenerator.setFacetSort(solrQuery, LogSearchConstants.FACET_INDEX); String jsonQuery = ""; - if (isTypeNumber(typeXAxis)) { + if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) { String function = (yAxisField.contains("count")) ? "sum" : yAxisField; jsonQuery = queryGenerator.buidlJSONFacetRangeQueryForNumber(stackField, xAxisField, function); @@ -323,7 +328,7 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { VBarGraphData vBarGraphData = new VBarGraphData(); Collection<VNameValue> vNameValues = new ArrayList<VNameValue>(); queryGenerator.setMainQuery(solrQuery, null); - if (isTypeNumber(typeXAxis)) { + if (solrUtil.isSolrFieldNumber(typeXAxis,solrDaoBase)) { queryGenerator.setSingleRangeFilter(solrQuery, fieldTime, from, to); return normalGraph(xAxisField, yAxisField, from, to, solrDaoBase, typeXAxis, fieldTime, solrQuery); @@ -363,9 +368,8 @@ public class GraphDataGenerator extends GraphDataGeneratorBase { } @SuppressWarnings("unchecked") - private VBarDataList rangeStackGraph(String xAxisField, String yAxisField, - String stackField, String from, String to, String unit, - SolrDaoBase solrDaoBase, String typeXAxis, String fieldTime, + private VBarDataList rangeStackGraph(String xAxisField, String stackField, + String from, String to, String unit, SolrDaoBase solrDaoBase, SolrQuery solrQuery) { VBarDataList dataList = new VBarDataList(); List<VBarGraphData> histogramData = new ArrayList<VBarGraphData>(); http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java index 58c3a4d..ab287bc 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/AuditMgr.java @@ -468,12 +468,11 @@ public class AuditMgr extends MgrBase { } public String getAuditLogsSchemaFieldsName() { - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX); String excludeArray[] = PropertiesUtil .getPropertyStringList("logsearch.solr.audit.logs.exclude.columnlist"); List<String> fieldNames = new ArrayList<String>(); HashMap<String, String> uiFieldColumnMapping = new HashMap<String, String>(); - ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames); + ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,auditSolrDao); for (String fieldName : fieldNames) { String uiField = ConfigUtil.auditLogsColumnMapping.get(fieldName @@ -492,15 +491,12 @@ public class AuditMgr extends MgrBase { public String getAnyGraphData(SearchCriteria searchCriteria) { searchCriteria.addParam("fieldTime", LogSearchConstants.AUDIT_EVTTIME); - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX); - searchCriteria.addParam("suffix", suffix); SolrQuery solrQuery = queryGenerator.commonAuditFilterQuery(searchCriteria); VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria, auditSolrDao, solrQuery); if (result == null) { result = new VBarDataList(); } - return convertObjToString(result); } http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java index 748d2f9..257f04d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/manager/LogsMgr.java @@ -1626,12 +1626,11 @@ public class LogsMgr extends MgrBase { public String getServiceLogsSchemaFieldsName() { List<String> fieldNames = new ArrayList<String>(); - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX); String excludeArray[] = PropertiesUtil .getPropertyStringList("logsearch.solr.service.logs.exclude.columnlist"); HashMap<String, String> uiFieldColumnMapping = new LinkedHashMap<String, String>(); - ConfigUtil.getSchemaFieldsName(suffix, excludeArray, fieldNames); + ConfigUtil.getSchemaFieldsName(excludeArray, fieldNames,serviceLogsSolrDao); for (String fieldName : fieldNames) { String uiField = ConfigUtil.serviceLogsColumnMapping.get(fieldName @@ -1695,8 +1694,6 @@ public class LogsMgr extends MgrBase { public String getAnyGraphData(SearchCriteria searchCriteria) { searchCriteria.addParam("fieldTime", LogSearchConstants.LOGTIME); - String suffix = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX); - searchCriteria.addParam("suffix", suffix); SolrQuery solrQuery = queryGenerator.commonServiceFilterQuery(searchCriteria); VBarDataList result = graphDataGenerator.getAnyGraphData(searchCriteria, serviceLogsSolrDao, solrQuery); http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java index 646abd6..d934e84 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGeneration.java @@ -27,11 +27,19 @@ import java.util.regex.Pattern; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.SearchCriteria; +import org.apache.ambari.logsearch.dao.SolrDaoBase; import org.apache.ambari.logsearch.manager.MgrBase.LOG_TYPE; import org.apache.ambari.logsearch.util.ConfigUtil; import org.apache.ambari.logsearch.util.PropertiesUtil; +import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; +import org.apache.lucene.analysis.core.KeywordTokenizerFactory; +import org.apache.lucene.analysis.path.PathHierarchyTokenizerFactory; +import org.apache.lucene.analysis.standard.StandardTokenizerFactory; import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.schema.TrieDoubleField; +import org.apache.solr.schema.TrieFloatField; +import org.apache.solr.schema.TrieLongField; import org.springframework.stereotype.Component; @Component @@ -245,6 +253,10 @@ public class QueryGeneration extends QueryGenerationBase { String value = getOriginalValue(originalKey, "" + columnListMap.get(key)); orQuery = putWildCardByType(value, originalKey, logType); + if (stringUtil.isEmpty(orQuery)) { + logger.debug("Removing invalid filter for key :"+originalKey +" and value :" +value ); + continue; + } boolean isSame = false; if (elments.contains(key)) { isSame = true; @@ -337,48 +349,72 @@ public class QueryGeneration extends QueryGenerationBase { } private String putWildCardByType(String str, String key, LOG_TYPE logType) { - String type; + String fieldType; + SolrDaoBase solrDaoBase = null; switch (logType) { case AUDIT: - String auditSuffix = PropertiesUtil - .getProperty("logsearch.solr.collection.audit.logs",LogSearchConstants.DEFAULT_AUDIT_COLUMN_SUFFIX); - type = ConfigUtil.schemaFieldsName.get(key + auditSuffix); + fieldType = auditSolrDao.schemaFieldsNameMap.get(key); + solrDaoBase = auditSolrDao; break; case SERVICE: - String serviceLogs = PropertiesUtil.getProperty("logsearch.solr.collection.service.logs",LogSearchConstants.DEFAULT_SERVICE_COLUMN_SUFFIX); - type = ConfigUtil.schemaFieldsName.get(key + serviceLogs); + fieldType = serviceLogsSolrDao.schemaFieldsNameMap.get(key); + solrDaoBase = serviceLogsSolrDao; if (key.equalsIgnoreCase(LogSearchConstants.SOLR_LOG_MESSAGE)) { return solrUtil.escapeForLogMessage(key, str); } break; default: // set as null - type = null; + logger.error("Invalid logtype :" + logType); + fieldType = null; } - if (type == null) { - return key + ":" + "*" + str + "*"; - } else if ("text_std_token_lower_case".equalsIgnoreCase(type)) { - return key + ":" + solrUtil.escapeForStandardTokenizer(str); - } else if ("key_lower_case".equalsIgnoreCase(type) - || "string".equalsIgnoreCase(type)) { - return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str); - } else if ("ip_address".equalsIgnoreCase(type)) { - return key + ":" + str; + if (!stringUtil.isEmpty(fieldType)) { + if (solrUtil.isSolrFieldNumber(fieldType, solrDaoBase)) { + String value = putEscapeCharacterForNumber(str, fieldType,solrDaoBase); + if (!stringUtil.isEmpty(value)) { + return key + ":" + value; + } else { + return null; + } + } else if (checkTokenizer(fieldType, StandardTokenizerFactory.class,solrDaoBase)) { + return key + ":" + solrUtil.escapeForStandardTokenizer(str); + } else if (checkTokenizer(fieldType, KeywordTokenizerFactory.class,solrDaoBase)|| "string".equalsIgnoreCase(fieldType)) { + return key + ":" + solrUtil.makeSolrSearchStringWithoutAsterisk(str); + } else if (checkTokenizer(fieldType, PathHierarchyTokenizerFactory.class,solrDaoBase)) { + return key + ":" + str; + } } - return key + ":" + putEscapeCharacterForNumber(str); + return key + ":" + "*" + str + "*"; } - private String putEscapeCharacterForNumber(String str) { - String escapeCharSting = "" + returnDefaultIfValueNotNumber(str); - escapeCharSting = str.replace("-", "\\-"); + private String putEscapeCharacterForNumber(String str,String fieldType,SolrDaoBase solrDaoBase) { + if (!stringUtil.isEmpty(str)) { + str = str.replace("*", ""); + } + String escapeCharSting = parseInputValueAsPerFieldType(str,fieldType,solrDaoBase); + if (escapeCharSting == null || escapeCharSting.isEmpty()) { + return null; + } + escapeCharSting = escapeCharSting.replace("-", "\\-"); return escapeCharSting; } - private String returnDefaultIfValueNotNumber(String str) { + private String parseInputValueAsPerFieldType(String str,String fieldType,SolrDaoBase solrDaoBase ) { try { - return "" + Integer.parseInt(str); + HashMap<String, Object> fieldTypeInfoMap= solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase); + String className = (String) fieldTypeInfoMap.get("class"); + if( className.equalsIgnoreCase(TrieDoubleField.class.getSimpleName())){ + return ""+ Double.parseDouble(str); + }else if(className.equalsIgnoreCase(TrieFloatField.class.getSimpleName())){ + return ""+ Float.parseFloat(str); + }else if(className.equalsIgnoreCase(TrieLongField.class.getSimpleName())){ + return ""+ Long.parseLong(str); + }else { + return "" + Integer.parseInt(str); + } } catch (Exception e) { - return "0"; + logger.debug("Invaid input str: " + str + " For fieldType :" + fieldType); + return null; } } @@ -436,4 +472,25 @@ public class QueryGeneration extends QueryGenerationBase { } return originalKey; } + + public boolean checkTokenizer(String fieldType,Class tokenizerFactoryClass,SolrDaoBase solrDaoBase) { + HashMap<String, Object> fieldTypeMap = solrUtil.getFieldTypeInfoMap(fieldType,solrDaoBase); + HashMap<String, Object> analyzer = (HashMap<String, Object>) fieldTypeMap + .get("analyzer"); + if (analyzer != null) { + HashMap<String, Object> tokenizerMap = (HashMap<String, Object>) analyzer + .get("tokenizer"); + if (tokenizerMap != null) { + String tokenizerClass = (String) tokenizerMap.get("class"); + if (!StringUtils.isEmpty(tokenizerClass)) { + tokenizerClass =tokenizerClass.replace("solr.", ""); + if (tokenizerClass.equalsIgnoreCase(tokenizerFactoryClass + .getSimpleName())) { + return true; + } + } + } + } + return false; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java index a49107d..a128098 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/query/QueryGenerationBase.java @@ -24,6 +24,8 @@ import java.util.List; import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.SearchCriteria; +import org.apache.ambari.logsearch.dao.AuditSolrDao; +import org.apache.ambari.logsearch.dao.ServiceLogsSolrDao; import org.apache.ambari.logsearch.util.JSONUtil; import org.apache.ambari.logsearch.util.QueryBase; import org.apache.ambari.logsearch.util.SolrUtil; @@ -34,6 +36,7 @@ import org.apache.solr.client.solrj.SolrQuery.ORDER; import org.springframework.beans.factory.annotation.Autowired; import com.google.gson.Gson; +import com.sun.jersey.api.spring.Autowire; public abstract class QueryGenerationBase extends QueryBase { @@ -47,6 +50,14 @@ public abstract class QueryGenerationBase extends QueryBase { @Autowired JSONUtil jsonUtil; + + @Autowired + AuditSolrDao auditSolrDao; + + @Autowired + ServiceLogsSolrDao serviceLogsSolrDao; + + public static enum CONDITION { OR, AND http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java index bdd304f..2661150 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/ConfigUtil.java @@ -23,9 +23,9 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; - import org.apache.ambari.logsearch.common.LogSearchConstants; import org.apache.ambari.logsearch.common.MessageEnums; +import org.apache.ambari.logsearch.dao.SolrDaoBase; import org.apache.ambari.logsearch.manager.MgrBase; import org.apache.log4j.Logger; import org.codehaus.jettison.json.JSONArray; @@ -38,7 +38,6 @@ public class ConfigUtil { public static HashMap<String, String> auditLogsColumnMapping = new HashMap<String, String>(); - public static HashMap<String, String> schemaFieldsName = new HashMap<String, String>(); public static void initializeApplicationConfig() { initializeColumnMapping(); @@ -78,57 +77,63 @@ public class ConfigUtil { auditLogsColumnMapping); } + public static void extractSchemaFieldsName(String responseString, - String suffix) { + HashMap<String, String> schemaFieldsNameMap, + HashMap<String, String> schemaFieldTypeMap) { try { JSONObject jsonObject = new JSONObject(responseString); - JSONArray jsonArrayList = jsonObject.getJSONArray("fields"); - - if(jsonArrayList == null){ + JSONObject schemajsonObject = jsonObject.getJSONObject("schema"); + JSONArray jsonArrayList = schemajsonObject.getJSONArray("fields"); + JSONArray fieldTypeJsonArray = schemajsonObject + .getJSONArray("fieldTypes"); + if (jsonArrayList == null) { + return; + } + if (fieldTypeJsonArray == null) { return; } + HashMap<String, String> _schemaFieldTypeMap = new HashMap<String, String>(); + HashMap<String, String> _schemaFieldsNameMap = new HashMap<String, String>(); + for (int i = 0; i < fieldTypeJsonArray.length(); i++) { + JSONObject typeObject = fieldTypeJsonArray.getJSONObject(i); + String name = typeObject.getString("name"); + String fieldTypeJson = typeObject.toString(); + _schemaFieldTypeMap.put(name, fieldTypeJson); + } for (int i = 0; i < jsonArrayList.length(); i++) { JSONObject explrObject = jsonArrayList.getJSONObject(i); String name = explrObject.getString("name"); String type = explrObject.getString("type"); - if (!name.contains("@") && !name.startsWith("_") - && !name.contains("_md5") && !name.contains("_ms") - && !name.contains(LogSearchConstants.NGRAM_SUFFIX) - && !name.contains("tags") && !name.contains("_str")) { - schemaFieldsName.put(name + suffix, type); + && !name.contains("_md5") && !name.contains("_ms") + && !name.contains(LogSearchConstants.NGRAM_SUFFIX) + && !name.contains("tags") && !name.contains("_str")) { + _schemaFieldsNameMap.put(name, type); } } - + schemaFieldsNameMap.clear(); + schemaFieldTypeMap.clear(); + schemaFieldsNameMap.putAll(_schemaFieldsNameMap); + schemaFieldTypeMap.putAll(_schemaFieldTypeMap); } catch (Exception e) { - logger.error(e + "Credentials not specified in logsearch.properties " - + MessageEnums.ERROR_SYSTEM); - + + MessageEnums.ERROR_SYSTEM); } - } @SuppressWarnings("rawtypes") - public static void getSchemaFieldsName(String suffix, String excludeArray[], - List<String> fieldNames) { - if (!schemaFieldsName.isEmpty()) { - Iterator iteratorSechmaFieldsName = schemaFieldsName.entrySet() - .iterator(); - + public static void getSchemaFieldsName(String excludeArray[], + List<String> fieldNames, SolrDaoBase solrDaoBase) { + if (!solrDaoBase.schemaFieldsNameMap.isEmpty()) { + Iterator iteratorSechmaFieldsName = solrDaoBase.schemaFieldsNameMap + .entrySet().iterator(); while (iteratorSechmaFieldsName.hasNext()) { - - Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName - .next(); + Map.Entry fieldName = (Map.Entry) iteratorSechmaFieldsName.next(); String field = "" + fieldName.getKey(); - - if (field.contains(suffix)) { - field = field.replace(suffix, ""); - if (!isExclude(field, excludeArray)) { - fieldNames.add(field); - } - + if (!isExclude(field, excludeArray)) { + fieldNames.add(field); } } } @@ -144,5 +149,4 @@ public class ConfigUtil { } return false; } - } http://git-wip-us.apache.org/repos/asf/ambari/blob/5bfcf70e/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java index f68891f..ee706bf 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/java/org/apache/ambari/logsearch/util/SolrUtil.java @@ -20,10 +20,16 @@ package org.apache.ambari.logsearch.util; import java.util.Collection; +import java.util.HashMap; import java.util.Locale; import org.apache.ambari.logsearch.common.LogSearchConstants; +import org.apache.ambari.logsearch.dao.SolrDaoBase; import org.apache.log4j.Logger; +import org.apache.solr.schema.TrieDoubleField; +import org.apache.solr.schema.TrieFloatField; +import org.apache.solr.schema.TrieIntField; +import org.apache.solr.schema.TrieLongField; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -34,6 +40,9 @@ public class SolrUtil { @Autowired StringUtil stringUtil; + @Autowired + JSONUtil jsonUtil; + public String setField(String fieldName, String value) { if (value == null || value.trim().length() == 0) { return ""; @@ -296,4 +305,49 @@ public class SolrUtil { return newSearch.replace(" ", "\\ "); } + + + public boolean isSolrFieldNumber(String fieldType,SolrDaoBase solrDaoBase) { + if (stringUtil.isEmpty(fieldType)) { + return false; + } else { + HashMap<String, Object> typeInfoMap = getFieldTypeInfoMap(fieldType,solrDaoBase); + if (typeInfoMap == null || typeInfoMap.isEmpty()) { + return false; + } + String fieldTypeClassName = (String) typeInfoMap.get("class"); + if (fieldTypeClassName.equalsIgnoreCase(TrieIntField.class + .getSimpleName())) { + return true; + } + if (fieldTypeClassName.equalsIgnoreCase(TrieDoubleField.class + .getSimpleName())) { + return true; + } + if (fieldTypeClassName.equalsIgnoreCase(TrieFloatField.class + .getSimpleName())) { + return true; + } + if (fieldTypeClassName.equalsIgnoreCase(TrieLongField.class + .getSimpleName())) { + return true; + } + return false; + } + } + + public HashMap<String, Object> getFieldTypeInfoMap(String fieldType,SolrDaoBase solrDaoBase) { + String fieldTypeMetaData = solrDaoBase.schemaFieldTypeMap.get(fieldType); + HashMap<String, Object> fieldTypeMap = jsonUtil + .jsonToMapObject(fieldTypeMetaData); + if (fieldTypeMap == null) { + return new HashMap<String, Object>(); + } + String classname = (String) fieldTypeMap.get("class"); + if (!stringUtil.isEmpty(classname)) { + classname = classname.replace("solr.", ""); + fieldTypeMap.put("class", classname); + } + return fieldTypeMap; + } } \ No newline at end of file
