AMBARI-16034. Incremental changes to LogSearch to bring it up to date in the trunk (Dharmesh Makwana via oleewere)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/888faf26 Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/888faf26 Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/888faf26 Branch: refs/heads/trunk Commit: 888faf260d2d9fc454072ed69d1149c733ca98f0 Parents: 25ed583 Author: oleewere <[email protected]> Authored: Mon Apr 25 16:48:31 2016 +0200 Committer: oleewere <[email protected]> Committed: Mon Apr 25 17:10:34 2016 +0200 ---------------------------------------------------------------------- .../logsearch/appender/LogsearchAppender.java | 50 - .../logsearch/appender/LogsearchConversion.java | 6 + .../appender/LogsearchRollingFileAppender.java | 43 + .../src/test/resources/log4j.properties | 11 +- .../ambari-logsearch-logfeeder/build.xml | 3 - .../ambari-logsearch-logfeeder/pom.xml | 8 +- .../apache/ambari/logfeeder/LogFeederUtil.java | 11 +- .../logconfig/FetchConfigFromSolr.java | 52 +- .../logfeeder/logconfig/LogfeederScheduler.java | 3 +- .../logconfig/filter/ApplyLogFilter.java | 7 +- .../logconfig/filter/DefaultDataFilter.java | 5 +- .../logconfig/filter/FilterLogData.java | 2 +- .../apache/ambari/logfeeder/output/Output.java | 14 + .../ambari/logfeeder/output/OutputKafka.java | 2 +- .../ambari/logfeeder/output/OutputSolr.java | 3 +- .../apache/ambari/logfeeder/util/SolrUtil.java | 28 +- .../src/main/resources/config.json.j2 | 21 +- .../src/main/resources/log4j.xml | 17 +- .../src/main/scripts/run.sh | 6 + .../logfeeder/filter/JSONFilterCodeTest.java | 10 +- .../ambari-logsearch-portal/build.xml | 3 - .../ambari-logsearch-portal/pom.xml | 7 +- ambari-logsearch/ambari-logsearch-portal/run.sh | 8 +- .../configsets/audit_logs/conf/managed-schema | 9 +- .../configsets/hadoop_logs/conf/managed-schema | 37 +- .../main/configsets/history/conf/managed-schema | 98 + .../src/main/configsets/history/conf/schema.xml | 49 - .../main/configsets/history/conf/solrconfig.xml | 1885 +++++++++++++++++- .../logsearch/common/LogSearchConstants.java | 25 + .../logsearch/common/ManageStartEndTime.java | 6 +- .../ambari/logsearch/common/MessageEnums.java | 33 +- .../ambari/logsearch/common/SearchCriteria.java | 9 +- .../logsearch/common/UserSessionInfo.java | 2 +- .../ambari/logsearch/dao/AuditSolrDao.java | 5 + .../logsearch/dao/ServiceLogsSolrDao.java | 6 + .../ambari/logsearch/dao/SolrDaoBase.java | 113 +- .../ambari/logsearch/dao/UserConfigSolrDao.java | 5 + .../logsearch/graph/GraphDataGenerator.java | 414 ++++ .../logsearch/graph/GraphDataGeneratorBase.java | 316 ++- .../logsearch/graph/GraphDataGnerator.java | 397 ---- .../ambari/logsearch/manager/AuditMgr.java | 302 ++- .../ambari/logsearch/manager/LogFileMgr.java | 50 +- .../ambari/logsearch/manager/LogsMgr.java | 1460 +++++++------- .../ambari/logsearch/manager/MgrBase.java | 104 +- .../ambari/logsearch/manager/PublicMgr.java | 11 +- .../ambari/logsearch/manager/UserConfigMgr.java | 212 +- .../ambari/logsearch/query/QueryGeneration.java | 465 +++-- .../logsearch/query/QueryGenerationBase.java | 338 ++-- .../apache/ambari/logsearch/rest/AuditREST.java | 2 - .../ambari/logsearch/rest/DashboardREST.java | 22 +- .../ambari/logsearch/rest/UserConfigREST.java | 2 - .../apache/ambari/logsearch/util/BizUtil.java | 292 ++- .../ambari/logsearch/util/ConfigUtil.java | 29 +- .../apache/ambari/logsearch/util/DateUtil.java | 54 +- .../apache/ambari/logsearch/util/FileUtil.java | 42 +- .../apache/ambari/logsearch/util/JSONUtil.java | 5 +- .../apache/ambari/logsearch/util/QueryBase.java | 61 +- .../ambari/logsearch/util/RESTErrorUtil.java | 4 +- .../apache/ambari/logsearch/util/SolrUtil.java | 177 +- .../ambari/logsearch/util/StringUtil.java | 6 +- .../logsearch/util/XMLPropertiesUtil.java | 57 +- .../ambari/logsearch/view/VBarDataList.java | 6 +- .../ambari/logsearch/view/VCountList.java | 3 +- .../ambari/logsearch/view/VGroupList.java | 3 +- .../org/apache/ambari/logsearch/view/VHost.java | 4 +- .../ambari/logsearch/view/VNameValue.java | 5 +- .../ambari/logsearch/view/VNameValueList.java | 2 +- .../apache/ambari/logsearch/view/VNodeList.java | 2 +- .../ambari/logsearch/view/VSolrLogList.java | 6 +- .../apache/ambari/logsearch/view/VSummary.java | 16 +- .../ambari/logsearch/view/VSummaryCount.java | 8 +- .../logsearch/view/VSummaryCountList.java | 2 +- .../ambari/logsearch/view/VUserConfig.java | 8 +- .../ambari/logsearch/view/VUserConfigList.java | 8 +- .../LogsearchLogoutSuccessHandler.java | 2 +- .../web/listeners/SpringEventListener.java | 2 +- ...LogsearchAbstractAuthenticationProvider.java | 4 +- .../LogsearchAuthenticationProvider.java | 6 +- .../LogsearchFileAuthenticationProvider.java | 4 + .../LogsearchLdapAuthenticationProvider.java | 4 + .../LogsearchSimpleAuthenticationProvider.java | 4 + .../src/main/resources/default.properties | 1 + .../src/main/resources/log4j.xml | 47 +- .../src/main/resources/logsearch.properties | 14 +- .../src/main/resources/logsearch.properties.j2 | 5 + .../src/main/scripts/run.sh | 5 + .../src/main/webapp/index.html | 6 +- .../src/main/webapp/scripts/utils/Utils.js | 72 +- .../src/main/webapp/scripts/utils/ViewUtils.js | 7 +- .../main/webapp/scripts/views/common/Header.js | 7 +- .../webapp/scripts/views/common/TableLayout.js | 2 +- .../dashboard/BubbleGraphTableLayoutView.js | 29 +- .../scripts/views/dashboard/HostListView.js | 5 +- .../views/dialog/SaveSearchFilterView.js | 4 +- .../views/filter/CreateLogfeederFilterView.js | 113 +- .../webapp/scripts/views/tabs/LogFileView.js | 13 +- .../main/webapp/scripts/views/tabs/TreeView.js | 1 - .../troubleshoot/TroubleShootLayoutView.js | 70 +- .../src/main/webapp/styles/style.css | 120 +- .../src/main/webapp/styles/style_v2.css | 2 +- .../webapp/templates/common/Header_tmpl.html | 13 +- .../dashboard/MainLayoutView_tmpl.html | 4 +- .../filter/CreateLogfeederFilter_tmpl.html | 19 +- .../webapp/templates/tabs/LogFileView_tmpl.html | 4 +- .../TroubleShootLayoutView_tmpl.html | 3 +- ambari-logsearch/pom.xml | 1 + 106 files changed, 5384 insertions(+), 2706 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java deleted file mode 100644 index 4339a21..0000000 --- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchAppender.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.ambari.logsearch.appender; - -import org.apache.log4j.DailyRollingFileAppender; -import org.apache.log4j.Layout; -import org.apache.log4j.Logger; -import org.apache.log4j.spi.LoggingEvent; - -public class LogsearchAppender extends DailyRollingFileAppender { - private static Logger logger = Logger.getLogger(LogsearchAppender.class); - - public LogsearchAppender() { - logger.debug("Initializing LogsearchAppender........... "); - } - - @Override - public void append(LoggingEvent event) { - super.append(event); - } - - @Override - public void setLayout(Layout layout) { - super.setLayout(layout); - } - - protected void subAppend(LoggingEvent event) { - this.qw.write(this.layout.format(event)); - if (shouldFlush(event)) { - this.qw.flush(); - } - } -} http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java index b9e7527..dbdfe6c 100644 --- a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java +++ b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchConversion.java @@ -70,4 +70,10 @@ public class LogsearchConversion extends EnhancedPatternLayout { e.printStackTrace(pw); return sw.toString(); } + + @Override + public boolean ignoresThrowable() { + //set false to ignore exception stacktrace + return false; + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java new file mode 100644 index 0000000..6a93db5 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-appender/src/main/java/org/apache/ambari/logsearch/appender/LogsearchRollingFileAppender.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ambari.logsearch.appender; + +import org.apache.log4j.Layout; +import org.apache.log4j.Logger; +import org.apache.log4j.RollingFileAppender; +import org.apache.log4j.spi.LoggingEvent; + +public class LogsearchRollingFileAppender extends RollingFileAppender { + private static Logger logger = Logger.getLogger(LogsearchRollingFileAppender.class); + + public LogsearchRollingFileAppender() { + logger.trace("Initializing LogsearchRollingFileAppender........... "); + } + + @Override + public void append(LoggingEvent event) { + super.append(event); + } + + @Override + public void setLayout(Layout layout) { + super.setLayout(layout); + } +} http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties b/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties index b51e87d..78fb66b 100644 --- a/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties +++ b/ambari-logsearch/ambari-logsearch-appender/src/test/resources/log4j.properties @@ -13,11 +13,12 @@ # log4j configuration used during build and unit tests # Root logger option -log4j.rootLogger=ALL, logsearchJson +log4j.rootLogger=INFO, logsearchJson # Redirect log messages to a logsearch json -log4j.appender.logsearchJson=org.apache.ambari.logsearch.appender.LogsearchAppender -log4j.appender.logsearchJson.File=target/logsearch/log.json -log4j.appender.logsearchJson.datePattern='.'yyyy-MM-dd -log4j.appender.allLog.Append=true +log4j.appender.logsearchJson=org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender +log4j.appender.logsearchJson.File=target/jsonlog/log.json +log4j.appender.logsearchJson.maxFileSize=10MB +log4j.appender.logsearchJson.maxBackupIndex=10 +log4j.appender.logsearchJson.Append=true log4j.appender.logsearchJson.layout=org.apache.ambari.logsearch.appender.LogsearchConversion \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/build.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml index 53f893e..738b2ef 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/build.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/build.xml @@ -32,9 +32,6 @@ <copy todir="target/package/libs" includeEmptyDirs="no"> <fileset dir="target/libs"/> </copy> - <copy todir="target/package/" includeEmptyDirs="no"> - <fileset file="target/LogFeeder.jar"/> - </copy> <copy todir="target/package/classes" includeEmptyDirs="no"> <fileset dir="target/classes"/> </copy> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml index 0888010..dc1b361 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/pom.xml @@ -39,6 +39,11 @@ <dependencies> <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-logsearch-appender</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> @@ -79,6 +84,7 @@ <artifactId>commons-logging</artifactId> <version>1.1.1</version> </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> @@ -109,8 +115,8 @@ <artifactId>jackson-xc</artifactId> <version>1.9.13</version> </dependency> - </dependencies> + </dependencies> <build> <finalName>LogFeeder</finalName> <pluginManagement> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java index 7303694..7a30d72 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/LogFeederUtil.java @@ -33,9 +33,11 @@ import java.util.Hashtable; import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.TimeZone; import org.apache.ambari.logfeeder.filter.Filter; import org.apache.ambari.logfeeder.input.Input; +import org.apache.ambari.logfeeder.logconfig.LogFeederConstants; import org.apache.ambari.logfeeder.mapper.Mapper; import org.apache.ambari.logfeeder.output.Output; import org.apache.commons.lang3.StringUtils; @@ -58,6 +60,7 @@ public class LogFeederUtil { final static int HASH_SEED = 31174077; public final static String DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; + public final static String SOLR_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; static Gson gson = new GsonBuilder().setDateFormat(DATE_FORMAT).create(); static Properties props; @@ -324,6 +327,9 @@ public class LogFeederUtil { } public static Map<String, Object> toJSONObject(String jsonStr) { + if(jsonStr==null || jsonStr.trim().isEmpty()){ + return new HashMap<String, Object>(); + } Type type = new TypeToken<Map<String, Object>>() { }.getType(); return gson.fromJson(jsonStr, type); @@ -380,7 +386,8 @@ public class LogFeederUtil { public static String getDate(String timeStampStr) { try { - DateFormat sdf = new SimpleDateFormat(DATE_FORMAT); + DateFormat sdf = new SimpleDateFormat(SOLR_DATE_FORMAT); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); Date netDate = (new Date(Long.parseLong(timeStampStr))); return sdf.format(netDate); } catch (Exception ex) { @@ -468,7 +475,7 @@ public class LogFeederUtil { return true; } } - if (value.equalsIgnoreCase("ALL")) { + if (value.equalsIgnoreCase(LogFeederConstants.ALL)) { return true; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java index fc12458..e645a3d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/FetchConfigFromSolr.java @@ -40,12 +40,18 @@ public class FetchConfigFromSolr extends Thread { private static String endTimeDateFormat = "yyyy-MM-dd'T'HH:mm:ss.SSS";//2016-04-05T04:30:00.000Z private static String sysTimeZone = "GMT"; - public FetchConfigFromSolr() { + public FetchConfigFromSolr(boolean isDaemon) { this.setName(this.getClass().getSimpleName()); + this.setDaemon(isDaemon); } @Override public void run() { + String zkHosts = LogFeederUtil.getStringProperty("logfeeder.solr.zkhosts"); + if( zkHosts == null || zkHosts.trim().length() == 0 ) { + logger.warn("Solr ZKHosts for UserConfig/History is not set. Won't look for level configuration from Solr."); + return; + } solrConfigInterval = LogFeederUtil.getIntProperty("logfeeder.solr.config.internal", solrConfigInterval); delay = 1000 * solrConfigInterval; do { @@ -60,11 +66,14 @@ public class FetchConfigFromSolr extends Thread { } private synchronized void pullConfigFromSolr() { - HashMap<String, Object> configDocMap = SolrUtil.getInstance().getConfigDoc(); - if (configDocMap != null) { - String configJson = (String) configDocMap.get(LogFeederConstants.VALUES); - if (configJson != null) { - logfeederFilterWrapper = LogFeederUtil.getGson().fromJson(configJson, VLogfeederFilterWrapper.class); + SolrUtil solrUtil = SolrUtil.getInstance(); + if(solrUtil!=null){ + HashMap<String, Object> configDocMap = solrUtil.getConfigDoc(); + if (configDocMap != null) { + String configJson = (String) configDocMap.get(LogFeederConstants.VALUES); + if (configJson != null) { + logfeederFilterWrapper = LogFeederUtil.getGson().fromJson(configJson, VLogfeederFilterWrapper.class); + } } } } @@ -117,19 +126,28 @@ public class FetchConfigFromSolr extends Thread { List<String> hosts = componentFilter.getHosts(); List<String> defaultLevels = componentFilter.getDefaultLevels(); List<String> overrideLevels = componentFilter.getOverrideLevels(); - if (LogFeederUtil.isListContains(hosts, hostName, false)) { - if (isFilterExpired(componentFilter)) { - // pick default - logger.debug("Filter for component " + componentName + " and host :" + hostName + " is expired at " - + componentFilter.getExpiryTime()); - return defaultLevels; - } else { - // return tmp filter levels - return overrideLevels; + String expiryTime=componentFilter.getExpiryTime(); + //check is user override or not + if ((expiryTime != null && !expiryTime.isEmpty()) + || (overrideLevels != null && !overrideLevels.isEmpty()) + || (hosts != null && !hosts.isEmpty())) { + if (hosts == null || hosts.isEmpty()) { + // hosts list is empty or null consider it apply on all hosts + hosts.add(LogFeederConstants.ALL); + } + if (LogFeederUtil.isListContains(hosts, hostName, false)) { + if (isFilterExpired(componentFilter)) { + // pick default + logger.debug("Filter for component " + componentName + " and host :" + + hostName + " is expired at " + componentFilter.getExpiryTime()); + return defaultLevels; + } else { + // return tmp filter levels + return overrideLevels; + } } - } else { - return defaultLevels; } + return defaultLevels; } public static VLogfeederFilter findComponentFilter(String componentName) { http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java index 7525dff..128c5c4 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/LogfeederScheduler.java @@ -52,7 +52,8 @@ public enum LogfeederScheduler { private List<Thread> getThreadList() { List<Thread> tasks = new ArrayList<Thread>(); - tasks.add(new FetchConfigFromSolr()); + Thread configMonitor = new FetchConfigFromSolr(true); + tasks.add(configMonitor); return tasks; } } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java index 3748445..f223207 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/ApplyLogFilter.java @@ -50,7 +50,12 @@ public class ApplyLogFilter extends DefaultDataFilter { //return default value if there is no filter found for particular component return defaultValue; } - List<String> allowedLevels = FetchConfigFromSolr.getAllowedLevels(hostName, componentFilter); + List<String> allowedLevels = FetchConfigFromSolr.getAllowedLevels( + hostName, componentFilter); + if (allowedLevels == null || allowedLevels.isEmpty()) { + // if allowedlevels list is empty then allow everything + allowedLevels.add(LogFeederConstants.ALL); + } return LogFeederUtil.isListContains(allowedLevels, level, false); } } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java index 9e98c6a..a064663 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/DefaultDataFilter.java @@ -18,7 +18,6 @@ */ package org.apache.ambari.logfeeder.logconfig.filter; -import java.util.List; import java.util.Map; import org.apache.log4j.Logger; @@ -28,9 +27,7 @@ import org.apache.log4j.Logger; */ public class DefaultDataFilter { private static Logger logger = Logger.getLogger(DefaultDataFilter.class); - - protected static final boolean CASE_SENSITIVE = false; - + public boolean applyFilter(Map<String, Object> outputJsonObj, boolean defaultValue) { return defaultValue; } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java index 643df98..bf33f93 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/logconfig/filter/FilterLogData.java @@ -40,7 +40,7 @@ public enum FilterLogData { return defaultValue; } Map<String, Object> jsonObj = LogFeederUtil.toJSONObject(jsonBlock); - return applyLogFilter.applyFilter(jsonObj, defaultValue); + return isAllowed(jsonObj); } public boolean isAllowed(Map<String, Object> jsonObj) { http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java index dd67d07..c067680 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/Output.java @@ -22,6 +22,7 @@ package org.apache.ambari.logfeeder.output; import java.lang.reflect.Type; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import org.apache.ambari.logfeeder.ConfigBlock; import org.apache.ambari.logfeeder.LogFeederUtil; @@ -115,5 +116,18 @@ public abstract class Output extends ConfigBlock { logStatForMetric(writeBytesMetric, "Stat: Bytes Written"); } + + public void trimStrValue(Map<String, Object> jsonObj) { + if (jsonObj != null) { + for (Entry<String, Object> entry : jsonObj.entrySet()) { + String key = entry.getKey(); + Object value = entry.getValue(); + if (value != null && value instanceof String) { + String valueStr = value.toString().trim(); + jsonObj.put(key, valueStr); + } + } + } + } } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java index cd4f951..efbc366 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputKafka.java @@ -283,4 +283,4 @@ public class OutputKafka extends Output { } } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java index 215f691..6fb0b0e 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/OutputSolr.java @@ -194,6 +194,7 @@ public class OutputSolr extends Output { @Override public void write(Map<String, Object> jsonObj, InputMarker inputMarker) throws Exception { try { + trimStrValue(jsonObj); outgoingBuffer.put(new OutputData(jsonObj, inputMarker)); } catch (InterruptedException e) { // ignore @@ -442,4 +443,4 @@ public class OutputSolr extends Output { return localBuffer.isEmpty(); } } -} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java index 4265dc6..200a603 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/util/SolrUtil.java @@ -23,6 +23,7 @@ import java.util.HashMap; import org.apache.ambari.logfeeder.LogFeederUtil; import org.apache.ambari.logfeeder.logconfig.LogFeederConstants; +import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; @@ -64,7 +65,12 @@ public class SolrUtil { try { instance = new SolrUtil(); } catch (Exception e) { - logger.error(e); + final String LOG_MESSAGE_KEY = SolrUtil.class + .getSimpleName() + "_SOLR_UTIL"; + LogFeederUtil.logErrorMessageByInterval( + LOG_MESSAGE_KEY, + "Error constructing solrUtil", e, logger, + Level.WARN); } } } @@ -162,6 +168,7 @@ public class SolrUtil { QueryResponse queryResponse = solrClient.query(solrQuery, METHOD.POST); return queryResponse; } else { + logger.error("solrClient can't be null"); return null; } } @@ -176,15 +183,18 @@ public class SolrUtil { String fq = LogFeederConstants.ROW_TYPE + ":" + LogFeederConstants.NAME; solrQuery.setFilterQueries(fq); try { - QueryResponse response = SolrUtil.getInstance().process(solrQuery); - SolrDocumentList documentList = response.getResults(); - if (documentList != null && documentList.size() > 0) { - SolrDocument configDoc = documentList.get(0); - String configJson = LogFeederUtil.getGson().toJson(configDoc); - configMap = (HashMap<String, Object>) LogFeederUtil.toJSONObject(configJson); + QueryResponse response = process(solrQuery); + if (response != null) { + SolrDocumentList documentList = response.getResults(); + if (documentList != null && documentList.size() > 0) { + SolrDocument configDoc = documentList.get(0); + String configJson = LogFeederUtil.getGson().toJson(configDoc); + configMap = (HashMap<String, Object>) LogFeederUtil + .toJSONObject(configJson); + } } - } catch (SolrException | SolrServerException | IOException e) { - logger.error(e); + } catch (Exception e) { + logger.error("Error getting config", e); } return configMap; } http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 index 163ee2b..1c5ee8d 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/config.json.j2 @@ -179,17 +179,17 @@ { "type":"logsearch_app", "rowtype":"service", - "path":"{{logsearch_log_dir}}/logsearch.log" + "path":"{{logsearch_log_dir}}/logsearch.json" }, { "type":"logsearch_feeder", "rowtype":"service", - "path":"{{logfeeder_log_dir}}/logfeeder.log" + "path":"{{logfeeder_log_dir}}/logfeeder.json" }, { "type":"logsearch_perf", "rowtype":"service", - "path":"{{logsearch_log_dir}}/logsearch-performance.log" + "path":"{{logsearch_log_dir}}/logsearch-performance.json" }, { "type":"ranger_admin", @@ -625,13 +625,22 @@ }, { + "filter": "json", + "conditions": { + "fields": { + "type": [ + "logsearch_app", + "logsearch_feeder", + "logsearch_perf" + ] + } + } + }, + { "filter":"grok", "conditions":{ "fields":{ "type":[ - "logsearch_app", - "logsearch_feeder", - "logsearch_perf", "ranger_admin", "ranger_dbpatch" ] http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml index 0717477..7ef967c 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/resources/log4j.xml @@ -33,24 +33,35 @@ <layout class="org.apache.log4j.PatternLayout"> <param name="ConversionPattern" value="%d [%t] %-5p %C{6} (%F:%L) - %m%n"/> </layout> + </appender> + + <appender name="rolling_file_json" + class="org.apache.ambari.logsearch.appender.LogsearchRollingFileAppender"> + <param name="file" value="logs/logsearch-logfeeder.json" /> + <param name="append" value="true" /> + <param name="maxFileSize" value="10MB" /> + <param name="maxBackupIndex" value="10" /> + <layout class="org.apache.ambari.logsearch.appender.LogsearchConversion" /> </appender> + <!-- Logs to suppress BEGIN --> <category name="org.apache.solr.common.cloud.ZkStateReader" additivity="false"> <priority value="error" /> - <appender-ref ref="rolling_file" /> + <appender-ref ref="daily_rolling_file" /> </category> <category name="apache.solr.client.solrj.impl.CloudSolrClient" additivity="false"> <priority value="fatal" /> - <appender-ref ref="rolling_file" /> + <appender-ref ref="daily_rolling_file" /> </category> <!-- Logs to suppress END --> <category name="org.apache.ambari.logfeeder" additivity="false"> - <priority value="info" /> + <priority value="INFO" /> <appender-ref ref="console" /> <!-- <appender-ref ref="daily_rolling_file" /> --> + <appender-ref ref="rolling_file_json"/> </category> <root> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh index 1e7185a..356a358 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/scripts/run.sh @@ -47,6 +47,12 @@ fi if [ "$LOGFEEDER_CONF_DIR" = "" ]; then LOGFEEDER_CONF_DIR="/etc/logfeeder/conf" + if [ ! -d $LOGFEEDER_CONF_DIR ]; then + if [ -d $script_dir/classes ]; then + LOGFEEDER_CONF_DIR=$script_dir/classes + fi + fi + fi LOGFEEDER_GC_LOGFILE=`dirname $LOGFILE`/logfeeder_gc.log http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java index ebfd0f5..fd14b48 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/test/java/org/apache/ambari/logfeeder/filter/JSONFilterCodeTest.java @@ -18,10 +18,12 @@ package org.apache.ambari.logfeeder.filter; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.HashMap; import java.util.Map; +import java.util.TimeZone; import org.apache.ambari.logfeeder.LogFeederUtil; import org.apache.ambari.logfeeder.OutputMgr; @@ -64,7 +66,9 @@ public class JSONFilterCodeTest { EasyMock.replay(mockOutputMgr); Date d = new Date(); - String dateString = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).format(d); + DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + String dateString = sdf.format(d); jsonFilterCode.apply("{ logtime: '" + d.getTime() + "', line_number: 100 }", new InputMarker()); EasyMock.verify(mockOutputMgr); @@ -86,7 +90,9 @@ public class JSONFilterCodeTest { EasyMock.replay(mockOutputMgr); Date d = new Date(); - String dateString = new SimpleDateFormat(LogFeederUtil.DATE_FORMAT).format(d); + DateFormat sdf = new SimpleDateFormat(LogFeederUtil.SOLR_DATE_FORMAT); + sdf.setTimeZone(TimeZone.getTimeZone("UTC")); + String dateString = sdf.format(d); jsonFilterCode.apply("{ logtime: '" + d.getTime() + "', some_field: 'abc' }", new InputMarker()); EasyMock.verify(mockOutputMgr); http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/build.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/build.xml b/ambari-logsearch/ambari-logsearch-portal/build.xml index 22c7938..d6db5f0 100644 --- a/ambari-logsearch/ambari-logsearch-portal/build.xml +++ b/ambari-logsearch/ambari-logsearch-portal/build.xml @@ -38,9 +38,6 @@ <copy todir="target/package/libs" includeEmptyDirs="no"> <fileset dir="target/libs"/> </copy> - <copy todir="target/package/" includeEmptyDirs="no"> - <fileset file="target/LogSearch.jar"/> - </copy> <copy todir="target/package/classes" includeEmptyDirs="no"> <fileset dir="target/classes"/> </copy> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/pom.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/pom.xml b/ambari-logsearch/ambari-logsearch-portal/pom.xml index 9c5ba72..799e95e 100755 --- a/ambari-logsearch/ambari-logsearch-portal/pom.xml +++ b/ambari-logsearch/ambari-logsearch-portal/pom.xml @@ -163,6 +163,7 @@ <exclude>src/main/webapp/styles/animate.css</exclude> <exclude>**/*.json</exclude> <exclude>src/main/webapp/styles/visualsearch.css</exclude> + <exclude>**/*.log</exclude> </excludes> </configuration> <executions> @@ -542,6 +543,10 @@ <artifactId>commons-io</artifactId> <version>2.4</version> </dependency> - + <dependency> + <groupId>org.apache.ambari</groupId> + <artifactId>ambari-logsearch-appender</artifactId> + <version>${project.version}</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/run.sh ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/run.sh b/ambari-logsearch/ambari-logsearch-portal/run.sh index 4b1be8b..ae3c779 100755 --- a/ambari-logsearch/ambari-logsearch-portal/run.sh +++ b/ambari-logsearch/ambari-logsearch-portal/run.sh @@ -17,6 +17,8 @@ echo " ââââââââââââââââââââââââââ âââââââââââââââââââ ââââââ ââââââââââââââ âââ ââââââââ âââââââ âââââââ âââââââââââââââââââ ââââââ âââ ââââââââââ âââ " -mvn clean compile -Pdev -mvn exec:java -Pdev - +cd .. +mvn clean compile package -Pdev +cd ambari-logsearch-portal +#mvn exec:java -Pdev +java -cp target/libs/*:target/classes/ org.apache.ambari.logsearch.LogSearch http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema index 4a6d3ac..d2a1b6d 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/audit_logs/conf/managed-schema @@ -47,6 +47,7 @@ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> <analyzer> <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> <fieldType name="n_gram" class="solr.TextField" sortMissingLast="true" omitNorms="true"> @@ -103,17 +104,13 @@ <field name="req_self_id" type="key_lower_case" multiValued="false"/> <field name="event_md5" type="string" multiValued="false"/> <field name="file" type="key_lower_case" multiValued="false"/> - <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/> + <field name="log_message" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/> <field name="logfile_line_number" type="tint" omitNorms="false"/> - <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/> + <!-- <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/> --> <field name="message_md5" type="string" multiValued="false"/> <field name="type" type="key_lower_case" multiValued="false"/> <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/> <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/> - <!-- Copying everything text --> - <copyField source="*" dest="text" maxChars="25000" /> - - </schema> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema index fa33516..cad0955 100644 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/hadoop_logs/conf/managed-schema @@ -1,4 +1,21 @@ <?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + <!-- Solr managed schema - automatically generated - DO NOT EDIT --> <schema name="hadoop-logs-schema" version="1.5"> <uniqueKey>id</uniqueKey> @@ -51,6 +68,7 @@ <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> <analyzer> <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/> @@ -78,12 +96,12 @@ <field name="ip" type="ip_address" multiValued="false"/> <field name="level" type="key_lower_case" multiValued="false"/> <field name="line_number" type="tint" omitNorms="false"/> - <field name="log_message" type="key_lower_case" multiValued="false" omitNorms="false"/> + <field name="log_message" type="text_std_token_lower_case" multiValued="false" omitNorms="false"/> <field name="logfile_line_number" type="tint" omitNorms="false"/> <field name="logger_name" type="key_lower_case" multiValued="false"/> <field name="logtime" type="tdate" multiValued="false" docValues="true"/> <field name="logtype" type="key_lower_case" multiValued="false"/> - <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/> + <!-- <field name="message" type="text_std_token_lower_case" indexed="true" stored="true"/> --> <field name="message_md5" type="string" multiValued="false"/> <field name="method" type="key_lower_case" multiValued="false" omitNorms="false"/> <field name="path" type="key_lower_case" multiValued="false"/> @@ -96,11 +114,16 @@ <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/> <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/> - + <dynamicField name='key_*' type="key_lower_case" multiValued="false" stored="false"/> + <dynamicField name="ws_*" type="text_ws" multiValued="false" omitNorms="false" stored="false"/> + <copyField source="log_message" dest="key_log_message"/> + <!-- <copyField source="log_message" dest="ws_log_message"/> --> + <!-- Copy Fields--> <!-- ngram fields --> <!-- Whenever using a copy field provide following structure "ngram_"<OriginalFieldName> --> +<!-- Commenting till we test properly. Now it is not used and it is taking unnecessary cpu, memory and disk space <copyField source="bundle_id" dest="ngram_bundle_id"/> <copyField source="case_id" dest="ngram_case_id"/> <copyField source="cluster" dest="ngram_cluster"/> @@ -115,11 +138,7 @@ <copyField source="thread_name" dest="ngram_thread_name"/> <copyField source="type" dest="ngram_type"/> - <!-- Standard text tokenizer --> - <copyField source="log_message" dest="std_log_message"/> - - <!-- Copying everything text --> - <copyField source="*" dest="text" maxChars="25000" /> - +--> + </schema> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema new file mode 100644 index 0000000..062c14f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/managed-schema @@ -0,0 +1,98 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- Solr managed schema - automatically generated - DO NOT EDIT --> +<schema name="history" version="1.1"> + + <uniqueKey>composite_filtername-username</uniqueKey> + + <fieldType name="binary" class="solr.BinaryField"/> + <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/> + <fieldType name="booleans" class="solr.BoolField" sortMissingLast="true" multiValued="true"/> + <fieldType name="date" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="double" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="float" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="ignored" class="solr.StrField" indexed="false" stored="false" multiValued="true"/> + <fieldType name="int" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="ip_address" class="solr.TextField"> + <analyzer> + <tokenizer class="solr.PathHierarchyTokenizerFactory" replace="." delimiter="."/> + </analyzer> + </fieldType> + <fieldType name="long" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="0"/> + <fieldType name="random" class="solr.RandomSortField" indexed="true"/> + <fieldType name="string" class="solr.StrField" sortMissingLast="true"/> + <fieldType name="tdate" class="solr.TrieDateField" positionIncrementGap="0" precisionStep="6"/> + <fieldType name="tdates" class="solr.TrieDateField" positionIncrementGap="0" multiValued="true" precisionStep="6"/> + <fieldType name="tdouble" class="solr.TrieDoubleField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tdoubles" class="solr.TrieDoubleField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + <fieldType name="n_gram" class="solr.TextField" sortMissingLast="true" omitNorms="true"> + <analyzer> + <tokenizer class="solr.NGramTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + <fieldType name="key_lower_case" class="solr.TextField" + sortMissingLast="true" omitNorms="true"> + <analyzer> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory" /> + </analyzer> + </fieldType> + <fieldType name="text_std_token_lower_case" class="solr.TextField" positionIncrementGap="100" multiValued="true"> + <analyzer> + <tokenizer class="solr.StandardTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100"> + <analyzer> + <tokenizer class="solr.WhitespaceTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory"/> + </analyzer> + </fieldType> + <fieldType name="tfloat" class="solr.TrieFloatField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tfloats" class="solr.TrieFloatField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + <fieldType name="tint" class="solr.TrieIntField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tints" class="solr.TrieIntField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + <fieldType name="tlong" class="solr.TrieLongField" positionIncrementGap="0" precisionStep="8"/> + <fieldType name="tlongs" class="solr.TrieLongField" positionIncrementGap="0" multiValued="true" precisionStep="8"/> + + <solrQueryParser defaultOperator="OR"/> + + <field name="_version_" type="long" indexed="true" stored="true"/> + <field name="composite_filtername-username" type="string"/> + <field name="filtername" type="key_lower_case" indexed="true" required="true" stored="true"/> + <field name="id" type="string" required="true"/> + <field name="jsons" type="string"/> + <field name="rowtype" type="string" required="true"/> + <field name="share_username_list" type="string" multiValued="true"/> + <field name="username" type="key_lower_case" indexed="true" required="true" stored="true"/> + + <dynamicField name='ngram_*' type="n_gram" multiValued="false" stored="false"/> + <dynamicField name='std_*' type="text_std_token_lower_case" multiValued="false" stored="false"/> + <dynamicField name="*_string" type="string" multiValued="true" indexed="true" stored="true"/> + + +</schema> http://git-wip-us.apache.org/repos/asf/ambari/blob/888faf26/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml ---------------------------------------------------------------------- diff --git a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml b/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml deleted file mode 100644 index 2b9515c..0000000 --- a/ambari-logsearch/ambari-logsearch-portal/src/main/configsets/history/conf/schema.xml +++ /dev/null @@ -1,49 +0,0 @@ -<?xml version='1.0' ?> -<!-- - Licensed to the Apache Software Foundation (ASF) under one or more - contributor license agreements. See the NOTICE file distributed with - this work for additional information regarding copyright ownership. - The ASF licenses this file to You under the Apache License, Version 2.0 - (the "License"); you may not use this file except in compliance with - the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. ---> -<schema name='history' version='1.1'> - <types> - <fieldtype name='string' class='solr.StrField' /> - <fieldtype name='long' class='solr.TrieLongField' /> - - <fieldType name="key_lower_case" class="solr.TextField" - sortMissingLast="true" omitNorms="true"> - <analyzer> - <tokenizer class="solr.KeywordTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory" /> - </analyzer> - </fieldType> - </types> - - <fields> - <field name='id' type='string' required='true' /> - <field name="_version_" type="long" indexed="true" stored="true"/> - - <field name='username' type='key_lower_case' indexed="true" stored="true" required='true'/> - <field name='jsons' type='string'/> - - <field name="rowtype" type='string' required='true'/> - <field name='filtername' type='key_lower_case' indexed="true" stored="true" required='true'/> - - <field name='share_username_list' type='string' multiValued='true' /> - <field name='composite_filtername-username' type='string'/> - - <dynamicField name='*_string' type='string' multiValued='true' indexed='true' stored='true'/> - </fields> - <uniqueKey>composite_filtername-username</uniqueKey> - <solrQueryParser defaultOperator='OR' /> -</schema>
