This is an automated email from the ASF dual-hosted git repository. oleewere pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push: new 8dfe69e AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense… (#1174) 8dfe69e is described below commit 8dfe69e9b43ec0d6cf4ad39e1143c120e1402380 Author: kasakrisz <33458261+kasakr...@users.noreply.github.com> AuthorDate: Fri May 4 12:40:32 2018 +0200 AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense… (#1174) * AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense etc. * AMBARI-23667 - Cleanup/Fix Log Feeder patterns for HDP/HDF/smartsense etc. revert input.config-hdfs.json --- ambari-logsearch/ambari-logsearch-it/log4j.dtd | 230 ++++++++++++++++++++ ambari-logsearch/ambari-logsearch-it/pom.xml | 63 +++++- .../patterns/AmbariInfraSolrLogPatternIT.java | 39 ++++ .../logsearch/patterns/AmbariLogPatternIT.java | 235 +++++++++++++++++++++ .../logsearch/patterns/AtlasLogPatternIT.java | 52 +++++ .../logsearch/patterns/HBaseLogPatternIT.java | 55 +++++ .../logsearch/patterns/HDFSLogPatternIT.java | 52 +++++ .../logsearch/patterns/HdfsAuditLogPatternIT.java | 89 ++++++++ .../logsearch/patterns/HiveLogPatterntIT.java | 45 ++++ .../ambari/logsearch/patterns/JinjaFunctions.java | 27 +++ .../logsearch/patterns/KafkaLogPatternIT.java | 100 +++++++++ .../logsearch/patterns/KnoxLogPatternIT.java | 69 ++++++ .../ambari/logsearch/patterns/LayoutQuery.java | 23 ++ .../ambari/logsearch/patterns/ListAppender.java | 59 ++++++ .../ambari/logsearch/patterns/Log4jContent.java | 25 +++ .../ambari/logsearch/patterns/Log4jProperties.java | 65 ++++++ .../apache/ambari/logsearch/patterns/Log4jXml.java | 74 +++++++ .../logsearch/patterns/Log4jXmlProperties.java | 45 ++++ .../logsearch/patterns/MetricsLogPatternIT.java | 77 +++++++ .../ambari/logsearch/patterns/PatternITBase.java | 147 +++++++++++++ .../logsearch/patterns/RangerLogPatternIT.java | 86 ++++++++ .../logsearch/patterns/Spark2LogPatternIT.java | 71 +++++++ .../ambari/logsearch/patterns/StackDefContent.java | 61 ++++++ .../logsearch/patterns/StormLogPatternIT.java | 100 +++++++++ .../logsearch/patterns/YarnLogPatternIT.java | 169 +++++++++++++++ .../logsearch/patterns/ZeppelinLogPatternIT.java | 55 +++++ .../logsearch/patterns/ZookeeperLogPatternIT.java | 53 +++++ .../ambari/logfeeder/common/ConfigHandler.java | 8 +- .../logfeeder/common/LogEntryParseTester.java | 3 +- ambari-server/conf/unix/log4j.properties | 2 +- .../package/templates/input.config-kafka.json.j2 | 2 +- .../0.5.0/properties/input.config-ambari.json.j2 | 8 +- .../package/templates/input.config-storm.json.j2 | 79 ++++++- .../templates/input.config-mapreduce2.json.j2 | 18 +- 34 files changed, 2260 insertions(+), 26 deletions(-) diff --git a/ambari-logsearch/ambari-logsearch-it/log4j.dtd b/ambari-logsearch/ambari-logsearch-it/log4j.dtd new file mode 100644 index 0000000..8918101 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/log4j.dtd @@ -0,0 +1,230 @@ +<?xml version="1.0" encoding="UTF-8" ?> + <!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> + + <!ELEMENT log4j:configuration (renderer*, throwableRenderer?, + appender*,plugin*, (category|logger)*,root?, + (categoryFactory|loggerFactory)?)> + + <!-- The "threshold" attribute takes a level value below which --> + <!-- all logging statements are disabled. --> + + <!-- Setting the "debug" enable the printing of internal log4j logging --> + <!-- statements. --> + + <!-- By default, debug attribute is "null", meaning that we not do touch --> + <!-- internal log4j logging settings. The "null" value for the threshold --> + <!-- attribute can be misleading. The threshold field of a repository --> + <!-- cannot be set to null. The "null" value for the threshold attribute --> + <!-- simply means don't touch the threshold field, the threshold field --> + <!-- keeps its old value. --> + + <!ATTLIST log4j:configuration + xmlns:log4j CDATA #FIXED "http://jakarta.apache.org/log4j/" + threshold (all|trace|debug|info|warn|error|fatal|off|null) "null" + debug (true|false|null) "null" + reset (true|false) "false" + > + + <!-- renderer elements allow the user to customize the conversion of --> + <!-- message objects to String. --> + + <!ELEMENT renderer EMPTY> + <!ATTLIST renderer + renderedClass CDATA #REQUIRED + renderingClass CDATA #REQUIRED + > + + <!-- throwableRenderer allows the user to customize the conversion + of exceptions to a string representation. --> + <!ELEMENT throwableRenderer (param*)> + <!ATTLIST throwableRenderer + class CDATA #REQUIRED + > + + + <!-- Appenders must have a name and a class. --> + <!-- Appenders may contain an error handler, a layout, optional parameters --> + <!-- and filters. They may also reference (or include) other appenders. --> + <!ELEMENT appender (errorHandler?, param*, + rollingPolicy?, triggeringPolicy?, connectionSource?, + layout?, filter*, appender-ref*)> + <!ATTLIST appender + name CDATA #REQUIRED + class CDATA #REQUIRED + > + + <!ELEMENT layout (param*)> + <!ATTLIST layout + class CDATA #REQUIRED + > + + <!ELEMENT filter (param*)> + <!ATTLIST filter + class CDATA #REQUIRED + > + + <!-- ErrorHandlers can be of any class. They can admit any number of --> + <!-- parameters. --> + + <!ELEMENT errorHandler (param*, root-ref?, logger-ref*, appender-ref?)> + <!ATTLIST errorHandler + class CDATA #REQUIRED + > + + <!ELEMENT root-ref EMPTY> + + <!ELEMENT logger-ref EMPTY> + <!ATTLIST logger-ref + ref CDATA #REQUIRED + > + + <!ELEMENT param EMPTY> + <!ATTLIST param + name CDATA #REQUIRED + value CDATA #REQUIRED + > + + + <!-- The priority class is org.apache.log4j.Level by default --> + <!ELEMENT priority (param*)> + <!ATTLIST priority + class CDATA #IMPLIED + value CDATA #REQUIRED + > + + <!-- The level class is org.apache.log4j.Level by default --> + <!ELEMENT level (param*)> + <!ATTLIST level + class CDATA #IMPLIED + value CDATA #REQUIRED + > + + + <!-- If no level element is specified, then the configurator MUST not --> + <!-- touch the level of the named category. --> + <!ELEMENT category (param*,(priority|level)?,appender-ref*)> + <!ATTLIST category + class CDATA #IMPLIED + name CDATA #REQUIRED + additivity (true|false) "true" + > + + <!-- If no level element is specified, then the configurator MUST not --> + <!-- touch the level of the named logger. --> + <!ELEMENT logger (param*,level?,appender-ref*)> + <!ATTLIST logger + class CDATA #IMPLIED + name CDATA #REQUIRED + additivity (true|false) "true" + > + + + <!ELEMENT categoryFactory (param*)> + <!ATTLIST categoryFactory + class CDATA #REQUIRED> + + <!ELEMENT loggerFactory (param*)> + <!ATTLIST loggerFactory + class CDATA #REQUIRED> + + <!ELEMENT appender-ref EMPTY> + <!ATTLIST appender-ref + ref CDATA #REQUIRED + > + + <!-- plugins must have a name and class and can have optional parameters --> + <!ELEMENT plugin (param*, connectionSource?)> + <!ATTLIST plugin + name CDATA #REQUIRED + class CDATA #REQUIRED + > + + <!ELEMENT connectionSource (dataSource?, param*)> + <!ATTLIST connectionSource + class CDATA #REQUIRED + > + + <!ELEMENT dataSource (param*)> + <!ATTLIST dataSource + class CDATA #REQUIRED + > + + <!ELEMENT triggeringPolicy ((param|filter)*)> + <!ATTLIST triggeringPolicy + name CDATA #IMPLIED + class CDATA #REQUIRED + > + + <!ELEMENT rollingPolicy (param*)> + <!ATTLIST rollingPolicy + name CDATA #IMPLIED + class CDATA #REQUIRED + > + + + <!-- If no priority element is specified, then the configurator MUST not --> + <!-- touch the priority of root. --> + <!-- The root category always exists and cannot be subclassed. --> + <!ELEMENT root (param*, (priority|level)?, appender-ref*)> + + + <!-- ==================================================================== --> + <!-- A logging event --> + <!-- ==================================================================== --> + <!ELEMENT log4j:eventSet (log4j:event*)> + <!ATTLIST log4j:eventSet + xmlns:log4j CDATA #FIXED "http://jakarta.apache.org/log4j/" + version (1.1|1.2) "1.2" + includesLocationInfo (true|false) "true" + > + + + + <!ELEMENT log4j:event (log4j:message, log4j:NDC?, log4j:throwable?, + log4j:locationInfo?, log4j:properties?) > + + <!-- The timestamp format is application dependent. --> + <!ATTLIST log4j:event + logger CDATA #REQUIRED + level CDATA #REQUIRED + thread CDATA #REQUIRED + timestamp CDATA #REQUIRED + time CDATA #IMPLIED + > + + <!ELEMENT log4j:message (#PCDATA)> + <!ELEMENT log4j:NDC (#PCDATA)> + + <!ELEMENT log4j:throwable (#PCDATA)> + + <!ELEMENT log4j:locationInfo EMPTY> + <!ATTLIST log4j:locationInfo + class CDATA #REQUIRED + method CDATA #REQUIRED + file CDATA #REQUIRED + line CDATA #REQUIRED + > + + <!ELEMENT log4j:properties (log4j:data*)> + + <!ELEMENT log4j:data EMPTY> + <!ATTLIST log4j:data + name CDATA #REQUIRED + value CDATA #REQUIRED + > \ No newline at end of file diff --git a/ambari-logsearch/ambari-logsearch-it/pom.xml b/ambari-logsearch/ambari-logsearch-it/pom.xml index db3e09f..6a2fdf1 100644 --- a/ambari-logsearch/ambari-logsearch-it/pom.xml +++ b/ambari-logsearch/ambari-logsearch-it/pom.xml @@ -117,7 +117,25 @@ <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> - <version>11.0.1</version> + <version>16.0.1</version> + </dependency> + + <dependency> + <groupId>com.hubspot.jinjava</groupId> + <artifactId>jinjava</artifactId> + <version>2.2.0</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.easymock</groupId> + <artifactId>easymock</artifactId> + <version>3.4</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-all</artifactId> + <version>1.3</version> </dependency> </dependencies> @@ -224,6 +242,49 @@ </build> </profile> <profile> + <id>input-config-tests</id> + <activation> + <property> + <name>input-config-tests</name> + </property> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-failsafe-plugin</artifactId> + <version>${failsafe-plugin.version}</version> + <executions> + <execution> + <id>run-integration-tests</id> + <phase>integration-test</phase> + <goals> + <goal>integration-test</goal> + </goals> + <configuration> + <includes> + <include>**/*IT.java</include> + </includes> + <systemPropertyVariables> + <log4j.configuration>file:${project.build.testOutputDirectory}/log4j.properties</log4j.configuration> + <docker.host>${docker.host}</docker.host> + <backend.stories.location>${backend.stories.location}</backend.stories.location> + </systemPropertyVariables> + </configuration> + </execution> + <execution> + <id>verify-integration-tests</id> + <phase>verify</phase> + <goals> + <goal>verify</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + <profile> <id>all-tests</id> <activation> <property> diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariInfraSolrLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariInfraSolrLogPatternIT.java new file mode 100644 index 0000000..7b55716 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariInfraSolrLogPatternIT.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.File; + +import org.junit.Test; + +public class AmbariInfraSolrLogPatternIT extends PatternITBase { + // TODO: use hdp_ambari_definitions + @Test + public void testAmbariInfraSolrLogLayout() { + String layout = Log4jProperties.loadFrom(new File(AMBARI_STACK_DEFINITIONS, "AMBARI_INFRA_SOLR/0.1.0/properties/solr-log4j.properties.j2")).getLayout("file"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testAmbariInfraSolrGrokPatter() throws Exception { + String layout = Log4jProperties.loadFrom(new File(AMBARI_STACK_DEFINITIONS, "AMBARI_INFRA_SOLR/0.1.0/properties/solr-log4j.properties.j2")).getLayout("file"); + testServiceLog("infra_solr", layout, inputConfigTemplate( + new File(AMBARI_STACK_DEFINITIONS, "AMBARI_INFRA_SOLR/0.1.0/package/templates/input.config-ambari-infra.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java new file mode 100644 index 0000000..34f17c7 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AmbariLogPatternIT.java @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.Date; +import java.util.Map; + +import org.junit.BeforeClass; +import org.junit.Test; + +public class AmbariLogPatternIT extends PatternITBase { + + public static File AMBARI_CONF; + + @BeforeClass + public static void setupAmbariConfig() throws Exception { + setupGlobal(); + AMBARI_CONF = new File(AMBARI_FOLDER, Paths.get("ambari-server", "conf", "unix").toString()); + } + + @Test + public void testAmbariAgentLogEntry() throws Exception { + // given + String logEntry = "INFO 2018-05-02 09:29:12,359 DataCleaner.py:39 - Data cleanup thread started"; + // when + Map<String, Object> result = testLogEntry(logEntry, "ambari_agent", ambariInputConfigTemplate()); + // then + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("ambari_agent")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("Data cleanup thread started")); + assertThat(result.get("file"), is("DataCleaner.py")); + assertThat(result.get("line_number"), is("39")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 29, 12, 359000000))); + } + + @Test + public void testAmbariAgentMultilineLogEntry() throws Exception { + // given + String logEntry = "INFO 2018-05-02 09:31:52,227 RecoveryManager.py:572 - RecoverConfig = {u'components': u'',\n" + + " u'maxCount': u'6',\n" + + " u'maxLifetimeCount': u'1024',\n" + + " u'retryGap': u'5',\n" + + " u'type': u'AUTO_START',\n" + + " u'windowInMinutes': u'60'}"; + // when + Map<String, Object> result = testLogEntry(logEntry, "ambari_agent", ambariInputConfigTemplate()); + // then + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("ambari_agent")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("RecoverConfig = {u'components': u'',\n" + + " u'maxCount': u'6',\n" + + " u'maxLifetimeCount': u'1024',\n" + + " u'retryGap': u'5',\n" + + " u'type': u'AUTO_START',\n" + + " u'windowInMinutes': u'60'}")); + assertThat(result.get("file"), is("RecoveryManager.py")); + assertThat(result.get("line_number"), is("572")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 31, 52, 227000000))); + } + + @Test + public void testAmbariServerLogLayout() { + testAmbariServerLogLayout("file"); + } + + @Test + public void testAmbariAlertsLogLayout() { + testAmbariServerLogLayout("alerts"); + } + + @Test + public void testAmbariConfigChangesLogLayout() { + testAmbariServerLogLayout("configchange"); + } + + @Test + public void testAmbariDbCheckLogLayout() { + testAmbariServerLogLayout("dbcheckhelper"); + } + + public void testAmbariServerLogLayout(String appenderName) { + String layout = Log4jProperties.loadFrom(new File(AMBARI_CONF, "log4j.properties")).getLayout(appenderName); + assertThatDateIsISO8601(layout); + } + + @Test + public void testAmbariServerLog() throws Exception { + testAmbariServerLog("file", "ambari_server"); + } + + @Test + public void testAmbariConfigChangesLog() throws Exception { + testAmbariServerLog("configchange", "ambari_config_changes"); + } + + @Test + public void testAmbariDBCheckLog() throws Exception { + testAmbariServerLog("dbcheckhelper", "ambari_server_check_database"); + } + + public void testAmbariServerLog(String appenderName, String logId) throws Exception { + String layout = Log4jProperties.loadFrom(new File(AMBARI_CONF, "log4j.properties")).getLayout(appenderName); + testServiceLog(logId, layout, ambariInputConfigTemplate()); + } + + private String ambariInputConfigTemplate() throws IOException { + return inputConfigTemplate( + new File(AMBARI_STACK_DEFINITIONS, "LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2")); + } + + @Test + public void testAmbariAlertsLog() throws Exception { + // given + String layout = Log4jProperties.loadFrom(new File(AMBARI_CONF, "log4j.properties")).getLayout("alerts"); + String logEntry = generateLogEntry(layout); + // when + Map<String, Object> result = testLogEntry(logEntry, "ambari_alerts", ambariInputConfigTemplate()); + // then + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("ambari_alerts")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message").toString().contains("This is a test message"), is(true)); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime.toLocalDate(), is(LocalDate.now())); + } + + @Test + public void testAmbariEclipseLinkEntry() throws Exception { + // given + String logEntry = "[EL Info]: 2018-05-02 09:27:17.79--ServerSession(1657512321)-- EclipseLink, version: Eclipse Persistence Services - 2.6.2.v20151217-774c696"; + // when + Map<String, Object> result = testLogEntry(logEntry, "ambari_eclipselink", ambariInputConfigTemplate()); + // then + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("ambari_eclipselink")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("--ServerSession(1657512321)-- EclipseLink, version: Eclipse Persistence Services - 2.6.2.v20151217-774c696")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 27, 17, 79000000))); + + } + + @Test + public void testAmbariAuditLogEntry() throws Exception { + // given + String logEntry = "2018-05-02T09:28:10.302Z, User(null), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" + + "), Status(Failed), Reason(Authentication required), Consecutive failures(UNKNOWN USER)\n" + + "2018-05-02T09:28:10.346Z, User(admin), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" + + " Ambari: Ambari Administrator\n" + + "), Status(Success)"; + // when + Map<String, Object> result = testLogEntry(logEntry, "ambari_audit", ambariInputConfigTemplate()); + // then + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("ambari_audit")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("User(null), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" + + "), Status(Failed), Reason(Authentication required), Consecutive failures(UNKNOWN USER)\n" + + "2018-05-02T09:28:10.346Z, User(admin), RemoteIp(192.175.27.2), Operation(User login), Roles(\n" + + " Ambari: Ambari Administrator\n" + + "), Status(Success)")); + Date logTime = (Date) result.get("evtTime"); + ZonedDateTime localDateTime = ZonedDateTime.ofInstant(logTime.toInstant(), ZoneId.of("Z")); + assertThat(localDateTime, is(ZonedDateTime.parse("2018-05-02T09:28:10.302Z"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java new file mode 100644 index 0000000..7aca100 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/AtlasLogPatternIT.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class AtlasLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testAtlasLogLayout() { + String layout = Log4jXml.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "ATLAS", "configuration", "atlas-log4j.xml").toString())).getLayout("FILE"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testAtlas() throws Exception { + String layout = Log4jXml.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "ATLAS", "configuration", "atlas-log4j.xml").toString())).getLayout("FILE"); + + testServiceLog("atlas_app", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER,"ATLAS/package/templates/input.config-atlas.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java new file mode 100644 index 0000000..d8bee9e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HBaseLogPatternIT.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class HBaseLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testHBaseLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "HBASE", "configuration", "hbase-log4j.xml").toString())).getLayout("RFA"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testHBase() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "HBASE", "configuration", "hbase-log4j.xml").toString())).getLayout("RFA"); + + testServiceLog("hbase_master", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HBASE/package/templates/input.config-hbase.json.j2"))); + testServiceLog("hbase_regionserver", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HBASE/package/templates/input.config-hbase.json.j2"))); +// testServiceLog("hbase_phoenix_server", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HBASE/package/templates/input.config-hbase.json.j2"))); + } + +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java new file mode 100644 index 0000000..981b0e2 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HDFSLogPatternIT.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class HDFSLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testHDFSLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "HDFS", "configuration", "hdfs-log4j.xml").toString())).getLayout("RFA"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testHDFS() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "HDFS", "configuration", "hdfs-log4j.xml").toString())).getLayout("RFA"); + + testServiceLog("hdfs_namenode", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "HDFS/package/templates/input.config-hdfs.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java new file mode 100644 index 0000000..66efebe --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HdfsAuditLogPatternIT.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.Map; + +import org.apache.log4j.PatternLayout; +import org.junit.Before; +import org.junit.Test; + +public class HdfsAuditLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testHDFSAudit() throws Exception { + // given + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "HDFS", "configuration", "hdfs-log4j.xml").toString())).getLayout("RFAS"); + listAppender.setLayout(new PatternLayout(layout)); + listAppender.activateOptions(); + + // when + LOG.info("allowed=true\tugi=hdfs (auth:SIMPLE)\tip=/192.168.73.101\tcmd=getfileinfo\tsrc=/user\tdst=null\tperm=null\tproto=rpc"); + + // then + String logEntry = listAppender.getLogList().get(0); + Map<String, Object> result = testLogEntry(logEntry, "hdfs_audit", inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "HDFS/package/templates/input.config-hdfs.json.j2"))); + + assertAuditLog(result); + } + + private void assertAuditLog(Map<String, Object> resultEntry) { + assertThat(resultEntry.isEmpty(), is(false)); + assertThat(resultEntry.get("logType"), is("HDFSAudit")); + assertThat(resultEntry.get("cluster"), is(CLUSTER)); + assertThat(resultEntry.get("dst"), is("null")); + assertThat(resultEntry.get("perm"), is("null")); + assertThat(resultEntry.get("event_count"), is(1)); + assertThat(resultEntry.get("repo"), is("hdfs")); + assertThat(resultEntry.get("reqUser"), is("hdfs")); + assertThat(resultEntry.get("type"), is("hdfs_audit")); + assertThat(resultEntry.get("level"), is("INFO")); + assertThat(resultEntry.containsKey("seq_num"), is(true)); + assertThat(LOG.getName().contains(resultEntry.get("logger_name").toString()), is(true)); + assertThat(resultEntry.containsKey("id"), is(true)); + assertThat(resultEntry.get("authType"), is("SIMPLE")); + assertThat(resultEntry.get("action"), is("getfileinfo")); + assertThat(resultEntry.containsKey("message_md5"), is(true)); + assertThat(resultEntry.containsKey("event_md5"), is(true)); + assertThat(resultEntry.containsKey("ip"), is(true)); + assertThat(resultEntry.containsKey("host"), is(true)); + Date logTime = (Date) resultEntry.get("evtTime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime.toLocalDate(), is(LocalDate.now())); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java new file mode 100644 index 0000000..7793485 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/HiveLogPatterntIT.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.File; + +import org.junit.Test; + +public class HiveLogPatterntIT extends PatternITBase { +// TODO: use hdp_ambari_definitions + @Test + public void testHiveLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testHiveServer2() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA"); + testServiceLog("hive_hiveserver2", layout, inputConfigTemplate(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2"))); + } + + @Test + public void testHiveMetastore() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/configuration/hive-log4j.xml")).getLayout("DRFA"); + testServiceLog("hive_metastore", layout, inputConfigTemplate(new File(AMBARI_STACK_DEFINITIONS, "HIVE/0.12.0.2.0/package/templates/input.config-hive.json.j2"))); + } +} + diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/JinjaFunctions.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/JinjaFunctions.java new file mode 100644 index 0000000..db000b6 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/JinjaFunctions.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +public class JinjaFunctions { + public static Object defaultFunc(Object value, Object defaultValue) { + if (value == null) + return defaultValue; + return value; + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java new file mode 100644 index 0000000..aaf4e46 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KafkaLogPatternIT.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class KafkaLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testKafkaRequestAppenderLayout() { + testKafkaAppenderLayout("requestAppender"); + } + + @Test + public void testKafkaControllerAppenderLayout() { + testKafkaAppenderLayout("controllerAppender"); + } + + @Test + public void testKafkaLogCleanerAppenderLayout() { + testKafkaAppenderLayout("cleanerAppender"); + } + + @Test + public void testKafkaStateChangeAppenderLayout() { + testKafkaAppenderLayout("stateChangeAppender"); + } + + @Test + public void testKafkaServerAppenderLayout() { + testKafkaAppenderLayout("kafkaAppender"); + } + + private void testKafkaAppenderLayout(String appender) { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "KAFKA", "configuration", "kafka-log4j.xml").toString())).getLayout(appender); + assertThatDateIsISO8601(layout); + } + + @Test + public void testKafkaRequestAppender() throws Exception { + testKafka("requestAppender", "kafka_request"); + } + + @Test + public void testKafkaControllerAppender() throws Exception { + testKafka("controllerAppender", "kafka_controller"); + } + + @Test + public void testKafkaLogCleanerAppender() throws Exception { + testKafka("cleanerAppender", "kafka_logcleaner"); + } + + @Test + public void testKafkaStateChangeAppender() throws Exception { + testKafka("stateChangeAppender", "kafka_statechange"); + } + + @Test + public void testKafkaServerAppender() throws Exception { + testKafka("kafkaAppender", "kafka_server"); + } + + private void testKafka(String appender, String logId) throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "KAFKA", "configuration", "kafka-log4j.xml").toString())).getLayout(appender); + + testServiceLog(logId, layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "KAFKA/package/templates/input.config-kafka.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java new file mode 100644 index 0000000..baad36f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/KnoxLogPatternIT.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class KnoxLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testKnoxGatewayAppenderLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "KNOX", "configuration", "gateway-log4j.xml").toString())).getLayout("drfa"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testKnoxLdapAppenderLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "KNOX", "configuration", "ldap-log4j.xml").toString())).getLayout("drfa"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testKnoxGateway() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "KNOX", "configuration", "gateway-log4j.xml").toString())).getLayout("drfa"); + + testServiceLog("knox_gateway", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "KNOX/package/templates/input.config-knox.json.j2"))); + } + + @Test + public void testKnoxLdap() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "KNOX", "configuration", "ldap-log4j.xml").toString())).getLayout("drfa"); + + testServiceLog("knox_ldap", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "KNOX/package/templates/input.config-knox.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/LayoutQuery.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/LayoutQuery.java new file mode 100644 index 0000000..0400a60 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/LayoutQuery.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +public interface LayoutQuery { + String query(String parameterName); +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ListAppender.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ListAppender.java new file mode 100644 index 0000000..658b8f6 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ListAppender.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Layout; +import org.apache.log4j.WriterAppender; +import org.apache.log4j.spi.LoggingEvent; + +public class ListAppender extends AppenderSkeleton { + + private final List<String> logList; + + public ListAppender() { + logList = new ArrayList<>(); + } + + @Override + protected void append(LoggingEvent event) { + StringWriter stringWriter = new StringWriter(); + WriterAppender writerAppender = new WriterAppender(layout, stringWriter); + writerAppender.append(event); + logList.add(stringWriter.toString()); + } + + @Override + public void close() { + + } + + @Override + public boolean requiresLayout() { + return true; + } + + public List<String> getLogList() { + return logList; + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jContent.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jContent.java new file mode 100644 index 0000000..fabe11e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jContent.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.File; + +public interface Log4jContent { + String loadContent(); +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jProperties.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jProperties.java new file mode 100644 index 0000000..d031f8d --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jProperties.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.File; +import java.io.IOException; +import java.io.StringReader; +import java.io.UncheckedIOException; +import java.nio.charset.Charset; +import java.util.Properties; + +import org.apache.commons.io.FileUtils; + +public class Log4jProperties { + public static Log4jProperties loadFrom(File file) { + return new Log4jProperties(() -> { + try { + return FileUtils.readFileToString(file, Charset.defaultCharset()); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + public static Log4jProperties unwrapFrom(File file) { + return new Log4jProperties(new StackDefContent(file, "content")); + } + + public static Log4jProperties unwrapFrom(File file, String propertyName) { + return new Log4jProperties(new StackDefContent(file, propertyName)); + } + + private final Log4jContent content; + + public Log4jProperties(Log4jContent content) { + this.content = content; + } + + public String getLayout(String appenderName) { + Properties properties = new Properties(); + try (StringReader reader = new StringReader(content.loadContent())) { + properties.load(reader); + return properties.getProperty("log4j.appender." + appenderName + ".layout.ConversionPattern"); + } + catch (IOException ex) { + throw new UncheckedIOException(ex); + } + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXml.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXml.java new file mode 100644 index 0000000..4b0b9e2 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXml.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.apache.ambari.logsearch.patterns.StackDefContent.DOCUMENT_BUILDER_FACTORY; +import static org.apache.ambari.logsearch.patterns.StackDefContent.X_PATH_FACTORY; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.InputStream; +import java.nio.charset.Charset; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathConstants; +import javax.xml.xpath.XPathExpression; + +import org.w3c.dom.Document; + +public class Log4jXml { + public static Log4jXml unwrapFrom(File file) { + return unwrapFrom(file, "content"); + } + + public static Log4jXml unwrapFrom(File file, String propertyName) { + return new Log4jXml( + new StackDefContent(file, propertyName), + (appenderName) -> "/configuration/appender[@name='" + appenderName + "']/layout/param[@name='ConversionPattern']/@value"); + } + + private final Log4jContent content; + private final LayoutQuery layoutQuery; + + public Log4jXml(Log4jContent content, LayoutQuery layoutQuery) { + this.content = content; + this.layoutQuery = layoutQuery; + } + + public String getLayout(String appenderName) { + return getLayout(content, layoutQuery, appenderName); + } + + public static String getLayout(Log4jContent content, LayoutQuery layoutQuery, String parameterName) { + try { + DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + Document doc; + try (InputStream stringReader = new ByteArrayInputStream(content.loadContent().getBytes(Charset.defaultCharset()))) { + doc = builder.parse(stringReader); + } + XPath xpath = X_PATH_FACTORY.newXPath(); + XPathExpression expr = xpath.compile(layoutQuery.query(parameterName)); + return (String) expr.evaluate(doc, XPathConstants.STRING); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXmlProperties.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXmlProperties.java new file mode 100644 index 0000000..ada5f2a --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Log4jXmlProperties.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.File; + +public class Log4jXmlProperties { + public static Log4jXmlProperties unwrapFrom(File file) { + return unwrapFrom(file, "content"); + } + + public static Log4jXmlProperties unwrapFrom(File file, String contentPropertyName) { + return new Log4jXmlProperties( + new StackDefContent(file, contentPropertyName), + (xmlPropertyName) -> "/configuration/properties/property[@name='" + xmlPropertyName + "']/text()"); + } + + public Log4jXmlProperties(Log4jContent content, LayoutQuery layoutQuery) { + this.content = content; + this.layoutQuery = layoutQuery; + } + + private final Log4jContent content; + private final LayoutQuery layoutQuery; + + public String getLayout(String propertyName) { + return Log4jXml.getLayout(content, layoutQuery, propertyName); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java new file mode 100644 index 0000000..80483b1 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/MetricsLogPatternIT.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class MetricsLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testMetricsLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "AMBARI_METRICS", "configuration", "ams-log4j.xml").toString())).getLayout("file"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testMetrics() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "AMBARI_METRICS", "configuration", "ams-log4j.xml").toString())).getLayout("file"); + + testServiceLog("ams_collector", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "AMBARI_METRICS/package/templates/input.config-ambari-metrics.json.j2"))); +// testServiceLog("ams_monitor", layout, Paths.get("AMBARI_METRICS", "package", "templates", "input.config-ambari-metrics.json.j2")); + } + +// @Test +// public void testMetricsGrafana() throws Exception { +// testServiceLog("ams_grafana", "%d{ISO8601} %-5p [%t] %c{2}: %m%n", Paths.get("AMBARI_METRICS", "package", "templates", "input.config-ambari-metrics.json.j2")); +// } + + @Test + public void testMetricsHBaseLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "AMBARI_METRICS", "configuration", "ams-hbase-log4j.xml").toString())).getLayout("DRFA"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testMetricsHBase() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "AMBARI_METRICS", "configuration", "ams-hbase-log4j.xml").toString())).getLayout("DRFA"); + + testServiceLog("ams_hbase_master", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "AMBARI_METRICS/package/templates/input.config-ambari-metrics.json.j2"))); + testServiceLog("ams_hbase_regionserver", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "AMBARI_METRICS/package/templates/input.config-ambari-metrics.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java new file mode 100644 index 0000000..9558ed9 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/PatternITBase.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import org.apache.ambari.logfeeder.common.LogEntryParseTester; +import org.apache.ambari.logsearch.config.zookeeper.model.inputconfig.impl.InputAdapter; +import org.apache.commons.io.FileUtils; +import org.apache.log4j.Logger; +import org.apache.log4j.PatternLayout; +import org.junit.Before; +import org.junit.BeforeClass; + +import com.google.gson.JsonElement; +import com.google.gson.JsonParser; +import com.hubspot.jinjava.Jinjava; +import com.hubspot.jinjava.lib.fn.ELFunctionDefinition; + +public class PatternITBase { + protected final static Logger LOG = Logger.getLogger(PatternITBase.class); + + public static final String HDP_AMBARI_DEFINITIONS_PATH = "/Users/kkasa/project/hdp_ambari_definitions/"; + public static final File HDP_AMBARI_DEFINITIONS = new File( + isBlank(System.getProperty("hdp.ambari.definitions.path")) ? HDP_AMBARI_DEFINITIONS_PATH : System.getProperty("hdp.ambari.definitions.path")); + public static File AMBARI_STACK_DEFINITIONS; + public static File AMBARI_FOLDER; + public static final File HDP_SERVICES_FOLDER = new File(HDP_AMBARI_DEFINITIONS, Paths.get( "src", "main", "resources", "stacks", "HDP", "3.0", "services").toString()); + public static final String CLUSTER = "cl1"; + public static final String GLOBAL_CONFIG = "[\n" + + " {\n" + + " \"add_fields\": {\n" + + " \"cluster\": \""+ CLUSTER +"\"\n" + + " },\n" + + " \"source\": \"file\",\n" + + " \"tail\": \"true\",\n" + + " \"gen_event_md5\": \"true\"\n" + + " }\n" + + "]"; + + private Jinjava jinjava = new Jinjava(); + protected ListAppender listAppender; + + + @BeforeClass + public static void setupGlobal() throws Exception { + URL location = PatternITBase.class.getProtectionDomain().getCodeSource().getLocation(); + + AMBARI_FOLDER = new File(new File(location.toURI()).getParentFile().getParentFile().getParentFile().getParent()); + AMBARI_STACK_DEFINITIONS = new File(AMBARI_FOLDER, Paths.get("ambari-server", "src", "main", "resources", "common-services").toString()); + } + + @Before + public void setUp() throws Exception { + JsonParser jsonParser = new JsonParser(); + JsonElement globalConfigJsonElement = jsonParser.parse(GLOBAL_CONFIG); + + InputAdapter.setGlobalConfigs(globalConfigJsonElement.getAsJsonArray()); + jinjava.getGlobalContext().registerFunction(new ELFunctionDefinition("", "default", JinjaFunctions.class, "defaultFunc", Object.class, Object.class)); + + listAppender = new ListAppender(); + LOG.addAppender(listAppender); + } + + protected String inputConfigTemplate(File templateFile) throws IOException { + return FileUtils.readFileToString(templateFile, Charset.defaultCharset()); + } + + protected void testServiceLog(String logId, String log4jLayout, String inputConfigTemplate) throws Exception { + String logEntry = generateLogEntry(log4jLayout); + Map<String, Object> resultEntry = testLogEntry(logEntry, logId, inputConfigTemplate); + assertServiceLog(logId, resultEntry); + } + + protected String generateLogEntry(String log4jLayout) { + return generateLogEntry(log4jLayout, "This is a test message"); + } + + protected String generateLogEntry(String log4jLayout, String message) { + listAppender.setLayout(new PatternLayout(log4jLayout)); + listAppender.activateOptions(); + LOG.error(message, new Exception("TEST")); + return listAppender.getLogList().get(0); + } + + protected Map<String, Object> testLogEntry(String logEntry, String logId, String inputConfigTemplate) throws Exception { + String grokFilter = jinjava.render(inputConfigTemplate, new HashMap<>()); + + LogEntryParseTester tester = new LogEntryParseTester(logEntry, grokFilter, GLOBAL_CONFIG, logId); + return tester.parse(); + } + + private void assertServiceLog(String logId, Map<String, Object> resultEntry) { + assertThat(resultEntry.isEmpty(), is(false)); + assertThat(resultEntry.get("cluster"), is(CLUSTER)); + assertThat(resultEntry.get("level"), is("ERROR")); + assertThat(resultEntry.get("event_count"), is(1)); + assertThat(resultEntry.get("type"), is(logId)); + assertThat(resultEntry.containsKey("seq_num"), is(true)); +// assertThat(LOG.getName().contains(resultEntry.get("logger_name").toString()), is(true)); + assertThat(resultEntry.containsKey("id"), is(true)); + assertThat(resultEntry.containsKey("message_md5"), is(true)); + assertThat(resultEntry.containsKey("event_md5"), is(true)); + assertThat(resultEntry.containsKey("ip"), is(true)); + assertThat(resultEntry.containsKey("host"), is(true)); + assertThat(resultEntry.get("log_message").toString().contains("This is a test message"), is(true)); + assertThat(resultEntry.get("log_message").toString().contains("java.lang.Exception: TEST"), is(true)); + Date logTime = (Date) resultEntry.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime.toLocalDate(), is(LocalDate.now())); + } + + protected void assertThatDateIsISO8601(String layout) { + assertThat(layout.toLowerCase().contains("%d{iso8601}"), is(true)); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java new file mode 100644 index 0000000..dc05611 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/RangerLogPatternIT.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class RangerLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testRangerAdminLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "RANGER", "configuration", "admin-log4j.xml").toString())).getLayout("xa_log_appender"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testRangerUserSynchLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "RANGER", "configuration", "usersync-log4j.xml").toString())).getLayout("logFile"); + assertThat(layout.contains("%d{dd MMM yyyy HH:mm:ss}"), is(true)); + } + + @Test + public void testRangerAdminLog() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "RANGER", "configuration", "admin-log4j.xml").toString())).getLayout("xa_log_appender"); + + testServiceLog("ranger_admin", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "RANGER/package/templates/input.config-ranger.json.j2"))); + } + + @Test + public void testRangerUserSynchLog() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "RANGER", "configuration", "usersync-log4j.xml").toString())).getLayout("logFile"); + + testServiceLog("ranger_usersync", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "RANGER/package/templates/input.config-ranger.json.j2"))); + } + + @Test + public void testRangerKMSLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "RANGER_KMS", "configuration", "kms-log4j.xml").toString())).getLayout("kms"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testRangerKMSLog() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "RANGER_KMS", "configuration", "kms-log4j.xml").toString())).getLayout("kms"); + + testServiceLog("ranger_kms", layout, inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "RANGER_KMS/package/templates/input.config-ranger-kms.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java new file mode 100644 index 0000000..41b7ff9 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/Spark2LogPatternIT.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class Spark2LogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testSpark2LogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "SPARK2", "configuration", "spark2-log4j-properties.xml").toString())).getLayout("console"); + assertThat(layout.contains("%d{yy/MM/dd HH:mm:ss}"), is(true)); + } + + @Test + public void testSpark2Livy2LogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "SPARK2", "configuration", "livy2-log4j-properties.xml").toString())).getLayout("console"); + assertThat(layout.contains("%d{yy/MM/dd HH:mm:ss}"), is(true)); + } + + @Test + public void testSpark2Log() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "SPARK2", "configuration", "spark2-log4j-properties.xml").toString())).getLayout("console"); + + testServiceLog("spark2_jobhistory_server", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "SPARK2/package/templates/input.config-spark2.json.j2"))); + } + + @Test + public void testSpark2Livy2Log() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "SPARK2", "configuration", "livy2-log4j-properties.xml").toString())).getLayout("console"); + + testServiceLog("livy2_server", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "SPARK2/package/templates/input.config-spark2.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StackDefContent.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StackDefContent.java new file mode 100644 index 0000000..5ae33f8 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StackDefContent.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import java.io.File; +import java.io.FileInputStream; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.xpath.XPath; +import javax.xml.xpath.XPathExpression; +import javax.xml.xpath.XPathFactory; + +import org.w3c.dom.Document; + +public class StackDefContent implements Log4jContent { + public static final XPathFactory X_PATH_FACTORY = XPathFactory.newInstance(); + public static final DocumentBuilderFactory DOCUMENT_BUILDER_FACTORY = DocumentBuilderFactory.newInstance(); + + private final File file; + private final String propertyName; + + public StackDefContent(File file, String propertyName) { + this.file = file; + this.propertyName = propertyName; + } + + @Override + public String loadContent() { + try { + DocumentBuilder builder = DOCUMENT_BUILDER_FACTORY.newDocumentBuilder(); + Document doc; + try (FileInputStream fileInputStream = new FileInputStream(file)) { + doc = builder.parse(fileInputStream); + } + XPath xpath = X_PATH_FACTORY.newXPath(); + XPathExpression expr = xpath.compile("/configuration/property[name/text()='" + propertyName + "']/value/text()"); + return expr.evaluate(doc); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java new file mode 100644 index 0000000..bf1b392 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/StormLogPatternIT.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +public class StormLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testStormClusterLogLayout() { + String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "STORM", "configuration", "storm-cluster-log4j.xml").toString())).getLayout("pattern"); + assertThat(layout.contains("yyyy-MM-dd HH:mm:ss.SSS"), is(true)); + } + + @Test + public void testStormWorkerLogLayout() { + String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "STORM", "configuration", "storm-worker-log4j.xml").toString())).getLayout("pattern"); + assertThat(layout.contains("yyyy-MM-dd HH:mm:ss.SSS"), is(true)); + } + + @Test + public void testStormLog() throws Exception { + String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "STORM", "configuration", "storm-cluster-log4j.xml").toString())).getLayout("pattern"); + + testServiceLog("storm_drpc", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "STORM/package/templates/input.config-storm.json.j2"))); + } + + @Test + public void testStormWorkerLog() throws Exception { + String layout = Log4jXmlProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "STORM", "configuration", "storm-worker-log4j.xml").toString())).getLayout("pattern"); + + testServiceLog("storm_worker", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "STORM/package/templates/input.config-storm.json.j2"))); + } + + @Test + public void testStormWorkerLogEntry() throws Exception { + String logEntry = "2018-05-04 05:10:00.120 o.a.s.d.executor main [INFO] Loaded executor tasks count:[5 5]"; + Map<String, Object> result = testLogEntry(logEntry, "storm_worker", inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "STORM/package/templates/input.config-storm.json.j2"))); + + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("level"), is("INFO")); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("storm_worker")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("Loaded executor tasks count:[5 5]")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime.toLocalDate(), is(LocalDate.now())); + + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java new file mode 100644 index 0000000..a2d1069 --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/YarnLogPatternIT.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.Is.is; +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.Date; +import java.util.Map; + +import org.junit.Before; +import org.junit.Test; + +public class YarnLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testYarnJobSummaryLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "YARN", "configuration", "yarn-log4j.xml").toString())).getLayout("RMSUMMARY"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testYarnJobSummaryLog() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "YARN", "configuration", "yarn-log4j.xml").toString())).getLayout("RMSUMMARY"); + + testServiceLog("yarn_jobsummary", layout, + inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2"))); + } + + @Test + public void testYarnNodemanagerLogEntry() throws Exception { + Map<String, Object> result = testLogEntry("2018-05-02 09:43:46,898 INFO zookeeper.ZooKeeper (Environment.java:logEnv(100)) - Client environment:zookeeper.version=3.4.6-1173--1,\n" + + " built on 04/10/2018 11:42 GMT", + "yarn_nodemanager", + inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2"))); + + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("level"), is("INFO")); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("yarn_nodemanager")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.get("logger_name"), is("zookeeper.ZooKeeper ")); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("Client environment:zookeeper.version=3.4.6-1173--1,\n built on 04/10/2018 11:42 GMT")); + assertThat(result.get("line_number"), is("100")); + assertThat(result.get("file"), is("Environment.java")); + assertThat(result.get("method"), is("logEnv")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 43, 46, 898000000))); + } + + @Test + public void testYarnResourcemanagerLogEntry() throws Exception { + Map<String, Object> result = testLogEntry("2018-05-02 09:41:43,917 INFO placement.UserGroupMappingPlacementRule (UserGroupMappingPlacementRule.java:get(232)) - Initialized queue mappings, override: false", + "yarn_resourcemanager", + inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2"))); + + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("level"), is("INFO")); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("yarn_resourcemanager")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.get("logger_name"), is("placement.UserGroupMappingPlacementRule ")); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("Initialized queue mappings, override: false")); + assertThat(result.get("line_number"), is("232")); + assertThat(result.get("file"), is("UserGroupMappingPlacementRule.java")); + assertThat(result.get("method"), is("get")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 9, 41, 43, 917000000))); + } + + @Test + public void testYarnTimelineServerLogEntry() throws Exception { + Map<String, Object> result = testLogEntry("2018-05-02 10:36:27,868 INFO timeline.RollingLevelDB (RollingLevelDB.java:evictOldDBs(345)) - Evicting entity-ldb DBs scheduled for eviction", + "yarn_timelineserver", + inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2"))); + + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("level"), is("INFO")); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("yarn_timelineserver")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.get("logger_name"), is("timeline.RollingLevelDB ")); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("Evicting entity-ldb DBs scheduled for eviction")); + assertThat(result.get("line_number"), is("345")); + assertThat(result.get("file"), is("RollingLevelDB.java")); + assertThat(result.get("method"), is("evictOldDBs")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 36, 27, 868000000))); + } + + @Test + public void testYarnHistoryServerLogEntry() throws Exception { + Map<String, Object> result = testLogEntry("2018-05-02 10:02:54,215 INFO webapp.View (HsJobsBlock.java:render(74)) - Getting list of all Jobs.", + "yarn_historyserver", + inputConfigTemplate(new File(HDP_SERVICES_FOLDER, "YARN/package/templates/input.config-yarn.json.j2"))); + + assertThat(result.isEmpty(), is(false)); + assertThat(result.get("cluster"), is(CLUSTER)); + assertThat(result.get("level"), is("INFO")); + assertThat(result.get("event_count"), is(1)); + assertThat(result.get("type"), is("yarn_historyserver")); + assertThat(result.get("logger_name"), is("webapp.View ")); + assertThat(result.containsKey("seq_num"), is(true)); + assertThat(result.containsKey("id"), is(true)); + assertThat(result.containsKey("message_md5"), is(true)); + assertThat(result.containsKey("event_md5"), is(true)); + assertThat(result.containsKey("ip"), is(true)); + assertThat(result.containsKey("host"), is(true)); + assertThat(result.get("log_message"), is("Getting list of all Jobs.")); + assertThat(result.get("line_number"), is("74")); + assertThat(result.get("file"), is("HsJobsBlock.java")); + assertThat(result.get("method"), is("render")); + Date logTime = (Date) result.get("logtime"); + LocalDateTime localDateTime = LocalDateTime.ofInstant(logTime.toInstant(), ZoneId.systemDefault()); + assertThat(localDateTime, is(LocalDateTime.of(2018, 5, 2, 10, 02, 54, 215000000))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java new file mode 100644 index 0000000..6db6e7e --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZeppelinLogPatternIT.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class ZeppelinLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testZeppelinLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "ZEPPELIN", "configuration", "zeppelin-log4j-properties.xml").toString()), + "log4j_properties_content").getLayout("dailyfile"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testZeppelinLog() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "ZEPPELIN", "configuration", "zeppelin-log4j-properties.xml").toString()), + "log4j_properties_content").getLayout("dailyfile"); + + testServiceLog("zeppelin", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "ZEPPELIN/package/templates/input.config-zeppelin.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java new file mode 100644 index 0000000..1e00a0f --- /dev/null +++ b/ambari-logsearch/ambari-logsearch-it/src/test/java/org/apache/ambari/logsearch/patterns/ZookeeperLogPatternIT.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.ambari.logsearch.patterns; + +import static org.junit.Assume.assumeTrue; + +import java.io.File; +import java.nio.file.Paths; + +import org.junit.Before; +import org.junit.Test; + +public class ZookeeperLogPatternIT extends PatternITBase { + + @Override + @Before + public void setUp() throws Exception { + super.setUp(); + assumeTrue(HDP_SERVICES_FOLDER.exists()); + } + + @Test + public void testZookeeperLogLayout() { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "ZOOKEEPER", "configuration", "zookeeper-log4j.xml").toString())).getLayout("ROLLINGFILE"); + assertThatDateIsISO8601(layout); + } + + @Test + public void testZookeeperLog() throws Exception { + String layout = Log4jProperties.unwrapFrom(new File(HDP_SERVICES_FOLDER, Paths.get( + "ZOOKEEPER", "configuration", "zookeeper-log4j.xml").toString())).getLayout("ROLLINGFILE"); + + testServiceLog("zookeeper", layout, inputConfigTemplate( + new File(HDP_SERVICES_FOLDER, "ZOOKEEPER/package/templates/input.config-zookeeper.json.j2"))); + } +} diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java index b4c4f22..2a23cd7 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/ConfigHandler.java @@ -187,10 +187,10 @@ public class ConfigHandler implements InputConfigMonitor { } for (FilterDescriptor filterDescriptor : inputConfig.getFilter()) { - if ("grok".equals(filterDescriptor.getFilter())) { - // Thus ensure that the log entry passed will be parsed immediately - ((FilterGrokDescriptor)filterDescriptor).setMultilinePattern(null); - } +// if ("grok".equals(filterDescriptor.getFilter())) { +// // Thus ensure that the log entry passed will be parsed immediately +// ((FilterGrokDescriptor)filterDescriptor).setMultilinePattern(null); +// } filterConfigList.add(filterDescriptor); } loadInputs("test"); diff --git a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java index 14b6f83..9e4edcc 100644 --- a/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java +++ b/ambari-logsearch/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogEntryParseTester.java @@ -137,7 +137,8 @@ public class LogEntryParseTester { } }); input.outputLine(logEntry, new InputFileMarker(input, null, 0)); - + input.outputLine(logEntry, new InputFileMarker(input, null, 0)); + return result.isEmpty() ? ImmutableMap.of("errorMessage", (Object)"Could not parse test log entry") : result; diff --git a/ambari-server/conf/unix/log4j.properties b/ambari-server/conf/unix/log4j.properties index a5b58bc..dec1334 100644 --- a/ambari-server/conf/unix/log4j.properties +++ b/ambari-server/conf/unix/log4j.properties @@ -35,7 +35,7 @@ log4j.appender.file.File=${ambari.log.dir}/${ambari.log.file} log4j.appender.file.MaxFileSize=80MB log4j.appender.file.MaxBackupIndex=60 log4j.appender.file.layout=org.apache.log4j.PatternLayout -log4j.appender.file.layout.ConversionPattern=%d{DATE} %5p [%t] %c{1}:%L - %m%n +log4j.appender.file.layout.ConversionPattern=%d{ISO8601} %5p [%t] %c{1}:%L - %m%n # Log config changes log4j.logger.configchange=INFO,configchange diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2 b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2 index 5b8f896..2b424b1 100644 --- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2 +++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1/package/templates/input.config-kafka.json.j2 @@ -57,7 +57,7 @@ }, "log4j_format":"[%d] %p %m (%c)%n", "multiline_pattern":"^(\\[%{TIMESTAMP_ISO8601:logtime}\\])", - "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{GREEDYDATA:log_message}", + "message_pattern":"(?m)^\\[%{TIMESTAMP_ISO8601:logtime}\\]%{SPACE}%{LOGLEVEL:level}%{SPACE}%{GREEDYDATA:log_message}", "post_map_values":{ "logtime":{ "map_date":{ diff --git a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 index d948ea4..9ec651c 100644 --- a/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 +++ b/ambari-server/src/main/resources/common-services/LOGSEARCH/0.5.0/properties/input.config-ambari.json.j2 @@ -108,13 +108,13 @@ } }, - "log4j_format":"%d{DATE} %5p [%t] %c{1}:%L - %m%n", - "multiline_pattern":"^(%{USER_SYNC_DATE:logtime})", - "message_pattern":"(?m)^%{USER_SYNC_DATE:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", + "log4j_format":"%d{ISO8601} %5p [%t] %c{1}:%L - %m%n", + "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}\\[%{DATA:thread_name}\\]%{SPACE}%{JAVACLASS:logger_name}:%{INT:line_number}%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", "post_map_values":{ "logtime":{ "map_date":{ - "target_date_pattern":"dd MMM yyyy HH:mm:ss" + "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" } } diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 index a2a4841..1767c28 100644 --- a/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 +++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1/package/templates/input.config-storm.json.j2 @@ -45,19 +45,40 @@ { "type":"storm_worker", "rowtype":"service", - "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/*worker*.log" + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/*/*/worker.log", + "init_default_fields":"true", + "cache_enabled":"true", + "cache_size":"100", + "cache_dedup_interval":"1000", + "cache_key_field":"log_message", + "cache_last_dedup_enabled":"true" + }, + { + "type":"storm_worker_event", + "rowtype":"service", + "path":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/*/*/events.log", + "init_default_fields":"true", + "cache_enabled":"true", + "cache_size":"100", + "cache_dedup_interval":"1000", + "cache_key_field":"log_message", + "cache_last_dedup_enabled":"true", + "add_fields":{ + "level":"INFO" + } } ], "filter":[ { "filter":"grok", + "sort_order":1, "conditions":{ "fields":{ "type":[ - "storm_drpc", - "storm_logviewer", "storm_nimbus", "storm_supervisor", + "storm_logviewer", + "storm_drpc", "storm_ui", "storm_worker" ] @@ -65,7 +86,7 @@ }, "log4j_format":"", "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", - "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\[%{LOGLEVEL:level}\\]%{SPACE}%{SPACE}%{GREEDYDATA:log_message}", + "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{JAVACLASS:logger_name}\\s%{GREEDYDATA:thread_name}\\s\\[%{LOGLEVEL:level}\\]\\s%{GREEDYDATA:log_message}", "post_map_values":{ "logtime":{ "map_date":{ @@ -73,6 +94,54 @@ } } } + }, + { + "filter":"grok", + "sort_order":5, + "conditions":{ + "fields":{ + "type":[ + "storm_worker_event" + ] + } + }, + "log4j_format":"", + "message_pattern":"^%{TIMESTAMP_ISO8601:logtime}(!_DELIM_!<STREAMLINE_EVENT>!_DELIM_!%{DATA:sdi_streamline_component_name}!_DELIM_!%{DATA:sdi_streamline_event_id}!_DELIM_!%{DATA:sdi_streamline_root_ids}!_DELIM_!%{DATA:sdi_streamline_parent_ids}!_DELIM_!%{DATA:sdi_streamline_event_fields_and_values}!_DELIM_!%{DATA:sdi_streamline_event_headers}!_DELIM_!%{DATA:sdi_streamline_event_aux_fields_and_values})|(%{GREEDYDATA})", + "post_map_values":{ + "logtime":{ + "map_date":{ + "target_date_pattern":"yyyy-MM-dd HH:mm:ss.SSS" + } + } + } + }, + { + "filter":"grok", + "sort_order":6, + "conditions":{ + "fields":{ + "type":[ + "storm_worker_event" + ] + } + }, + "source_field":"path", + "remove_source_field":"false", + "message_pattern":"{{default('/configurations/storm-env/storm_log_dir', '/var/log/storm')}}/workers-artifacts/%{DATA:sdi_storm_topology_id}/%{DATA:sdi_storm_worker_port}/events\\.log" + }, + { + "filter":"grok", + "sort_order":7, + "conditions":{ + "fields":{ + "type":[ + "storm_worker_event" + ] + } + }, + "source_field":"sdi_storm_topology_id", + "remove_source_field":"false", + "message_pattern":"(streamline\\-%{DATA:sdi_streamline_topology_id}\\-%{DATA:sdi_streamline_topology_name}\\-[0-9]+\\-[0-9]+)|(%{DATA:sdi_storm_topology_id})" } ] -} \ No newline at end of file +} diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 index 8034843..1bab72b 100644 --- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 +++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/templates/input.config-mapreduce2.json.j2 @@ -15,14 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. #} -{ + { "input":[ { "type":"mapred_historyserver", "rowtype":"service", "path":"{{default('/configurations/mapred-env/mapred_log_dir_prefix', '/var/log/hadoop')}}/{{default('configurations/mapred-env/mapred_user', 'mapred')}}/mapred-{{default('configurations/mapred-env/mapred_user', 'mapred')}}-historyserver*.log" } - ], + ], "filter":[ { "filter":"grok", @@ -31,8 +31,8 @@ "type":[ "mapred_historyserver" ] - } - }, + } + }, "log4j_format":"%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n", "multiline_pattern":"^(%{TIMESTAMP_ISO8601:logtime})", "message_pattern":"(?m)^%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{LOGLEVEL:level}%{SPACE}%{JAVACLASS:logger_name}%{SPACE}\\(%{JAVAFILE:file}:%{JAVAMETHOD:method}\\(%{INT:line_number}\\)\\)%{SPACE}-%{SPACE}%{GREEDYDATA:log_message}", @@ -41,8 +41,8 @@ "map_date":{ "target_date_pattern":"yyyy-MM-dd HH:mm:ss,SSS" } - } - } - } - ] -} \ No newline at end of file + } + } + } + ] + } \ No newline at end of file -- To stop receiving notification emails like this one, please contact oleew...@apache.org.