Repository: logging-log4j2 Updated Branches: refs/heads/Lucene5 [created] 929cc9ba9
New branch for Lucene 5 Appender based on https://github.com/apache/logging-log4j2/pull/82 Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/929cc9ba Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/929cc9ba Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/929cc9ba Branch: refs/heads/Lucene5 Commit: 929cc9ba9edcfa811d4beba4418ffbe7e0097922 Parents: 864b7a8 Author: Gary Gregory <[email protected]> Authored: Mon Jun 5 18:22:46 2017 -0700 Committer: Gary Gregory <[email protected]> Committed: Mon Jun 5 18:22:46 2017 -0700 ---------------------------------------------------------------------- log4j-bom/pom.xml | 6 + log4j-distribution/pom.xml | 17 + log4j-lucene5/pom.xml | 262 +++++++++++++++ .../log4j/lucene5/appender/LuceneAnalyzer.java | 31 ++ .../log4j/lucene5/appender/LuceneAppender.java | 318 +++++++++++++++++++ .../lucene5/appender/LuceneIndexField.java | 114 +++++++ .../log4j/lucene5/appender/LuceneTokenizer.java | 35 ++ .../log4j/lucene5/appender/package-info.java | 23 ++ .../lucene5/appender/LuceneAppenderTest.java | 136 ++++++++ .../src/test/resources/log4j2-lucene.xml | 30 ++ pom.xml | 1 + 11 files changed, 973 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-bom/pom.xml ---------------------------------------------------------------------- diff --git a/log4j-bom/pom.xml b/log4j-bom/pom.xml index 920f6a0..a43cf54 100644 --- a/log4j-bom/pom.xml +++ b/log4j-bom/pom.xml @@ -108,6 +108,12 @@ <artifactId>log4j-liquibase</artifactId> <version>${project.version}</version> </dependency> + <!-- Apache Lucene 5 Appender --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-lucene5</artifactId> + <version>${project.version}</version> + </dependency> <!-- Scala 2.10 API --> <dependency> <groupId>org.apache.logging.log4j</groupId> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-distribution/pom.xml ---------------------------------------------------------------------- diff --git a/log4j-distribution/pom.xml b/log4j-distribution/pom.xml index e77e883..af63ce0 100644 --- a/log4j-distribution/pom.xml +++ b/log4j-distribution/pom.xml @@ -294,6 +294,23 @@ <version>${project.version}</version> <classifier>javadoc</classifier> </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-lucene5</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-lucene5</artifactId> + <version>${project.version}</version> + <classifier>sources</classifier> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-lucene5</artifactId> + <version>${project.version}</version> + <classifier>javadoc</classifier> + </dependency> </dependencies> <build> <plugins> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/pom.xml ---------------------------------------------------------------------- diff --git a/log4j-lucene5/pom.xml b/log4j-lucene5/pom.xml new file mode 100644 index 0000000..f578b6b --- /dev/null +++ b/log4j-lucene5/pom.xml @@ -0,0 +1,262 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j</artifactId> + <version>2.8.3-SNAPSHOT</version> + </parent> + <modelVersion>4.0.0</modelVersion> + + <artifactId>log4j-lucene5</artifactId> + <name>Apache Log4j Lucene 5</name> + <description> + Apache Log4j Lucene 5 appender. + </description> + <properties> + <log4jParentDir>${basedir}/..</log4jParentDir> + <docLabel>Lucene 5 Documentation</docLabel> + <projectDir>/log4j-lucene5</projectDir> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.lucene</groupId> + <artifactId>lucene-analyzers-common</artifactId> + <version>5.5.4</version> + </dependency> + <!-- Pull in useful test classes from API --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-api</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + <!-- Apache Commons Compress --> + <dependency> + <groupId>org.tukaani</groupId> + <artifactId>xz</artifactId> + <scope>test</scope> + </dependency> + <!-- Zeroconf advertiser tests --> + <dependency> + <groupId>org.jmdns</groupId> + <artifactId>jmdns</artifactId> + <version>3.5.1</version> + <scope>test</scope> + </dependency> + <!-- Log4j 1.2 tests --> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-1.2-api</artifactId> + <scope>test</scope> + </dependency> + <!-- SLF4J tests --> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-ext</artifactId> + <scope>test</scope> + </dependency> + <!-- JUnit, naturally --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-all</artifactId> + <scope>test</scope> + </dependency> + <!-- Mocking framework for use with JUnit --> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <!-- Embedded JDBC drivers for database appender tests --> + <dependency> + <groupId>org.hsqldb</groupId> + <artifactId>hsqldb</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.h2database</groupId> + <artifactId>h2</artifactId> + <scope>test</scope> + </dependency> + <!-- JPA Tests --> + <dependency> + <groupId>org.eclipse.persistence</groupId> + <artifactId>org.eclipse.persistence.jpa</artifactId> + <scope>test</scope> + </dependency> + <!-- Useful mock classes and utilities --> + <dependency> + <groupId>org.springframework</groupId> + <artifactId>spring-test</artifactId> + <scope>test</scope> + </dependency> + <!-- JPA, JNDI and JMS tests --> + <dependency> + <groupId>org.apache.activemq</groupId> + <artifactId>activemq-broker</artifactId> + <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.geronimo.specs</groupId> + <artifactId>geronimo-jms_1.1_spec</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>commons-logging</groupId> + <artifactId>commons-logging</artifactId> + <scope>test</scope> + </dependency> + <!-- Logback performance tests --> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <scope>test</scope> + </dependency> + <!-- OSGi tests --> + <dependency> + <groupId>org.eclipse.tycho</groupId> + <artifactId>org.eclipse.osgi</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.felix</groupId> + <artifactId>org.apache.felix.framework</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.codehaus.plexus</groupId> + <artifactId>plexus-utils</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <scope>test</scope> + </dependency> + <!-- GELF --> + <dependency> + <groupId>net.javacrumbs.json-unit</groupId> + <artifactId>json-unit</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.xmlunit</groupId> + <artifactId>xmlunit-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.xmlunit</groupId> + <artifactId>xmlunit-matchers</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>commons-io</groupId> + <artifactId>commons-io</artifactId> + <scope>test</scope> + </dependency> + <!-- Other --> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.commons</groupId> + <artifactId>commons-lang3</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache-extras.beanshell</groupId> + <artifactId>bsh</artifactId> + <version>2.0b5</version> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.codehaus.groovy</groupId> + <artifactId>groovy-all</artifactId> + <version>2.4.7</version> + <scope>test</scope> + </dependency> + <!-- GC-free --> + <dependency> + <groupId>com.google.code.java-allocation-instrumenter</groupId> + <artifactId>java-allocation-instrumenter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.hdrhistogram</groupId> + <artifactId>HdrHistogram</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.apache.logging.log4j</groupId> + <artifactId>log4j-core</artifactId> + <type>test-jar</type> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-remote-resources-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>process</goal> + </goals> + <configuration> + <skip>false</skip> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.felix</groupId> + <artifactId>maven-bundle-plugin</artifactId> + <configuration> + <instructions> + <Export-Package>*</Export-Package> + </instructions> + </configuration> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAnalyzer.java ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAnalyzer.java b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAnalyzer.java new file mode 100644 index 0000000..bfff51e --- /dev/null +++ b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAnalyzer.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.lucene5.appender; + +import org.apache.lucene.analysis.Analyzer; + +/** + * Tokenizes the entire stream as a single token, but case insensitive. + */ +public class LuceneAnalyzer extends Analyzer { + + @SuppressWarnings("resource") + @Override + protected TokenStreamComponents createComponents(final String fieldName) { + return new TokenStreamComponents(new LuceneTokenizer()); + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAppender.java ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAppender.java b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAppender.java new file mode 100644 index 0000000..8964fb1 --- /dev/null +++ b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneAppender.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.lucene5.appender; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.file.Paths; +import java.util.Calendar; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.appender.AppenderLoggingException; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.Scheduled; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.validation.constraints.Required; +import org.apache.logging.log4j.core.util.CronExpression; +import org.apache.logging.log4j.util.Strings; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.LongField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.store.FSDirectory; + +/** + * This Appender writes logging events to a Lucene index library. It takes a + * list of {@link IndexField} with which determines which fields are written to + * the index library. + * + * <pre> + * <Lucene5 name="lucene" ignoreExceptions="true" target="/target/lucene/index"> + * <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level %class{36} %L %M + * - %msg%xEx%n"/> + * + * <IndexField name="time" pattern="%d{UNIX_MILLIS}" type="LongField"/> + * <IndexField name="level" pattern="%-5level" /> + * <IndexField name="content" pattern="%d{HH:mm:ss.SSS} %-5level %class{36} %L + * %M - %msg%xEx%n"/> + * </Lucene5> + * </pre> + */ +@Plugin(name = "Lucene5", category = Node.CATEGORY, elementType = Appender.ELEMENT_TYPE, printObject = true) +@Scheduled +public class LuceneAppender extends AbstractAppender { + public static class Builder<B extends Builder<B>> extends AbstractAppender.Builder<B> + implements org.apache.logging.log4j.core.util.Builder<LuceneAppender> { + + @PluginConfiguration + private Configuration configuration; + + @PluginBuilderAttribute + private Integer expirySeconds; + + @PluginElement("IndexField") + @Required(message = "No IndexField provided") + private LuceneIndexField[] indexField; + + @PluginBuilderAttribute + @Required(message = "No target provided") + private String target; + + @Override + public LuceneAppender build() { + return new LuceneAppender(getName(), isIgnoreExceptions(), getFilter(), this.getLayout(), this.target, + this.expirySeconds, this.indexField, this.configuration); + } + + @Override + public B setConfiguration(final Configuration config) { + this.configuration = config; + return asBuilder(); + } + + public B setExpirySeconds(final Integer expirySeconds) { + this.expirySeconds = expirySeconds; + return this.asBuilder(); + } + + public B setIndexField(final LuceneIndexField... indexField) { + this.indexField = indexField; + return this.asBuilder(); + } + + public B setTarget(final String target) { + this.target = target; + return this.asBuilder(); + } + } + + /** + * IndexWriter corresponding to each index directory. + */ + private static final ConcurrentHashMap<String, IndexWriter> writerMap = new ConcurrentHashMap<>(); + + @PluginBuilderFactory + public static <B extends Builder<B>> B newBuilder() { + return new Builder<B>().asBuilder(); + } + + private final Configuration configuration; + + /** + * Index expiration time (seconds) + */ + private final Integer expirySeconds; + + /** + * IndexField array. + */ + private final LuceneIndexField[] indexFields; + + /** + * Index directory + */ + private final String target; + + protected LuceneAppender(String name, boolean ignoreExceptions, Filter filter, + Layout<? extends Serializable> layout, String target, Integer expiryTime, LuceneIndexField[] indexFields, + final Configuration configuration) { + super(name, filter, layout, ignoreExceptions); + this.target = target; + this.expirySeconds = expiryTime; + this.indexFields = indexFields; + this.configuration = configuration; + getIndexWriter(); + initialize(); + } + + /** + * create lucene index. + * + * @param event + */ + @Override + public void append(LogEvent event) { + if (null != indexFields && indexFields.length > 0) { + IndexWriter indexWriter = getIndexWriter(); + if (null != indexWriter) { + Document doc = new Document(); + doc.add(new LongField("timestamp", event.getTimeMillis(), Field.Store.YES)); + try { + for (LuceneIndexField field : indexFields) { + String value = field.getLayout().toSerializable(event); + if (Strings.isEmpty(value) || value.matches("[$]\\{.+\\}")) { + return; + } + value = value.trim(); + String type = field.getType(); + if (Strings.isNotEmpty(type)) { + Class<?> clazz = Class.forName("org.apache.lucene.document." + type); + if (clazz == LongField.class) { + doc.add(new LongField(field.getName(), Long.valueOf(value), Field.Store.YES)); + } else if (clazz == StringField.class) { + doc.add(new StringField(field.getName(), value, Field.Store.YES)); + } else if (clazz == TextField.class) { + doc.add(new TextField(field.getName(), value, Field.Store.YES)); + } else { + // TODO Should we throw an AppenderLoggingException here? + throw new UnsupportedOperationException(type + " type currently not supported."); + } + } else { + doc.add(new TextField(field.getName(), value, Field.Store.YES)); + } + } + indexWriter.addDocument(doc); + } catch (Exception e) { + LOGGER.error(e.getMessage(), e); + if (!ignoreExceptions()) { + throw new AppenderLoggingException(e); + } + } + } + } + } + + @Override + public void initialize() { + try { + registerCommitTimer(); + if (this.expirySeconds != null) { + registerClearTimer(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + super.initialize(); + } + + /** + * IndexWriter initialization. + */ + private IndexWriter getIndexWriter() { + if (null == writerMap.get(target)) { + try { + // TODO Who closes this FSDirectory? + FSDirectory fsDir = FSDirectory.open(Paths.get(this.target)); + // TODO Who closes this LuceneAnalyzer? + IndexWriterConfig writerConfig = new IndexWriterConfig(new LuceneAnalyzer()); + writerMap.putIfAbsent(target, new IndexWriter(fsDir, writerConfig)); + } catch (IOException e) { + LOGGER.error("IndexWriter initialization failed: {}", e.getMessage(), e); + } + } + return writerMap.get(target); + } + + /** + * Register IndexWriter clean timertask. Delete the index before + * {@link LuceneAppender#expirySeconds} second every day at 0. + * + * @see LuceneAppender#expirySeconds + */ + private void registerClearTimer() throws Exception { + Calendar calendar = Calendar.getInstance(); + long curMillis = calendar.getTimeInMillis(); + calendar.add(Calendar.DAY_OF_MONTH, 1); + calendar.set(Calendar.HOUR_OF_DAY, 0); + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); + long difMinutes = (calendar.getTimeInMillis() - curMillis) / (1000 * 60); + configuration.getScheduler().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + LOGGER.info("Deleting index {} {}...", target, expirySeconds); + IndexWriter indexWriter = getIndexWriter(); + if (null != indexWriter) { + Long start = 0L; + Long end = System.currentTimeMillis() - expirySeconds * 1000; + NumericRangeQuery<Long> rangeQuery = NumericRangeQuery.newLongRange("timestamp", start, end, true, + true); + try { + indexWriter.deleteDocuments(rangeQuery); + indexWriter.commit(); + LOGGER.info("Deleted index end {} {}", target, expirySeconds); + } catch (IOException e) { + LOGGER.error("Failed to delete index: {}", e.getMessage(), e); + } + } + } + }, difMinutes, 1440, TimeUnit.MINUTES); + } + + /** + * Register IndexWriter commit timertask. + */ + private void registerCommitTimer() throws Exception { + configuration.getScheduler().scheduleWithCron(new CronExpression("0 1/1 * * * ? "), new Runnable() { + @Override + public void run() { + IndexWriter indexWriter = getIndexWriter(); + if (null != indexWriter && indexWriter.numRamDocs() > 0) { + try { + indexWriter.commit(); + } catch (IOException e) { + LOGGER.error("IndexWriter commit failed: {}", e.getMessage(), e); + } + } + } + }); + } + + @Override + public final void start() { + if (null == writerMap.get(target)) { + LOGGER.error("No IndexWriter set for appender [{}].", this.getName()); + } + super.start(); + } + + @Override + public boolean stop(final long timeout, final TimeUnit timeUnit) { + setStopping(); + boolean stopped = super.stop(timeout, timeUnit, false); + IndexWriter indexWriter = writerMap.get(target); + if (null != indexWriter) { + try { + indexWriter.commit(); + if (indexWriter.isOpen()) { + indexWriter.close(); + } + writerMap.remove(target); + } catch (IOException e) { + LOGGER.error(e.getMessage(), e); + } + } + setStopped(); + return stopped; + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneIndexField.java ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneIndexField.java b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneIndexField.java new file mode 100644 index 0000000..39f2aea --- /dev/null +++ b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneIndexField.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.lucene5.appender; + +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; +import org.apache.logging.log4j.core.config.plugins.validation.constraints.Required; +import org.apache.logging.log4j.core.filter.AbstractFilterable; +import org.apache.logging.log4j.core.layout.PatternLayout; + +/** + * {@link LuceneAppender}'s configuration element that logs the log event + * attributes to the Field in the Lucene document. + */ +@Plugin(name = "IndexField", category = Node.CATEGORY, printObject = true) +public final class LuceneIndexField { + + public static class Builder<B extends Builder<B>> extends AbstractFilterable.Builder<B> + implements org.apache.logging.log4j.core.util.Builder<LuceneIndexField> { + + @PluginConfiguration + private Configuration configuration; + + @PluginBuilderAttribute + @Required(message = "No name provided") + private String name; + + @PluginBuilderAttribute + @Required(message = "No pattern provided") + private String pattern; + + @PluginBuilderAttribute + private String type; + + @Override + public LuceneIndexField build() { + // @formatter:off + final PatternLayout layout = PatternLayout.newBuilder() + .withPattern(pattern) + .withConfiguration(configuration) + .withAlwaysWriteExceptions(false) + .build(); + // @formatter:on + return new LuceneIndexField(name, layout, type); + } + + public B withName(final String name) { + this.name = name; + return this.asBuilder(); + } + + public B withPattern(final String pattern) { + this.pattern = pattern; + return this.asBuilder(); + } + + public B withType(final String type) { + this.type = type; + return this.asBuilder(); + } + } + + @PluginBuilderFactory + public static <B extends Builder<B>> B newBuilder() { + return new Builder<B>().asBuilder(); + } + + private final PatternLayout layout; + + private final String name; + + private final String type; + + private LuceneIndexField(final String name, final PatternLayout layout, final String type) { + this.name = name; + this.layout = layout; + this.type = type; + } + + public PatternLayout getLayout() { + return this.layout; + } + + public String getName() { + return this.name; + } + + public String getType() { + return type; + } + + @Override + public String toString() { + return "{name=" + this.name + ", layout=" + this.layout + " }"; + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneTokenizer.java ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneTokenizer.java b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneTokenizer.java new file mode 100644 index 0000000..c588a33 --- /dev/null +++ b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/LuceneTokenizer.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.lucene5.appender; + +import org.apache.lucene.analysis.util.CharTokenizer; + +/** + * Emits the entire input as a single token. + */ +public class LuceneTokenizer extends CharTokenizer { + + @Override + protected boolean isTokenChar(int c) { + return true; + } + + @Override + protected int normalize(int c) { + return Character.toLowerCase(c); + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/package-info.java ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/package-info.java b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/package-info.java new file mode 100644 index 0000000..bc0da3c --- /dev/null +++ b/log4j-lucene5/src/main/java/org/apache/logging/log4j/lucene5/appender/package-info.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +/** + * The classes in this package provide appenders for Lucene 5 and related + * configuration scheme. + */ +package org.apache.logging.log4j.lucene5.appender; + +; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/test/java/org/apache/logging/log4j/lucene5/appender/LuceneAppenderTest.java ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/test/java/org/apache/logging/log4j/lucene5/appender/LuceneAppenderTest.java b/log4j-lucene5/src/test/java/org/apache/logging/log4j/lucene5/appender/LuceneAppenderTest.java new file mode 100644 index 0000000..1286d78 --- /dev/null +++ b/log4j-lucene5/src/test/java/org/apache/logging/log4j/lucene5/appender/LuceneAppenderTest.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ + +package org.apache.logging.log4j.lucene5.appender; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.core.impl.Log4jLogEvent; +import org.apache.logging.log4j.junit.CleanFolders; +import org.apache.logging.log4j.junit.LoggerContextRule; +import org.apache.logging.log4j.lucene5.appender.LuceneAppender; +import org.apache.logging.log4j.message.SimpleMessage; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.FSDirectory; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; + +public class LuceneAppenderTest { + + private class LuceneAppenderRunner implements Runnable { + @Override + public void run() { + try { + write(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + private static final String CONFIGURATION_FILE = "log4j2-lucene.xml"; + private static final String FIELD_1 = "field1"; + private static final String FIELD_2 = "field2"; + private static final String LOG_MESSAGE = "Hello world!"; + private static final String LOGGER_NAME = "TestLogger"; + private static final String TARGET_FOLDER = "target/lucene"; + private static final String EXEPECTED_REGEX = "^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2},\\d{3} \\[[^\\]]*\\] INFO " + + LOGGER_NAME + " - " + LOG_MESSAGE; + private static final Path PATH = Paths.get(TARGET_FOLDER); + + private static final int THREAD_COUNT = 50; + + @Rule + public LoggerContextRule ctx = new LoggerContextRule(CONFIGURATION_FILE); + + @Rule + public CleanFolders folders = new CleanFolders(PATH); + + @Test + public void testMultipleThreads() throws Exception { + final ExecutorService threadPool = Executors.newFixedThreadPool(THREAD_COUNT); + final LuceneAppenderRunner runner = new LuceneAppenderRunner(); + for (int i = 0; i < THREAD_COUNT; ++i) { + threadPool.execute(runner); + } + + // Waiting for lucene records to complete and submit + Thread.sleep(3000); + + verify(THREAD_COUNT); + } + + @Test + public void testSimple() throws Exception { + write(); + verify(1); + } + + private final synchronized void verify(final int exepectedTotalHits) throws Exception { + try (final FSDirectory fsDir = FSDirectory.open(PATH); final IndexReader reader = DirectoryReader.open(fsDir)) { + final IndexSearcher searcher = new IndexSearcher(reader); + final TopDocs all = searcher.search(new MatchAllDocsQuery(), Integer.MAX_VALUE); + Assert.assertEquals(all.totalHits, exepectedTotalHits); + for (ScoreDoc scoreDoc : all.scoreDocs) { + final Document doc = searcher.doc(scoreDoc.doc); + Assert.assertEquals(doc.getFields().size(), 3); + final String field1 = doc.get(FIELD_1); + Assert.assertTrue("Unexpected field1: " + field1, Level.INFO.toString().equals(field1)); + final String field2 = doc.get(FIELD_2); + final Pattern pattern = Pattern.compile(EXEPECTED_REGEX); + final Matcher matcher = pattern.matcher(field2); + Assert.assertTrue("Unexpected field2: " + field2, matcher.matches()); + } + } + } + + private final void write() throws Exception { + final LuceneAppender appender = (LuceneAppender) ctx.getRequiredAppender("Lucene5Appender"); + try { + appender.start(); + assertTrue("Appender did not start", appender.isStarted()); + // @formatter:off + final Log4jLogEvent event = Log4jLogEvent.newBuilder() + .setLoggerName(LOGGER_NAME) + .setLoggerFqcn(LuceneAppenderTest.class.getName()) + .setLevel(Level.INFO) + .setMessage(new SimpleMessage(LOG_MESSAGE)) + .build(); + // @formatter:on + appender.append(event); + } finally { + appender.stop(); + } + assertFalse("Appender did not stop", appender.isStarted()); + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/log4j-lucene5/src/test/resources/log4j2-lucene.xml ---------------------------------------------------------------------- diff --git a/log4j-lucene5/src/test/resources/log4j2-lucene.xml b/log4j-lucene5/src/test/resources/log4j2-lucene.xml new file mode 100644 index 0000000..f8ce397 --- /dev/null +++ b/log4j-lucene5/src/test/resources/log4j2-lucene.xml @@ -0,0 +1,30 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<Configuration name="Lucene5AppenderTest" packages="com.wb.testCase"> + <Appenders> + <Lucene5 name="Lucene5Appender" target="target/lucene"> + <IndexField name="field1" pattern="%-5level" type="TextField" /> + <IndexField name="field2" pattern="%d [%t] %p %c - %m%n" /> + </Lucene5> + </Appenders> + <Loggers> + <Root level="info"> + <AppenderRef ref="Lucene5Appender" /> + </Root> + </Loggers> +</Configuration> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/929cc9ba/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index e395d7e..b40f778 100644 --- a/pom.xml +++ b/pom.xml @@ -1341,6 +1341,7 @@ <module>log4j-iostreams</module> <module>log4j-jul</module> <module>log4j-liquibase</module> + <module>log4j-lucene5</module> <module>log4j-api-scala_2.10</module> <module>log4j-api-scala_2.11</module> </modules>
