[LOG4J2-1730]: Add literal values and date type support to CassandraAppender
The previously assumed method of encoding dates as a long value does not work properly. Instead, this commit adds the ability to use a java.util.Date-compatible type conversion for ColumnMapping which uses the LogEvent timestamp instead of a StringLayout->TypeConverter chain. This also adds the ability to use literal values in column mappings similar to the equivalent feature in JdbcAppender. Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/0bca4ca0 Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/0bca4ca0 Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/0bca4ca0 Branch: refs/heads/master Commit: 0bca4ca03b0e32e4cc3cafa9268964560ddede20 Parents: dd65750 Author: Matt Sicker <[email protected]> Authored: Sat Jan 7 02:31:46 2017 -0600 Committer: Matt Sicker <[email protected]> Committed: Sat Jan 7 02:41:02 2017 -0600 ---------------------------------------------------------------------- .../log4j/core/appender/db/ColumnMapping.java | 62 ++++++++++++---- .../plugins/convert/DateTypeConverter.java | 52 ++++++++++++++ .../plugins/convert/DateTypeConverterTest.java | 44 ++++++++++++ .../appender/cassandra/CassandraManager.java | 24 ++++--- .../nosql/appender/cassandra/TypeCodecs.java | 75 -------------------- .../appender/cassandra/CassandraAppenderIT.java | 8 ++- .../test/resources/CassandraAppenderTest.xml | 3 +- src/site/xdoc/manual/appenders.xml | 15 ++-- 8 files changed, 178 insertions(+), 105 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/db/ColumnMapping.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/db/ColumnMapping.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/db/ColumnMapping.java index c485c21..cec7c8c 100644 --- a/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/db/ColumnMapping.java +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/appender/db/ColumnMapping.java @@ -16,6 +16,9 @@ */ package org.apache.logging.log4j.core.appender.db; +import java.util.Date; + +import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.core.Core; import org.apache.logging.log4j.core.StringLayout; import org.apache.logging.log4j.core.config.Configuration; @@ -28,6 +31,7 @@ import org.apache.logging.log4j.core.config.plugins.validation.constraints.Requi import org.apache.logging.log4j.core.layout.PatternLayout; import org.apache.logging.log4j.spi.ThreadContextMap; import org.apache.logging.log4j.spi.ThreadContextStack; +import org.apache.logging.log4j.status.StatusLogger; import org.apache.logging.log4j.util.ReadOnlyStringMap; /** @@ -38,24 +42,17 @@ import org.apache.logging.log4j.util.ReadOnlyStringMap; @Plugin(name = "ColumnMapping", category = Core.CATEGORY_NAME, printObject = true) public class ColumnMapping { - /** - * Column name. - */ + private static final Logger LOGGER = StatusLogger.getLogger(); + private final String name; - /** - * Layout of value to write to database (before type conversion). Not applicable if {@link #type} is a collection. - */ private final StringLayout layout; - /** - * Class to convert value to before storing in database. If the type is compatible with {@link ThreadContextMap} or - * {@link ReadOnlyStringMap}, then the MDC will be used. If the type is compatible with {@link ThreadContextStack}, - * then the NDC will be used. - */ + private final String literalValue; private final Class<?> type; - private ColumnMapping(final String name, final StringLayout layout, final Class<?> type) { + private ColumnMapping(final String name, final StringLayout layout, final String literalValue, final Class<?> type) { this.name = name; this.layout = layout; + this.literalValue = literalValue; this.type = type; } @@ -67,6 +64,10 @@ public class ColumnMapping { return layout; } + public String getLiteralValue() { + return literalValue; + } + public Class<?> getType() { return type; } @@ -89,27 +90,55 @@ public class ColumnMapping { private String pattern; @PluginBuilderAttribute + private String literal; + + @PluginBuilderAttribute @Required(message = "No conversion type provided") private Class<?> type = String.class; @PluginConfiguration private Configuration configuration; + /** + * Column name. + */ public Builder setName(final String name) { this.name = name; return this; } + /** + * Layout of value to write to database (before type conversion). Not applicable if {@link #setType(Class)} is + * a {@link ReadOnlyStringMap}, {@link ThreadContextMap}, or {@link ThreadContextStack}. + */ public Builder setLayout(final StringLayout layout) { this.layout = layout; return this; } + /** + * Pattern to use as a {@link PatternLayout}. Convenient shorthand for {@link #setLayout(StringLayout)} with a + * PatternLayout. + */ public Builder setPattern(final String pattern) { this.pattern = pattern; return this; } + /** + * Literal value to use for populating a column. This is generally useful for functions, stored procedures, + * etc. No escaping will be done on this value. + */ + public Builder setLiteral(final String literal) { + this.literal = literal; + return this; + } + + /** + * Class to convert value to before storing in database. If the type is compatible with {@link ThreadContextMap} or + * {@link ReadOnlyStringMap}, then the MDC will be used. If the type is compatible with {@link ThreadContextStack}, + * then the NDC will be used. If the type is compatible with {@link Date}, then the event timestamp will be used. + */ public Builder setType(final Class<?> type) { this.type = type; return this; @@ -129,13 +158,16 @@ public class ColumnMapping { .build(); } if (!(layout != null + || literal != null + || Date.class.isAssignableFrom(type) || ReadOnlyStringMap.class.isAssignableFrom(type) || ThreadContextMap.class.isAssignableFrom(type) || ThreadContextStack.class.isAssignableFrom(type))) { - throw new IllegalStateException( - "No layout specified and type (" + type + ") is not compatible with ThreadContextMap or ThreadContextStack"); + LOGGER.error("No layout or literal value specified and type ({}) is not compatible with " + + "ThreadContextMap, ThreadContextStack, or java.util.Date", type); + return null; } - return new ColumnMapping(name, layout, type); + return new ColumnMapping(name, layout, literal, type); } } } http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverter.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverter.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverter.java new file mode 100644 index 0000000..efc0596 --- /dev/null +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverter.java @@ -0,0 +1,52 @@ +package org.apache.logging.log4j.core.config.plugins.convert; + +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Arrays; +import java.util.Date; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Utility methods for Date classes. + */ +public final class DateTypeConverter { + + private static final Map<Class<? extends Date>, MethodHandle> CONSTRUCTORS = new ConcurrentHashMap<>(); + + static { + final MethodHandles.Lookup lookup = MethodHandles.publicLookup(); + for (final Class<? extends Date> dateClass : Arrays.asList(Date.class, java.sql.Date.class, Time.class, + Timestamp.class)) { + try { + CONSTRUCTORS.put(dateClass, + lookup.findConstructor(dateClass, MethodType.methodType(void.class, long.class))); + } catch (final NoSuchMethodException | IllegalAccessException ignored) { + // these classes all have this exact constructor + } + } + } + + /** + * Create a Date-related object from a timestamp in millis. + * + * @param millis timestamp in millis + * @param type date type to use + * @param <D> date class to use + * @return new instance of D or null if there was an error + */ + @SuppressWarnings("unchecked") + public static <D extends Date> D fromMillis(final long millis, final Class<D> type) { + try { + return (D) CONSTRUCTORS.get(type).invoke(millis); + } catch (final Throwable ignored) { + return null; + } + } + + private DateTypeConverter() { + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-core/src/test/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverterTest.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverterTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverterTest.java new file mode 100644 index 0000000..223d8b6 --- /dev/null +++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/config/plugins/convert/DateTypeConverterTest.java @@ -0,0 +1,44 @@ +package org.apache.logging.log4j.core.config.plugins.convert; + +import java.sql.Time; +import java.sql.Timestamp; +import java.util.Date; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import static org.junit.Assert.assertEquals; + +/** + * + */ +@RunWith(Parameterized.class) +public class DateTypeConverterTest { + + private final Class<? extends Date> dateClass; + private final long timestamp; + private final Object expected; + + @Parameterized.Parameters + public static Object[][] data() { + final long millis = System.currentTimeMillis(); + return new Object[][]{ + {Date.class, millis, new Date(millis)}, + {java.sql.Date.class, millis, new java.sql.Date(millis)}, + {Time.class, millis, new Time(millis)}, + {Timestamp.class, millis, new Timestamp(millis)} + }; + } + + public DateTypeConverterTest(final Class<? extends Date> dateClass, final long timestamp, final Object expected) { + this.dateClass = dateClass; + this.timestamp = timestamp; + this.expected = expected; + } + + @Test + public void testFromMillis() throws Exception { + assertEquals(expected, DateTypeConverter.fromMillis(timestamp, dateClass)); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraManager.java ---------------------------------------------------------------------- diff --git a/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraManager.java b/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraManager.java index 4178053..cfbc528 100644 --- a/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraManager.java +++ b/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraManager.java @@ -17,7 +17,8 @@ package org.apache.logging.log4j.nosql.appender.cassandra; import java.net.InetSocketAddress; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.Date; import java.util.List; import com.datastax.driver.core.BatchStatement; @@ -29,6 +30,7 @@ import org.apache.logging.log4j.core.LogEvent; import org.apache.logging.log4j.core.appender.ManagerFactory; import org.apache.logging.log4j.core.appender.db.AbstractDatabaseManager; import org.apache.logging.log4j.core.appender.db.ColumnMapping; +import org.apache.logging.log4j.core.config.plugins.convert.DateTypeConverter; import org.apache.logging.log4j.core.config.plugins.convert.TypeConverters; import org.apache.logging.log4j.core.net.SocketAddress; import org.apache.logging.log4j.spi.ThreadContextMap; @@ -41,11 +43,6 @@ import org.apache.logging.log4j.util.Strings; */ public class CassandraManager extends AbstractDatabaseManager { - static { - // pre-register custom type codecs - TypeCodecs.registerCustomCodecs(); - } - private static final int DEFAULT_PORT = 9042; private final Cluster cluster; @@ -98,6 +95,8 @@ public class CassandraManager extends AbstractDatabaseManager { values[i] = event.getContextData().toMap(); } else if (ThreadContextStack.class.isAssignableFrom(columnMapping.getType())) { values[i] = event.getContextStack().asList(); + } else if (Date.class.isAssignableFrom(columnMapping.getType())) { + values[i] = DateTypeConverter.fromMillis(event.getTimeMillis(), columnMapping.getType().asSubclass(Date.class)); } else { values[i] = TypeConverters.convert(columnMapping.getLayout().toSerializable(event), columnMapping.getType(), null); @@ -156,14 +155,21 @@ public class CassandraManager extends AbstractDatabaseManager { } sb.setCharAt(sb.length() - 1, ')'); sb.append(" VALUES ("); - for (int i = 0; i < data.columns.length; i++) { - sb.append("?,"); + final List<ColumnMapping> columnMappings = new ArrayList<>(data.columns.length); + for (final ColumnMapping column : data.columns) { + if (Strings.isNotEmpty(column.getLiteralValue())) { + sb.append(column.getLiteralValue()); + } else { + sb.append('?'); + columnMappings.add(column); + } + sb.append(','); } sb.setCharAt(sb.length() - 1, ')'); final String insertQueryTemplate = sb.toString(); LOGGER.debug("Using CQL for appender {}: {}", name, insertQueryTemplate); return new CassandraManager(name, data.getBufferSize(), cluster, data.keyspace, insertQueryTemplate, - Arrays.asList(data.columns), data.batched ? new BatchStatement(data.batchType) : null); + columnMappings, data.batched ? new BatchStatement(data.batchType) : null); } } http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/TypeCodecs.java ---------------------------------------------------------------------- diff --git a/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/TypeCodecs.java b/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/TypeCodecs.java deleted file mode 100644 index ef83852..0000000 --- a/log4j-nosql/src/main/java/org/apache/logging/log4j/nosql/appender/cassandra/TypeCodecs.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache license, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the license for the specific language governing permissions and - * limitations under the license. - */ -package org.apache.logging.log4j.nosql.appender.cassandra; - -import java.nio.ByteBuffer; - -import com.datastax.driver.core.CodecRegistry; -import com.datastax.driver.core.DataType; -import com.datastax.driver.core.ProtocolVersion; -import com.datastax.driver.core.TypeCodec; -import com.datastax.driver.core.exceptions.InvalidTypeException; -import org.apache.logging.log4j.util.Strings; - -/** - * Custom TypeCodecs for use with the Datastax Cassandra driver. - */ -public final class TypeCodecs { - - public static void registerCustomCodecs() { - CodecRegistry.DEFAULT_INSTANCE.register(new LongTimestampCodec()); - } - - /** - * TypeCodec that allows a long value to be used as a timestamp. - */ - public static class LongTimestampCodec extends TypeCodec.PrimitiveLongCodec { - - private LongTimestampCodec() { - super(DataType.timestamp()); - } - - @Override - public ByteBuffer serializeNoBoxing(final long v, final ProtocolVersion protocolVersion) { - final ByteBuffer bb = ByteBuffer.allocate(8); - bb.putLong(v); - return bb; - } - - @Override - public long deserializeNoBoxing(final ByteBuffer v, final ProtocolVersion protocolVersion) { - if (v == null || v.remaining() == 0) { - return 0; - } - if (v.remaining() != 8) { - throw new InvalidTypeException("Expected an 8 byte value, but got " + v.remaining() + " bytes"); - } - return v.getLong(v.position()); - } - - @Override - public Long parse(final String value) throws InvalidTypeException { - return Strings.isEmpty(value) || "NULL".equalsIgnoreCase(value) ? null : Long.parseLong(value); - } - - @Override - public String format(final Long value) throws InvalidTypeException { - return value == null ? "NULL" : value.toString(); - } - } - -} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-nosql/src/test/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraAppenderIT.java ---------------------------------------------------------------------- diff --git a/log4j-nosql/src/test/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraAppenderIT.java b/log4j-nosql/src/test/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraAppenderIT.java index 9c7b98c..0c5f24b 100644 --- a/log4j-nosql/src/test/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraAppenderIT.java +++ b/log4j-nosql/src/test/java/org/apache/logging/log4j/nosql/appender/cassandra/CassandraAppenderIT.java @@ -16,8 +16,10 @@ */ package org.apache.logging.log4j.nosql.appender.cassandra; +import java.util.Date; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.concurrent.TimeUnit; import com.datastax.driver.core.Row; @@ -30,7 +32,7 @@ import org.junit.ClassRule; import org.junit.Test; import org.junit.rules.RuleChain; -import static org.junit.Assert.assertEquals; +import static org.junit.Assert.*; /** * Integration test for CassandraAppender. @@ -39,6 +41,7 @@ public class CassandraAppenderIT { private static final String DDL = "CREATE TABLE logs (" + "id timeuuid PRIMARY KEY," + + "timeid timeuuid," + "message text," + "level text," + "marker text," + @@ -69,6 +72,9 @@ public class CassandraAppenderIT { int i = 0; try (final Session session = CASSANDRA.connect()) { for (final Row row : session.execute("SELECT * FROM logs")) { + assertNotNull(row.get("id", UUID.class)); + assertNotNull(row.get("timeid", UUID.class)); + assertNotNull(row.get("timestamp", Date.class)); assertEquals("Test log message", row.getString("message")); assertEquals("MARKER", row.getString("marker")); assertEquals("INFO", row.getString("level")); http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/log4j-nosql/src/test/resources/CassandraAppenderTest.xml ---------------------------------------------------------------------- diff --git a/log4j-nosql/src/test/resources/CassandraAppenderTest.xml b/log4j-nosql/src/test/resources/CassandraAppenderTest.xml index 4879750..b3956d1 100644 --- a/log4j-nosql/src/test/resources/CassandraAppenderTest.xml +++ b/log4j-nosql/src/test/resources/CassandraAppenderTest.xml @@ -21,11 +21,12 @@ <Cassandra name="Cassandra" clusterName="Test Cluster" keyspace="test" table="logs" bufferSize="10" batched="true"> <SocketAddress host="localhost" port="9042"/> <ColumnMapping name="id" pattern="%uuid{TIME}" type="java.util.UUID"/> + <ColumnMapping name="timeid" literal="now()"/> <ColumnMapping name="message" pattern="%message"/> <ColumnMapping name="level" pattern="%level"/> <ColumnMapping name="marker" pattern="%marker"/> <ColumnMapping name="logger" pattern="%logger"/> - <ColumnMapping name="timestamp" pattern="%date{UNIX_MILLIS}" type="java.lang.Long"/> + <ColumnMapping name="timestamp" type="java.util.Date"/> <ColumnMapping name="mdc" type="org.apache.logging.log4j.spi.ThreadContextMap"/> <ColumnMapping name="ndc" type="org.apache.logging.log4j.spi.ThreadContextStack"/> </Cassandra> http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/0bca4ca0/src/site/xdoc/manual/appenders.xml ---------------------------------------------------------------------- diff --git a/src/site/xdoc/manual/appenders.xml b/src/site/xdoc/manual/appenders.xml index 31c2b93..d1f6223 100644 --- a/src/site/xdoc/manual/appenders.xml +++ b/src/site/xdoc/manual/appenders.xml @@ -246,7 +246,9 @@ <a href="layouts.html#PatternLayout">PatternLayout</a>) along with an optional conversion type, or only a conversion type for <code>org.apache.logging.log4j.spi.ThreadContextMap</code> or <code>org.apache.logging.log4j.spi.ThreadContextStack</code> to store the <a href="thread-context.html">MDC or NDC</a> - in a map or list column respectively. + in a map or list column respectively. A conversion type compatible with <code>java.util.Date</code> will + use the log event timestamp converted to that type (e.g., use <code>java.util.Date</code> to fill a + <code>timestamp</code> column type in Cassandra). </p> <table> <caption align="top">CassandraAppender Parameters</caption> @@ -286,8 +288,11 @@ <a class="javadoc" href="../log4j-api/apidocs/org/apache/logging/log4j/spi/ThreadContextMap.html">ThreadContextMap</a> or <a class="javadoc" href="../log4j-api/apidocs/org/apache/logging/log4j/spi/ThreadContextStack.html">ThreadContextStack</a>, - then that column will be populated with the MDC or NDC respectively. Otherwise, the layout or pattern - specified will be converted into the configured type and stored in that column. + then that column will be populated with the MDC or NDC respectively. If the configured type is + assignment-compatible with <code>java.util.Date</code>, then the log timestamp will be converted to + that configured date type. If a <code>literal</code> attribute is given, then its value will be used as + is in the <code>INSERT</code> query without any escaping. Otherwise, the layout or pattern specified + will be converted into the configured type and stored in that column. </td> </tr> <tr> @@ -358,11 +363,12 @@ <Cassandra name="Cassandra" clusterName="Test Cluster" keyspace="test" table="logs" bufferSize="10" batched="true"> <SocketAddress host="localhost" port="9042"/> <ColumnMapping name="id" pattern="%uuid{TIME}" type="java.util.UUID"/> + <ColumnMapping name="timeid" literal="now()"/> <ColumnMapping name="message" pattern="%message"/> <ColumnMapping name="level" pattern="%level"/> <ColumnMapping name="marker" pattern="%marker"/> <ColumnMapping name="logger" pattern="%logger"/> - <ColumnMapping name="timestamp" pattern="%date{UNIX_MILLIS}" type="java.lang.Long"/> + <ColumnMapping name="timestamp" type="java.util.Date"/> <ColumnMapping name="mdc" type="org.apache.logging.log4j.spi.ThreadContextMap"/> <ColumnMapping name="ndc" type="org.apache.logging.log4j.spi.ThreadContextStack"/> </Cassandra> @@ -381,6 +387,7 @@ <pre class="prettyprint linenums"><