This is an automated email from the ASF dual-hosted git repository. sorabh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/drill.git
commit 12af476247789a1decba8d4c715bd6c41d6690e8 Author: Anton Gozhiy <[email protected]> AuthorDate: Fri Jul 20 16:59:24 2018 +0300 DRILL-6544: Allow timestamp / date / time formatting when displaying on Web UI Added the following options that are setting the format pattens: web.timestamp.display_format, web.date.display_format, web.time.display_format. See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable values. Default formatting is used, if the corresponding option has empty string. --- .../java/org/apache/drill/exec/ExecConstants.java | 22 ++++ .../exec/server/options/SystemOptionManager.java | 3 + .../drill/exec/server/options/TypeValidators.java | 26 ++++ .../drill/exec/server/rest/WebUserConnection.java | 6 +- .../exec/util/ValueVectorElementFormatter.java | 121 +++++++++++++++++ .../java-exec/src/main/resources/drill-module.conf | 3 + .../exec/util/TestValueVectorElementFormatter.java | 145 +++++++++++++++++++++ 7 files changed, 325 insertions(+), 1 deletion(-) diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java index 2ecb3c8..03707b2 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java @@ -22,6 +22,7 @@ import org.apache.drill.exec.physical.impl.common.HashTable; import org.apache.drill.exec.rpc.user.InboundImpersonationManager; import org.apache.drill.exec.server.options.OptionValidator; import org.apache.drill.exec.server.options.OptionValidator.OptionDescription; +import org.apache.drill.exec.server.options.TypeValidators.DateTimeFormatValidator; import org.apache.drill.exec.server.options.TypeValidators.IntegerValidator; import org.apache.drill.exec.server.options.TypeValidators.BooleanValidator; import org.apache.drill.exec.server.options.TypeValidators.DoubleValidator; @@ -684,6 +685,27 @@ public final class ExecConstants { public static final String WEB_LOGS_MAX_LINES = "web.logs.max_lines"; public static final OptionValidator WEB_LOGS_MAX_LINES_VALIDATOR = new PositiveLongValidator(WEB_LOGS_MAX_LINES, Integer.MAX_VALUE, null); + public static final String WEB_DISPLAY_FORMAT_TIMESTAMP = "web.display_format.timestamp"; + public static final OptionValidator WEB_DISPLAY_FORMAT_TIMESTAMP_VALIDATOR = new DateTimeFormatValidator(WEB_DISPLAY_FORMAT_TIMESTAMP, + new OptionDescription("Display format template for timestamp. " + + "It will be passed to java.time.format.DateTimeFormatter. " + + "See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable patterns. " + + "If empty then the default formatting will be used. (Drill 1.15 and later)")); + + public static final String WEB_DISPLAY_FORMAT_DATE = "web.display_format.date"; + public static final OptionValidator WEB_DISPLAY_FORMAT_DATE_VALIDATOR = new DateTimeFormatValidator(WEB_DISPLAY_FORMAT_DATE, + new OptionDescription("Display format template for date. " + + "It will be passed to java.time.format.DateTimeFormatter. " + + "See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable patterns. " + + "If empty then the default formatting will be used. (Drill 1.15 and later)")); + + public static final String WEB_DISPLAY_FORMAT_TIME = "web.display_format.time"; + public static final OptionValidator WEB_DISPLAY_FORMAT_TIME_VALIDATOR = new DateTimeFormatValidator(WEB_DISPLAY_FORMAT_TIME, + new OptionDescription("Display format template for time. " + + "It will be passed to java.time.format.DateTimeFormatter. " + + "See https://docs.oracle.com/javase/8/docs/api/java/time/format/DateTimeFormatter.html for the details about acceptable patterns. " + + "If empty then the default formatting will be used. (Drill 1.15 and later)")); + public static final String CODE_GEN_EXP_IN_METHOD_SIZE = "exec.java.compiler.exp_in_method_size"; public static final LongValidator CODE_GEN_EXP_IN_METHOD_SIZE_VALIDATOR = new LongValidator(CODE_GEN_EXP_IN_METHOD_SIZE, new OptionDescription("Introduced in Drill 1.8. For queries with complex or multiple expressions in the query logic, this option limits the number of expressions allowed in each method to prevent Drill from generating code that exceeds the Java limit of 64K bytes. If a method approaches the 64K limit, the Java compiler returns a message stating that the code is too large to compile. If queries return such a message, reduce the value of this option at the session level. The default [...] diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java index 80b8ceb..1a168c3 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java @@ -220,6 +220,9 @@ public class SystemOptionManager extends BaseOptionManager implements AutoClosea new OptionDefinition(ExecConstants.ENABLE_BULK_LOAD_TABLE_LIST), new OptionDefinition(ExecConstants.BULK_LOAD_TABLE_LIST_BULK_SIZE), new OptionDefinition(ExecConstants.WEB_LOGS_MAX_LINES_VALIDATOR), + new OptionDefinition(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP_VALIDATOR), + new OptionDefinition(ExecConstants.WEB_DISPLAY_FORMAT_DATE_VALIDATOR), + new OptionDefinition(ExecConstants.WEB_DISPLAY_FORMAT_TIME_VALIDATOR), new OptionDefinition(ExecConstants.IMPLICIT_FILENAME_COLUMN_LABEL_VALIDATOR), new OptionDefinition(ExecConstants.IMPLICIT_SUFFIX_COLUMN_LABEL_VALIDATOR), new OptionDefinition(ExecConstants.IMPLICIT_FQN_COLUMN_LABEL_VALIDATOR), diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java index 106d436..3f8f8a4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/TypeValidators.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.options; +import java.time.format.DateTimeFormatter; import java.util.Set; import org.apache.drill.shaded.guava.com.google.common.base.Joiner; @@ -289,6 +290,31 @@ public class TypeValidators { } } + /** + * Validator that checks if the given DateTime format template is valid. + * See {@link DateTimeFormatter} for the acceptable values. + */ + public static class DateTimeFormatValidator extends StringValidator { + + public DateTimeFormatValidator(String name, OptionDescription description) { + super(name, description); + } + + @Override + public void validate(OptionValue v, OptionMetaData metaData, OptionSet manager) { + super.validate(v, metaData, manager); + if (!v.string_val.isEmpty()) { + try { + DateTimeFormatter.ofPattern(v.string_val); + } catch (IllegalArgumentException e) { + throw UserException.validationError() + .message("'%s' is not a valid DateTime format pattern: %s", v.string_val, e.getMessage()) + .build(logger); + } + } + } + } + public static abstract class TypeValidator extends OptionValidator { private final Kind kind; diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java index 80a0a7e..df4c8a5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/WebUserConnection.java @@ -17,6 +17,7 @@ */ package org.apache.drill.exec.server.rest; +import org.apache.drill.exec.util.ValueVectorElementFormatter; import org.apache.drill.shaded.guava.com.google.common.collect.Lists; import org.apache.drill.shaded.guava.com.google.common.collect.Maps; import org.apache.drill.shaded.guava.com.google.common.collect.Sets; @@ -24,6 +25,7 @@ import io.netty.buffer.ByteBuf; import io.netty.buffer.DrillBuf; import io.netty.channel.ChannelFuture; import org.apache.drill.common.exceptions.UserException; +import org.apache.drill.common.types.TypeProtos; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.physical.impl.materialize.QueryWritableBatch; import org.apache.drill.exec.proto.GeneralRPCProtos.Ack; @@ -106,13 +108,15 @@ public class WebUserConnection extends AbstractDisposableUserClientConnection im for (int i = 0; i < loader.getSchema().getFieldCount(); ++i) { columns.add(loader.getSchema().getColumn(i).getName()); } + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(webSessionResources.getSession().getOptions()); for (int i = 0; i < rows; ++i) { final Map<String, String> record = Maps.newHashMap(); for (VectorWrapper<?> vw : loader) { final String field = vw.getValueVector().getMetadata().getNamePart().getName(); + final TypeProtos.MinorType fieldMinorType = vw.getValueVector().getMetadata().getMajorType().getMinorType(); final Accessor accessor = vw.getValueVector().getAccessor(); final Object value = i < accessor.getValueCount() ? accessor.getObject(i) : null; - final String display = value == null ? null : value.toString(); + final String display = value == null ? null : formatter.format(value, fieldMinorType); record.put(field, display); } results.add(record); diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java new file mode 100644 index 0000000..bbb13a7 --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/ValueVectorElementFormatter.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.util; + +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.server.options.OptionManager; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.function.BiFunction; + +/** + * This class is responsible for formatting ValueVector elements. + * Specific format templates are taken from the options. + * It creates and reuses the concrete formatter instances for performance purposes. + */ +public class ValueVectorElementFormatter { + private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVectorElementFormatter.class); + private final OptionManager options; + + private DateTimeFormatter timestampFormatter; + private DateTimeFormatter dateFormatter; + private DateTimeFormatter timeFormatter; + + public ValueVectorElementFormatter(OptionManager options) { + this.options = options; + } + + /** + * Formats ValueVector elements in accordance with it's minor type. + * + * @param value ValueVector element to format + * @param minorType the minor type of the element + * @return the formatted value, null if failed + */ + public String format(Object value, TypeProtos.MinorType minorType) { + switch (minorType) { + case TIMESTAMP: + if (value instanceof LocalDateTime) { + return format((LocalDateTime) value, + options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP), + (v, p) -> v.format(getTimestampFormatter(p))); + } + case DATE: + if (value instanceof LocalDate) { + return format((LocalDate) value, + options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE), + (v, p) -> v.format(getDateFormatter(p))); + } + case TIME: + if (value instanceof LocalTime) { + return format((LocalTime) value, + options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME), + (v, p) -> v.format(getTimeFormatter(p))); + } + default: + return value.toString(); + } + } + + /** + * Formats ValueVector elements using given function. + * + * @param value ValueVector element to format, casted to it's actual type. + * @param formatPattern pattern used for the formatting. If empty then value.toString() will be returned. + * @param formatFunction function that takes ValueVector element and format pattern as arguments and returns the formatted string. + * @param <T> actual type of the ValueVector element. + * @return the formatted value, null if failed + */ + private <T> String format(T value, String formatPattern, BiFunction<T, String, String> formatFunction) { + if (formatPattern.isEmpty()) { + return value.toString(); + } + String formattedValue = null; + try { + formattedValue = formatFunction.apply(value, formatPattern); + } catch (Exception e) { + logger.debug(String.format("Could not format the value '%s' with the pattern '%s': %s", value, formatPattern, e.getMessage())); + } + return formattedValue; + } + + private DateTimeFormatter getTimestampFormatter(String formatPattern) { + if (timestampFormatter == null) { + timestampFormatter = DateTimeFormatter.ofPattern(formatPattern); + } + return timestampFormatter; + } + + private DateTimeFormatter getDateFormatter(String formatPattern) { + if (dateFormatter == null) { + dateFormatter = DateTimeFormatter.ofPattern(formatPattern); + } + return dateFormatter; + } + + private DateTimeFormatter getTimeFormatter(String formatPattern) { + if (timeFormatter == null) { + timeFormatter = DateTimeFormatter.ofPattern(formatPattern); + } + return timeFormatter; + } +} diff --git a/exec/java-exec/src/main/resources/drill-module.conf b/exec/java-exec/src/main/resources/drill-module.conf index e9e1940..66ad9e8 100644 --- a/exec/java-exec/src/main/resources/drill-module.conf +++ b/exec/java-exec/src/main/resources/drill-module.conf @@ -615,6 +615,9 @@ drill.exec.options: { store.kafka.record.reader: "org.apache.drill.exec.store.kafka.decoders.JsonMessageReader", store.kafka.poll.timeout: 200, web.logs.max_lines: 10000, + web.display_format.timestamp: "", + web.display_format.date: "", + web.display_format.time: "", window.enable: true, storage.list_files_recursively: false } diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java new file mode 100644 index 0000000..5c6666d --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/util/TestValueVectorElementFormatter.java @@ -0,0 +1,145 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.util; + +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.server.options.OptionManager; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.mockito.Mockito.when; + +public class TestValueVectorElementFormatter { + + @Mock + private OptionManager options; + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + } + + @Test + public void testFormatValueVectorElementTimestampEmptyPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP)).thenReturn(""); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalDateTime.of(2012, 11, 5, 13, 0, 30, 120000000), + TypeProtos.MinorType.TIMESTAMP); + assertEquals("2012-11-05T13:00:30.120", formattedValue); + } + + @Test + public void testFormatValueVectorElementTimestampValidPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP)).thenReturn("yyyy-MM-dd HH:mm:ss.SS"); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalDateTime.of(2012, 11, 5, 13, 0, 30, 120000000), + TypeProtos.MinorType.TIMESTAMP); + assertEquals("2012-11-05 13:00:30.12", formattedValue); + } + + @Test + public void testFormatValueVectorElementDateEmptyPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn(""); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalDate.of(2012, 11, 5), + TypeProtos.MinorType.DATE); + assertEquals("2012-11-05", formattedValue); + } + + @Test + public void testFormatValueVectorElementDateValidPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("EEE, MMM d, yyyy"); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalDate.of(2012, 11, 5), + TypeProtos.MinorType.DATE); + assertEquals("Mon, Nov 5, 2012", formattedValue); + } + + @Test + public void testFormatValueVectorElementDateUnsupportedPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("yyyy-MM-dd HH:mm:ss.SS"); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalDate.of(2012, 11, 5), + TypeProtos.MinorType.DATE); + assertNull(formattedValue); + } + + @Test + public void testFormatValueVectorElementTimeEmptyPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn(""); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalTime.of(13, 0, 30, 120000000), + TypeProtos.MinorType.TIME); + assertEquals("13:00:30.120", formattedValue); + } + + @Test + public void testFormatValueVectorElementTimeValidPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("h:mm:ss a"); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalTime.of(13, 0, 30), + TypeProtos.MinorType.TIME); + assertEquals("1:00:30 PM", formattedValue); + } + + @Test + public void testFormatValueVectorElementTimeUnsupportedPattern() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("yyyy-MM-dd HH:mm:ss.SS"); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedValue = formatter.format( + LocalTime.of(13, 0, 30), + TypeProtos.MinorType.TIME); + assertNull(formattedValue); + } + + @Test + public void testFormatValueVectorElementAllDateTimeFormats() { + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIMESTAMP)).thenReturn("yyyy-MM-dd HH:mm:ss.SS"); + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_DATE)).thenReturn("EEE, MMM d, yyyy"); + when(options.getString(ExecConstants.WEB_DISPLAY_FORMAT_TIME)).thenReturn("h:mm:ss a"); + ValueVectorElementFormatter formatter = new ValueVectorElementFormatter(options); + String formattedTimestamp = formatter.format( + LocalDateTime.of(2012, 11, 5, 13, 0, 30, 120000000), + TypeProtos.MinorType.TIMESTAMP); + String formattedDate = formatter.format( + LocalDate.of(2012, 11, 5), + TypeProtos.MinorType.DATE); + String formattedTime = formatter.format( + LocalTime.of(13, 0, 30), + TypeProtos.MinorType.TIME); + assertEquals("2012-11-05 13:00:30.12", formattedTimestamp); + assertEquals("Mon, Nov 5, 2012", formattedDate); + assertEquals("1:00:30 PM", formattedTime); + } +}
