This is an automated email from the ASF dual-hosted git repository.
nehapawar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new e682d49 Add toDateTime DateTimeFunction (#5326)
e682d49 is described below
commit e682d49b2fdb6deadfe69d4efeda2d515cce07b0
Author: Charlie Summers <[email protected]>
AuthorDate: Fri May 15 10:31:04 2020 -0700
Add toDateTime DateTimeFunction (#5326)
Adds toDateTime and fromDateTime inbuilt functions (issue #5313 ).
1) toDateTime takes a long of millis since epoch and a pattern string and
returns a string corresponding to the DateTime since epoch as the passed
millis, formatted by the pattern.
2) fromDateTime takes in a DateTime string and a pattern that the DateTime
string is formatted in and returns a long of millis since epoch corresponding
to the DateTime string.
Also renamed DefaultFunctionEvaluator to InbuiltFunctionEvaluator and
FunctionRegistry to InbuiltFunctionRegistry. Adds a FunctionRegistryFactory to
create InbuiltFunctionRegistrys with a specified set of inbuilt functions. In
doing so, enables InbuiltFunctionRegistry to register both non-static and
static functions.
Adds a DateTimePatternHandler that converts DateTime strings to epoch longs
and epoch longs to DateTime strings backed by a cache of joda
DateTimeFormatters (each cache is specific to the particular upload).
Adds new tests for toDateTime and fromDateTime and an
InbuiltFunctionEvaluator test that makes sure the InbuiltFunctionRegistry
internal state persists between rows for the lifetime of the
InbuiltFunctionEvaluator.
---
.../core/data/function/DateTimeFunctions.java | 16 +++
.../core/data/function/DateTimePatternHandler.java | 54 +++++++++
.../data/function/FunctionEvaluatorFactory.java | 15 +--
.../pinot/core/data/function/FunctionRegistry.java | 101 ----------------
.../data/function/FunctionRegistryFactory.java | 81 +++++++++++++
...valuator.java => InbuiltFunctionEvaluator.java} | 9 +-
.../data/function/InbuiltFunctionRegistry.java | 55 +++++++++
.../recordtransformer/ExpressionTransformer.java | 4 +-
.../org/apache/pinot/core/util/SchemaUtils.java | 7 +-
.../function/DateTimeFunctionEvaluatorTest.java | 71 +++++++++---
.../function/DefaultFunctionEvaluatorTest.java | 102 -----------------
.../function/InbuiltFunctionEvaluatorTest.java | 127 +++++++++++++++++++++
12 files changed, 409 insertions(+), 233 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
index 24aa4db..2c51197 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimeFunctions.java
@@ -64,6 +64,8 @@ import java.util.concurrent.TimeUnit;
*/
public class DateTimeFunctions {
+ private final DateTimePatternHandler _dateTimePatternHandler = new
DateTimePatternHandler();
+
/**
* Convert epoch millis to epoch seconds
*/
@@ -203,4 +205,18 @@ public class DateTimeFunctions {
static Long fromEpochDaysBucket(Number daysSinceEpoch, Number bucket) {
return TimeUnit.DAYS.toMillis(daysSinceEpoch.longValue() *
bucket.intValue());
}
+
+ /**
+ * Converts epoch millis to DateTime string represented by pattern
+ */
+ String toDateTime(Long millis, String pattern) {
+ return _dateTimePatternHandler.parseEpochMillisToDateTimeString(millis,
pattern);
+ }
+
+ /**
+ * Converts DateTime string represented by pattern to epoch millis
+ */
+ Long fromDateTime(String dateTimeString, String pattern) {
+ return
_dateTimePatternHandler.parseDateTimeStringToEpochMillis(dateTimeString,
pattern);
+ }
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimePatternHandler.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimePatternHandler.java
new file mode 100644
index 0000000..ae9017d
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/DateTimePatternHandler.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.function;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
+
+
+/**
+ * Handles DateTime conversions from long to strings and strings to longs
based on passed patterns
+ */
+public class DateTimePatternHandler {
+ private final Map<String, DateTimeFormatter> patternCache = new
ConcurrentHashMap<>();
+
+ /**
+ * Converts the dateTimeString of passed pattern into a long of the millis
since epoch
+ */
+ public Long parseDateTimeStringToEpochMillis(String dateTimeString, String
pattern) {
+ DateTimeFormatter dateTimeFormatter =
getDateTimeFormatterFromCache(pattern);
+ return dateTimeFormatter.parseMillis(dateTimeString);
+ }
+
+ /**
+ * Converts the millis representing seconds since epoch into a string of
passed pattern
+ */
+ public String parseEpochMillisToDateTimeString(Long millis, String pattern) {
+ DateTimeFormatter dateTimeFormatter =
getDateTimeFormatterFromCache(pattern);
+ return dateTimeFormatter.print(millis);
+ }
+
+ private DateTimeFormatter getDateTimeFormatterFromCache(String pattern) {
+ // Note: withZoneUTC is overwritten if the timezone is specified directly
in the pattern
+ return patternCache
+ .computeIfAbsent(pattern, missingPattern ->
DateTimeFormat.forPattern(missingPattern).withZoneUTC());
+ }
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
index ddd4c5c..ef3f7e9 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionEvaluatorFactory.java
@@ -30,9 +30,8 @@ import org.apache.pinot.spi.data.TimeGranularitySpec;
*/
public class FunctionEvaluatorFactory {
- private FunctionEvaluatorFactory() {
-
- }
+ private final InbuiltFunctionRegistry _inbuiltFunctionRegistry =
+ FunctionRegistryFactory.getInbuiltFunctionRegistry();
/**
* Creates the {@link FunctionEvaluator} for the given field spec
@@ -43,7 +42,7 @@ public class FunctionEvaluatorFactory {
* 4. Return null, if none of the above
*/
@Nullable
- public static FunctionEvaluator getExpressionEvaluator(FieldSpec fieldSpec) {
+ public FunctionEvaluator getExpressionEvaluator(FieldSpec fieldSpec) {
FunctionEvaluator functionEvaluator = null;
String columnName = fieldSpec.getName();
@@ -73,12 +72,10 @@ public class FunctionEvaluatorFactory {
.getName() + " is same");
}
}
-
} else if (columnName.endsWith(SchemaUtils.MAP_KEY_COLUMN_SUFFIX)) {
// for backward compatible handling of Map type (currently only in Avro)
- String sourceMapName =
- columnName.substring(0, columnName.length() -
SchemaUtils.MAP_KEY_COLUMN_SUFFIX.length());
+ String sourceMapName = columnName.substring(0, columnName.length() -
SchemaUtils.MAP_KEY_COLUMN_SUFFIX.length());
String defaultMapKeysTransformExpression =
getDefaultMapKeysTransformExpression(sourceMapName);
functionEvaluator =
getExpressionEvaluator(defaultMapKeysTransformExpression);
} else if (columnName.endsWith(SchemaUtils.MAP_VALUE_COLUMN_SUFFIX)) {
@@ -91,13 +88,13 @@ public class FunctionEvaluatorFactory {
return functionEvaluator;
}
- private static FunctionEvaluator getExpressionEvaluator(String
transformExpression) {
+ private FunctionEvaluator getExpressionEvaluator(String transformExpression)
{
FunctionEvaluator functionEvaluator;
try {
if
(transformExpression.startsWith(GroovyFunctionEvaluator.getGroovyExpressionPrefix()))
{
functionEvaluator = new GroovyFunctionEvaluator(transformExpression);
} else {
- functionEvaluator = new DefaultFunctionEvaluator(transformExpression);
+ functionEvaluator = new InbuiltFunctionEvaluator(transformExpression,
_inbuiltFunctionRegistry);
}
} catch (Exception e) {
throw new IllegalStateException(
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
deleted file mode 100644
index fa5b88e..0000000
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistry.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.core.data.function;
-
-import com.google.common.base.Preconditions;
-import java.lang.reflect.Method;
-import java.lang.reflect.Modifier;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Registry for inbuilt Pinot functions
- */
-public class FunctionRegistry {
-
- private static final Logger LOGGER =
LoggerFactory.getLogger(FunctionRegistry.class);
-
- static Map<String, List<FunctionInfo>> _functionInfoMap = new HashMap<>();
-
- static {
- try {
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochSeconds",
Long.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochMinutes",
Long.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHours",
Long.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDays",
Long.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("toEpochSecondsRounded",
Long.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("toEpochMinutesRounded",
Long.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("toEpochHoursRounded",
Long.class, Number.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDaysRounded",
Long.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("toEpochSecondsBucket",
Long.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("toEpochMinutesBucket",
Long.class, Number.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochHoursBucket",
Long.class, Number.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("toEpochDaysBucket",
Long.class, Number.class));
-
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochSeconds",
Long.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochMinutes",
Number.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochHours",
Number.class));
-
registerStaticFunction(DateTimeFunctions.class.getDeclaredMethod("fromEpochDays",
Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("fromEpochSecondsBucket",
Long.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("fromEpochMinutesBucket",
Number.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("fromEpochHoursBucket",
Number.class, Number.class));
- registerStaticFunction(
- DateTimeFunctions.class.getDeclaredMethod("fromEpochDaysBucket",
Number.class, Number.class));
-
-
registerStaticFunction(JsonFunctions.class.getDeclaredMethod("toJsonMapStr",
Map.class));
- } catch (NoSuchMethodException e) {
- LOGGER.error("Caught exception when registering function", e);
- }
- }
-
- public static FunctionInfo resolve(String functionName, Class<?>[]
argumentTypes) {
- List<FunctionInfo> list = _functionInfoMap.get(functionName.toLowerCase());
- FunctionInfo bestMatch = null;
- if (list != null && list.size() > 0) {
- for (FunctionInfo functionInfo : list) {
- if (functionInfo.isApplicable(argumentTypes)) {
- bestMatch = functionInfo;
- break;
- }
- }
- }
- return bestMatch;
- }
-
- public static void registerStaticFunction(Method method) {
- Preconditions.checkArgument(Modifier.isStatic(method.getModifiers()),
"Method needs to be static:" + method);
- List<FunctionInfo> list = new ArrayList<>();
- FunctionInfo functionInfo = new FunctionInfo(method,
method.getDeclaringClass());
- list.add(functionInfo);
- _functionInfoMap.put(method.getName().toLowerCase(), list);
- }
-}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistryFactory.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistryFactory.java
new file mode 100644
index 0000000..ea13778
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/FunctionRegistryFactory.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.function;
+
+import com.google.common.collect.Lists;
+import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Factory class to create a FunctionRegistry (currently only {@link
InbuiltFunctionRegistry})
+ */
+public class FunctionRegistryFactory {
+ private static final Logger LOGGER =
LoggerFactory.getLogger(FunctionRegistryFactory.class);
+
+ private FunctionRegistryFactory() {
+
+ }
+
+ /**
+ * Creates an {@link InbuiltFunctionRegistry}
+ *
+ * The functionsToRegister list inside includes all the methods added to the
InbuiltFunctionRegistry
+ */
+ public static InbuiltFunctionRegistry getInbuiltFunctionRegistry() {
+ List<Method> functionsToRegister;
+ DateTimeFunctions dateTimeFunctions = new DateTimeFunctions();
+
+ try {
+ functionsToRegister = Lists
+
.newArrayList(dateTimeFunctions.getClass().getDeclaredMethod("toEpochSeconds",
Long.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("toEpochMinutes",
Long.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("toEpochHours",
Long.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("toEpochDays",
Long.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochSecondsRounded",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochMinutesRounded",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochHoursRounded",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochDaysRounded",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochSecondsBucket",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochMinutesBucket",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochHoursBucket",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("toEpochDaysBucket", Long.class,
Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("fromEpochSeconds", Long.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("fromEpochMinutes",
Number.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("fromEpochHours",
Number.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("fromEpochDays",
Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("fromEpochSecondsBucket",
Long.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("fromEpochMinutesBucket",
Number.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("fromEpochHoursBucket",
Number.class, Number.class),
+
dateTimeFunctions.getClass().getDeclaredMethod("fromEpochDaysBucket",
Number.class, Number.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("toDateTime",
Long.class, String.class),
+ dateTimeFunctions.getClass().getDeclaredMethod("fromDateTime",
String.class, String.class),
+
+ JsonFunctions.class.getDeclaredMethod("toJsonMapStr",
Map.class));
+ } catch (NoSuchMethodException e) {
+ LOGGER.error("Caught exception when registering function", e);
+ throw new IllegalStateException(e);
+ }
+
+ return new InbuiltFunctionRegistry(functionsToRegister);
+ }
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluator.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java
similarity index 92%
rename from
pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluator.java
rename to
pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java
index f41dfa8..0471930 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluator.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluator.java
@@ -44,14 +44,16 @@ import org.apache.pinot.spi.data.readers.GenericRow;
* </li>
* </ul>
*/
-public class DefaultFunctionEvaluator implements FunctionEvaluator {
+public class InbuiltFunctionEvaluator implements FunctionEvaluator {
// Root of the execution tree
private final ExecutableNode _rootNode;
+ private final InbuiltFunctionRegistry _inbuiltFunctionRegistry;
private final List<String> _arguments;
- public DefaultFunctionEvaluator(String expression)
+ public InbuiltFunctionEvaluator(String expression, InbuiltFunctionRegistry
inbuiltFunctionRegistry)
throws Exception {
_arguments = new ArrayList<>();
+ _inbuiltFunctionRegistry = inbuiltFunctionRegistry;
_rootNode =
planExecution(TransformExpressionTree.compileToExpressionTree(expression));
}
@@ -83,7 +85,8 @@ public class DefaultFunctionEvaluator implements
FunctionEvaluator {
argumentTypes[i] = childNode.getReturnType();
}
- FunctionInfo functionInfo = FunctionRegistry.resolve(transformName,
argumentTypes);
+ FunctionInfo functionInfo =
+
_inbuiltFunctionRegistry.getFunctionByNameWithApplicableArgumentTypes(transformName,
argumentTypes);
return new FunctionExecutionNode(functionInfo, childNodes);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionRegistry.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionRegistry.java
new file mode 100644
index 0000000..f26a2c5
--- /dev/null
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/function/InbuiltFunctionRegistry.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.function;
+
+import com.google.common.base.Preconditions;
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+
+/**
+ * Registry for inbuilt Pinot functions
+ */
+public class InbuiltFunctionRegistry {
+ private final Map<String, FunctionInfo> _functionInfoMap = new HashMap<>();
+
+ InbuiltFunctionRegistry(List<Method> functionsToRegister) {
+ for (Method function : functionsToRegister) {
+ registerFunction(function);
+ }
+ }
+
+ /**
+ * Given a function name and a set of argument types, asserts that a
corresponding function
+ * was registered during construction and returns it
+ */
+ public FunctionInfo getFunctionByNameWithApplicableArgumentTypes(String
functionName, Class<?>[] argumentTypes) {
+
Preconditions.checkArgument(_functionInfoMap.containsKey(functionName.toLowerCase()));
+ FunctionInfo functionInfo =
_functionInfoMap.get(functionName.toLowerCase());
+ Preconditions.checkArgument(functionInfo.isApplicable(argumentTypes));
+ return functionInfo;
+ }
+
+ private void registerFunction(Method method) {
+ FunctionInfo functionInfo = new FunctionInfo(method,
method.getDeclaringClass());
+ _functionInfoMap.put(method.getName().toLowerCase(), functionInfo);
+ }
+}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
index 00babe5..8815682 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/data/recordtransformer/ExpressionTransformer.java
@@ -37,9 +37,11 @@ public class ExpressionTransformer implements
RecordTransformer {
private final Map<String, FunctionEvaluator> _expressionEvaluators = new
HashMap<>();
public ExpressionTransformer(Schema schema) {
+ FunctionEvaluatorFactory functionEvaluatorFactory = new
FunctionEvaluatorFactory();
+
for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
if (!fieldSpec.isVirtualColumn()) {
- FunctionEvaluator functionEvaluator =
FunctionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
+ FunctionEvaluator functionEvaluator =
functionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
if (functionEvaluator != null) {
_expressionEvaluators.put(fieldSpec.getName(), functionEvaluator);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
b/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
index 550c754..84b9317 100644
--- a/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
+++ b/pinot-core/src/main/java/org/apache/pinot/core/util/SchemaUtils.java
@@ -49,10 +49,12 @@ public class SchemaUtils {
* TODO: for now, we assume that arguments to transform function are in the
source i.e. there's no columns which are derived from transformed columns
*/
public static Set<String> extractSourceFields(Schema schema) {
+ FunctionEvaluatorFactory functionEvaluatorFactory = new
FunctionEvaluatorFactory();
Set<String> sourceFieldNames = new HashSet<>();
+
for (FieldSpec fieldSpec : schema.getAllFieldSpecs()) {
if (!fieldSpec.isVirtualColumn()) {
- FunctionEvaluator functionEvaluator =
FunctionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
+ FunctionEvaluator functionEvaluator =
functionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
if (functionEvaluator != null) {
sourceFieldNames.addAll(functionEvaluator.getArguments());
}
@@ -83,7 +85,8 @@ public class SchemaUtils {
String column = fieldSpec.getName();
String transformFunction = fieldSpec.getTransformFunction();
if (transformFunction != null) {
- FunctionEvaluator functionEvaluator =
FunctionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
+ FunctionEvaluatorFactory functionEvaluatorFactory = new
FunctionEvaluatorFactory();
+ FunctionEvaluator functionEvaluator =
functionEvaluatorFactory.getExpressionEvaluator(fieldSpec);
if (functionEvaluator != null) {
List<String> arguments = functionEvaluator.getArguments();
// output column used as input
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
index c58484b..a35222c 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DateTimeFunctionEvaluatorTest.java
@@ -33,9 +33,11 @@ import org.testng.annotations.Test;
public class DateTimeFunctionEvaluatorTest {
@Test(dataProvider = "dateTimeFunctionsTestDataProvider")
- public void testDateTimeTransformFunctions(String transformFunction,
List<String> arguments, GenericRow row, Object result)
+ public void testDateTimeTransformFunctions(String transformFunction,
List<String> arguments, GenericRow row,
+ Object result)
throws Exception {
- DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(transformFunction);
+ InbuiltFunctionRegistry inbuiltFunctionRegistry =
FunctionRegistryFactory.getInbuiltFunctionRegistry();
+ InbuiltFunctionEvaluator evaluator = new
InbuiltFunctionEvaluator(transformFunction, inbuiltFunctionRegistry);
Assert.assertEquals(evaluator.getArguments(), arguments);
Assert.assertEquals(evaluator.evaluate(row), result);
}
@@ -52,12 +54,14 @@ public class DateTimeFunctionEvaluatorTest {
// toEpochSeconds w/ rounding
GenericRow row1_1 = new GenericRow();
row1_1.putValue("timestamp", 1578685189000L);
- inputs.add(new Object[]{"toEpochSecondsRounded(timestamp, 10)",
Lists.newArrayList("timestamp"), row1_1, 1578685180L});
+ inputs.add(
+ new Object[]{"toEpochSecondsRounded(timestamp, 10)",
Lists.newArrayList("timestamp"), row1_1, 1578685180L});
// toEpochSeconds w/ bucketing
GenericRow row1_2 = new GenericRow();
row1_2.putValue("timestamp", 1578685189000L);
- inputs.add(new Object[]{"toEpochSecondsBucket(timestamp, 10)",
Lists.newArrayList("timestamp"), row1_2, 157868518L});
+ inputs
+ .add(new Object[]{"toEpochSecondsBucket(timestamp, 10)",
Lists.newArrayList("timestamp"), row1_2, 157868518L});
// toEpochMinutes
GenericRow row2_0 = new GenericRow();
@@ -67,7 +71,8 @@ public class DateTimeFunctionEvaluatorTest {
// toEpochMinutes w/ rounding
GenericRow row2_1 = new GenericRow();
row2_1.putValue("timestamp", 1578685189000L);
- inputs.add(new Object[]{"toEpochMinutesRounded(timestamp, 15)",
Lists.newArrayList("timestamp"), row2_1, 26311410L});
+ inputs
+ .add(new Object[]{"toEpochMinutesRounded(timestamp, 15)",
Lists.newArrayList("timestamp"), row2_1, 26311410L});
// toEpochMinutes w/ bucketing
GenericRow row2_2 = new GenericRow();
@@ -107,8 +112,8 @@ public class DateTimeFunctionEvaluatorTest {
// fromEpochDays
GenericRow row5_0 = new GenericRow();
row5_0.putValue("daysSinceEpoch", 14000);
- inputs
- .add(new Object[]{"fromEpochDays(daysSinceEpoch)",
Lists.newArrayList("daysSinceEpoch"), row5_0, 1209600000000L});
+ inputs.add(
+ new Object[]{"fromEpochDays(daysSinceEpoch)",
Lists.newArrayList("daysSinceEpoch"), row5_0, 1209600000000L});
// fromEpochDays w/ bucketing
GenericRow row5_1 = new GenericRow();
@@ -119,8 +124,8 @@ public class DateTimeFunctionEvaluatorTest {
// fromEpochHours
GenericRow row6_0 = new GenericRow();
row6_0.putValue("hoursSinceEpoch", 336000);
- inputs
- .add(new Object[]{"fromEpochHours(hoursSinceEpoch)",
Lists.newArrayList("hoursSinceEpoch"), row6_0, 1209600000000L});
+ inputs.add(
+ new Object[]{"fromEpochHours(hoursSinceEpoch)",
Lists.newArrayList("hoursSinceEpoch"), row6_0, 1209600000000L});
// fromEpochHours w/ bucketing
GenericRow row6_1 = new GenericRow();
@@ -131,8 +136,8 @@ public class DateTimeFunctionEvaluatorTest {
// fromEpochMinutes
GenericRow row7_0 = new GenericRow();
row7_0.putValue("minutesSinceEpoch", 20160000);
- inputs
- .add(new Object[]{"fromEpochMinutes(minutesSinceEpoch)",
Lists.newArrayList("minutesSinceEpoch"), row7_0, 1209600000000L});
+ inputs.add(new Object[]{"fromEpochMinutes(minutesSinceEpoch)",
Lists.newArrayList(
+ "minutesSinceEpoch"), row7_0, 1209600000000L});
// fromEpochMinutes w/ bucketing
GenericRow row7_1 = new GenericRow();
@@ -143,8 +148,8 @@ public class DateTimeFunctionEvaluatorTest {
// fromEpochSeconds
GenericRow row8_0 = new GenericRow();
row8_0.putValue("secondsSinceEpoch", 1209600000L);
- inputs
- .add(new Object[]{"fromEpochSeconds(secondsSinceEpoch)",
Lists.newArrayList("secondsSinceEpoch"), row8_0, 1209600000000L});
+ inputs.add(new Object[]{"fromEpochSeconds(secondsSinceEpoch)",
Lists.newArrayList(
+ "secondsSinceEpoch"), row8_0, 1209600000000L});
// fromEpochSeconds w/ bucketing
GenericRow row8_1 = new GenericRow();
@@ -160,8 +165,44 @@ public class DateTimeFunctionEvaluatorTest {
GenericRow row9_1 = new GenericRow();
row9_1.putValue("fifteenSecondsSinceEpoch", 80640000L);
- inputs.add(new
Object[]{"toEpochMinutesBucket(fromEpochSecondsBucket(fifteenSecondsSinceEpoch,
15), 10)", Lists.newArrayList(
- "fifteenSecondsSinceEpoch"), row9_1, 2016000L});
+ inputs.add(
+ new
Object[]{"toEpochMinutesBucket(fromEpochSecondsBucket(fifteenSecondsSinceEpoch,
15), 10)", Lists.newArrayList(
+ "fifteenSecondsSinceEpoch"), row9_1, 2016000L});
+
+ // toDateTime simple
+ GenericRow row10_0 = new GenericRow();
+ row10_0.putValue("dateTime", 98697600000L);
+ inputs.add(new Object[]{"toDateTime(dateTime, 'yyyyMMdd')",
Lists.newArrayList("dateTime"), row10_0, "19730216"});
+
+ // toDateTime complex
+ GenericRow row10_1 = new GenericRow();
+ row10_1.putValue("dateTime", 1234567890000L);
+ inputs.add(new Object[]{"toDateTime(dateTime, 'MM/yyyy/dd HH:mm:ss')",
Lists.newArrayList(
+ "dateTime"), row10_1, "02/2009/13 23:31:30"});
+
+ // toDateTime with timezone
+ GenericRow row10_2 = new GenericRow();
+ row10_2.putValue("dateTime", 7897897890000L);
+ inputs.add(new Object[]{"toDateTime(dateTime, 'EEE MMM dd HH:mm:ss z
yyyy')", Lists.newArrayList(
+ "dateTime"), row10_2, "Mon Apr 10 20:31:30 +00:00 2220"});
+
+ // fromDateTime simple
+ GenericRow row11_0 = new GenericRow();
+ row11_0.putValue("dateTime", "19730216");
+ inputs
+ .add(new Object[]{"fromDateTime(dateTime, 'yyyyMMdd')",
Lists.newArrayList("dateTime"), row11_0, 98668800000L});
+
+ // fromDateTime complex
+ GenericRow row11_1 = new GenericRow();
+ row11_1.putValue("dateTime", "02/2009/13 15:31:30");
+ inputs.add(new Object[]{"fromDateTime(dateTime, 'MM/yyyy/dd HH:mm:ss')",
Lists.newArrayList(
+ "dateTime"), row11_1, 1234539090000L});
+
+ // fromDateTime with timezone
+ GenericRow row11_2 = new GenericRow();
+ row11_2.putValue("dateTime", "Mon Aug 24 12:36:46 America/Los_Angeles
2009");
+ inputs.add(new Object[]{"fromDateTime(dateTime, \"EEE MMM dd HH:mm:ss ZZZ
yyyy\")", Lists.newArrayList(
+ "dateTime"), row11_2, 1251142606000L});
return inputs.toArray(new Object[0][]);
}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluatorTest.java
deleted file mode 100644
index 7c6d4cf..0000000
---
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/DefaultFunctionEvaluatorTest.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pinot.core.data.function;
-
-import com.google.common.collect.Lists;
-import java.lang.reflect.Method;
-import org.apache.pinot.spi.data.readers.GenericRow;
-import org.joda.time.DateTime;
-import org.joda.time.Days;
-import org.joda.time.MutableDateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-
-public class DefaultFunctionEvaluatorTest {
-
- @Test
- public void testExpressionWithColumn()
- throws Exception {
- Method method = MyFunc.class.getDeclaredMethod("reverseString",
String.class);
- FunctionRegistry.registerStaticFunction(method);
- FunctionInfo functionInfo = FunctionRegistry.resolve("reverseString", new
Class<?>[]{Object.class});
- System.out.println(functionInfo);
- String expression = "reverseString(testColumn)";
-
- DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(expression);
- Assert.assertEquals(evaluator.getArguments(),
Lists.newArrayList("testColumn"));
- GenericRow row = new GenericRow();
- for (int i = 0; i < 5; i++) {
- String value = "testValue" + i;
- row.putField("testColumn", value);
- Object result = evaluator.evaluate(row);
- Assert.assertEquals(result, new
StringBuilder(value).reverse().toString());
- }
- }
-
- @Test
- public void testExpressionWithConstant()
- throws Exception {
- FunctionRegistry
-
.registerStaticFunction(MyFunc.class.getDeclaredMethod("daysSinceEpoch",
String.class, String.class));
- String input = "1980-01-01";
- String format = "yyyy-MM-dd";
- String expression = String.format("daysSinceEpoch('%s', '%s')", input,
format);
- DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(expression);
- Assert.assertTrue(evaluator.getArguments().isEmpty());
- GenericRow row = new GenericRow();
- Object result = evaluator.evaluate(row);
- Assert.assertEquals(result, MyFunc.daysSinceEpoch(input, format));
- }
-
- @Test
- public void testMultiFunctionExpression()
- throws Exception {
-
FunctionRegistry.registerStaticFunction(MyFunc.class.getDeclaredMethod("reverseString",
String.class));
- FunctionRegistry
-
.registerStaticFunction(MyFunc.class.getDeclaredMethod("daysSinceEpoch",
String.class, String.class));
- String input = "1980-01-01";
- String reversedInput = MyFunc.reverseString(input);
- String format = "yyyy-MM-dd";
- String expression = String.format("daysSinceEpoch(reverseString('%s'),
'%s')", reversedInput, format);
- DefaultFunctionEvaluator evaluator = new
DefaultFunctionEvaluator(expression);
- Assert.assertTrue(evaluator.getArguments().isEmpty());
- GenericRow row = new GenericRow();
- Object result = evaluator.evaluate(row);
- Assert.assertEquals(result, MyFunc.daysSinceEpoch(input, format));
- }
-
- private static class MyFunc {
- static String reverseString(String input) {
- return new StringBuilder(input).reverse().toString();
- }
-
- static MutableDateTime EPOCH_START = new MutableDateTime();
-
- static {
- EPOCH_START.setDate(0L); // Set to Epoch time
- }
-
- static int daysSinceEpoch(String input, String format) {
- DateTime dateTime =
DateTimeFormat.forPattern(format).parseDateTime(input);
- return Days.daysBetween(EPOCH_START, dateTime).getDays();
- }
- }
-}
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java
new file mode 100644
index 0000000..c06a40d
--- /dev/null
+++
b/pinot-core/src/test/java/org/apache/pinot/core/data/function/InbuiltFunctionEvaluatorTest.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.core.data.function;
+
+import com.google.common.collect.Lists;
+import org.apache.pinot.spi.data.readers.GenericRow;
+import org.joda.time.DateTime;
+import org.joda.time.Days;
+import org.joda.time.MutableDateTime;
+import org.joda.time.format.DateTimeFormat;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+public class InbuiltFunctionEvaluatorTest {
+
+ @Test
+ public void testExpressionWithColumn()
+ throws Exception {
+ MyFunc myFunc = new MyFunc();
+ InbuiltFunctionRegistry inbuiltFunctionRegistry = new
InbuiltFunctionRegistry(
+
Lists.newArrayList(myFunc.getClass().getDeclaredMethod("reverseString",
String.class)));
+ FunctionInfo functionInfo = inbuiltFunctionRegistry
+ .getFunctionByNameWithApplicableArgumentTypes("reverseString", new
Class<?>[]{Object.class});
+ System.out.println(functionInfo);
+ String expression = "reverseString(testColumn)";
+
+ InbuiltFunctionEvaluator evaluator = new
InbuiltFunctionEvaluator(expression, inbuiltFunctionRegistry);
+ Assert.assertEquals(evaluator.getArguments(),
Lists.newArrayList("testColumn"));
+ GenericRow row = new GenericRow();
+ for (int i = 0; i < 5; i++) {
+ String value = "testValue" + i;
+ row.putField("testColumn", value);
+ Object result = evaluator.evaluate(row);
+ Assert.assertEquals(result, new
StringBuilder(value).reverse().toString());
+ }
+ }
+
+ @Test
+ public void testExpressionWithConstant()
+ throws Exception {
+ MyFunc myFunc = new MyFunc();
+ InbuiltFunctionRegistry inbuiltFunctionRegistry = new
InbuiltFunctionRegistry(
+
Lists.newArrayList(myFunc.getClass().getDeclaredMethod("daysSinceEpoch",
String.class, String.class)));
+ String input = "1980-01-01";
+ String format = "yyyy-MM-dd";
+ String expression = String.format("daysSinceEpoch('%s', '%s')", input,
format);
+ InbuiltFunctionEvaluator evaluator = new
InbuiltFunctionEvaluator(expression, inbuiltFunctionRegistry);
+ Assert.assertTrue(evaluator.getArguments().isEmpty());
+ GenericRow row = new GenericRow();
+ Object result = evaluator.evaluate(row);
+ Assert.assertEquals(result, myFunc.daysSinceEpoch(input, format));
+ }
+
+ @Test
+ public void testMultiFunctionExpression()
+ throws Exception {
+ MyFunc myFunc = new MyFunc();
+ InbuiltFunctionRegistry inbuiltFunctionRegistry = new
InbuiltFunctionRegistry(Lists
+ .newArrayList(myFunc.getClass().getDeclaredMethod("reverseString",
String.class),
+ myFunc.getClass().getDeclaredMethod("daysSinceEpoch",
String.class, String.class)));
+ String input = "1980-01-01";
+ String reversedInput = myFunc.reverseString(input);
+ String format = "yyyy-MM-dd";
+ String expression = String.format("daysSinceEpoch(reverseString('%s'),
'%s')", reversedInput, format);
+ InbuiltFunctionEvaluator evaluator = new
InbuiltFunctionEvaluator(expression, inbuiltFunctionRegistry);
+ Assert.assertTrue(evaluator.getArguments().isEmpty());
+ GenericRow row = new GenericRow();
+ Object result = evaluator.evaluate(row);
+ Assert.assertEquals(result, myFunc.daysSinceEpoch(input, format));
+ }
+
+ @Test
+ public void testStateSharedBetweenRowsForExecution()
+ throws Exception {
+ MyFunc myFunc = new MyFunc();
+ InbuiltFunctionRegistry inbuiltFunctionRegistry = new
InbuiltFunctionRegistry(
+
Lists.newArrayList(myFunc.getClass().getDeclaredMethod("appendToStringAndReturn",
String.class)));
+ String expression = String.format("appendToStringAndReturn('%s')", "test
");
+ InbuiltFunctionEvaluator evaluator = new
InbuiltFunctionEvaluator(expression, inbuiltFunctionRegistry);
+ Assert.assertTrue(evaluator.getArguments().isEmpty());
+ GenericRow row = new GenericRow();
+ Assert.assertEquals(evaluator.evaluate(row), "test ");
+ Assert.assertEquals(evaluator.evaluate(row), "test test ");
+ Assert.assertEquals(evaluator.evaluate(row), "test test test ");
+ }
+}
+
+class MyFunc {
+ String reverseString(String input) {
+ return new StringBuilder(input).reverse().toString();
+ }
+
+ MutableDateTime EPOCH_START = new MutableDateTime();
+
+ public MyFunc() {
+ EPOCH_START.setDate(0L);
+ }
+
+ int daysSinceEpoch(String input, String format) {
+ DateTime dateTime = DateTimeFormat.forPattern(format).parseDateTime(input);
+ return Days.daysBetween(EPOCH_START, dateTime).getDays();
+ }
+
+ private String baseString = "";
+
+ String appendToStringAndReturn(String addedString) {
+ baseString += addedString;
+ return baseString;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
