This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9327fc4 [IOTDB-1071] Built-in UDF registration service (#2380)
9327fc4 is described below
commit 9327fc49950d7b902b492d8fa05835c80a78e94b
Author: Steve Yurong Su <[email protected]>
AuthorDate: Wed Dec 30 20:02:40 2020 +0800
[IOTDB-1071] Built-in UDF registration service (#2380)
---
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 11 +-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 10 ++
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 62 +++++++-
.../apache/iotdb/db/query/dataset/ListDataSet.java | 12 +-
.../BuiltinFunction.java} | 100 ++++++-------
.../udf/service/UDFRegistrationInformation.java | 25 +++-
.../query/udf/service/UDFRegistrationService.java | 137 ++++++++++++-----
.../iotdb/db/integration/IoTDBUDFManagementIT.java | 162 +++++++++++++++++++--
8 files changed, 397 insertions(+), 122 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
index 679bc85..edd8607 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConstant.java
@@ -86,8 +86,15 @@ public class IoTDBConstant {
public static final String COLUMN_CANCELLED = "cancelled";
public static final String COLUMN_DONE = "done";
- public static final String COLUMN_FUNCTION_NAME = "UDF name";
- public static final String COLUMN_FUNCTION_CLASS = "class name";
+ public static final String COLUMN_FUNCTION_NAME = "function name";
+ public static final String COLUMN_FUNCTION_TYPE = "function type";
+ public static final String COLUMN_FUNCTION_CLASS = "class name (UDF)";
+
+ public static final String FUNCTION_TYPE_NATIVE = "native";
+ public static final String FUNCTION_TYPE_BUILTIN_UDAF = "built-in UDAF";
+ public static final String FUNCTION_TYPE_BUILTIN_UDTF = "built-in UDTF";
+ public static final String FUNCTION_TYPE_EXTERNAL_UDAF = "external UDAF";
+ public static final String FUNCTION_TYPE_EXTERNAL_UDTF = "external UDTF";
public static final String PATH_WILDCARD = "*";
public static final String TIME = "time";
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
index 3109af4..ecbc667 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/constant/SQLConstant.java
@@ -18,8 +18,11 @@
*/
package org.apache.iotdb.db.qp.constant;
+import java.util.Arrays;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.db.qp.sql.SqlBaseLexer;
@@ -70,6 +73,9 @@ public class SQLConstant {
public static final String ALL = "all";
+ private static final Set<String> NATIVE_FUNCTION_NAMES = new
HashSet<>(Arrays.asList(
+ MIN_TIME, MAX_TIME, MIN_VALUE, MAX_VALUE, FIRST_VALUE, LAST_VALUE,
COUNT, SUM, AVG));
+
public static final int KW_AND = 1;
public static final int KW_OR = 2;
public static final int KW_NOT = 3;
@@ -265,4 +271,8 @@ public class SQLConstant {
public static boolean isReservedPath(PartialPath pathStr) {
return pathStr.equals(TIME_PATH);
}
+
+ public static Set<String> getNativeFunctionNames() {
+ return NATIVE_FUNCTION_NAMES;
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
index 7313109..7130bdb 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/executor/PlanExecutor.java
@@ -27,6 +27,7 @@ import static
org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_DEVICES;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_DONE;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_CLASS;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_NAME;
+import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_FUNCTION_TYPE;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_ITEM;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PRIVILEGE;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_PROGRESS;
@@ -36,10 +37,16 @@ import static
org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TASK_NAME;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_TTL;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_USER;
import static org.apache.iotdb.db.conf.IoTDBConstant.COLUMN_VALUE;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDAF;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDTF;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_NATIVE;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDAF;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDTF;
import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -83,6 +90,7 @@ import org.apache.iotdb.db.metadata.mnode.MNode;
import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.metadata.mnode.StorageGroupMNode;
import org.apache.iotdb.db.monitor.StatMonitor;
+import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
import org.apache.iotdb.db.qp.logical.sys.AuthorOperator;
import org.apache.iotdb.db.qp.logical.sys.AuthorOperator.AuthorType;
@@ -690,28 +698,78 @@ public class PlanExecutor implements IPlanExecutor {
return listDataSet;
}
- private QueryDataSet processShowFunctions(ShowFunctionsPlan showPlan) {
+ private QueryDataSet processShowFunctions(ShowFunctionsPlan showPlan)
+ throws QueryProcessException {
ListDataSet listDataSet = new ListDataSet(
Arrays.asList(
new PartialPath(COLUMN_FUNCTION_NAME, false),
+ new PartialPath(COLUMN_FUNCTION_TYPE, false),
new PartialPath(COLUMN_FUNCTION_CLASS, false)
),
Arrays.asList(
TSDataType.TEXT,
+ TSDataType.TEXT,
TSDataType.TEXT
)
);
+
+ appendUDFs(listDataSet, showPlan);
+ appendNativeFunctions(listDataSet, showPlan);
+
+ listDataSet.sort((r1, r2) -> String.CASE_INSENSITIVE_ORDER
+ .compare(r1.getFields().get(0).getStringValue(),
r2.getFields().get(0).getStringValue()));
+ return listDataSet;
+ }
+
+ @SuppressWarnings("squid:S3776")
+ private void appendUDFs(ListDataSet listDataSet, ShowFunctionsPlan showPlan)
+ throws QueryProcessException {
for (UDFRegistrationInformation info : UDFRegistrationService.getInstance()
.getRegistrationInformation()) {
if (showPlan.showTemporary() && !info.isTemporary()) {
continue;
}
+
RowRecord rowRecord = new RowRecord(0); // ignore timestamp
rowRecord.addField(Binary.valueOf(info.getFunctionName()),
TSDataType.TEXT);
+ String functionType = "";
+ try {
+ if (info.isBuiltin()) {
+ if (info.isUDTF()) {
+ functionType = FUNCTION_TYPE_BUILTIN_UDTF;
+ } else if (info.isUDAF()) {
+ functionType = FUNCTION_TYPE_BUILTIN_UDAF;
+ }
+ } else {
+ if (info.isUDTF()) {
+ functionType = FUNCTION_TYPE_EXTERNAL_UDTF;
+ } else if (info.isUDAF()) {
+ functionType = FUNCTION_TYPE_EXTERNAL_UDAF;
+ }
+ }
+ } catch (InstantiationException | InvocationTargetException |
NoSuchMethodException | IllegalAccessException e) {
+ throw new QueryProcessException(e.toString());
+ }
+ rowRecord.addField(Binary.valueOf(functionType), TSDataType.TEXT);
rowRecord.addField(Binary.valueOf(info.getClassName()), TSDataType.TEXT);
listDataSet.putRecord(rowRecord);
}
- return listDataSet;
+ }
+
+ private void appendNativeFunctions(ListDataSet listDataSet,
ShowFunctionsPlan showPlan) {
+ if (showPlan.showTemporary()) {
+ return;
+ }
+
+ final Binary functionType = Binary.valueOf(FUNCTION_TYPE_NATIVE);
+ final Binary className = Binary.valueOf("");
+ for (String functionName : SQLConstant.getNativeFunctionNames()) {
+ RowRecord rowRecord = new RowRecord(0); // ignore timestamp
+ rowRecord.addField(Binary.valueOf(functionName), TSDataType.TEXT);
+ rowRecord.addField(functionType, TSDataType.TEXT);
+ rowRecord.addField(className, TSDataType.TEXT);
+ listDataSet.putRecord(rowRecord);
+ }
}
private void addRowRecordForShowQuery(
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/dataset/ListDataSet.java
b/server/src/main/java/org/apache/iotdb/db/query/dataset/ListDataSet.java
index a312750..c8620ac 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/dataset/ListDataSet.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/dataset/ListDataSet.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.query.dataset;
import java.util.ArrayList;
+import java.util.Comparator;
import java.util.List;
import org.apache.iotdb.db.metadata.PartialPath;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -28,11 +29,10 @@ import
org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
public class ListDataSet extends QueryDataSet {
- private List<RowRecord> records = new ArrayList<>();
+ private final List<RowRecord> records = new ArrayList<>();
private int index = 0;
- public ListDataSet(List<PartialPath> paths,
- List<TSDataType> dataTypes) {
+ public ListDataSet(List<PartialPath> paths, List<TSDataType> dataTypes) {
super(new ArrayList<>(paths), dataTypes);
}
@@ -51,6 +51,10 @@ public class ListDataSet extends QueryDataSet {
}
public void sortByTime() {
- records.sort(((o1, o2) -> Long.compare(o2.getTimestamp(),
o1.getTimestamp())));
+ records.sort((o1, o2) -> Long.compare(o2.getTimestamp(),
o1.getTimestamp()));
+ }
+
+ public void sort(Comparator<RowRecord> c) {
+ records.sort(c);
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
similarity index 60%
copy from
server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
copy to
server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
index c25bf69..c8bb083 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/udf/builtin/BuiltinFunction.java
@@ -1,57 +1,43 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.db.query.udf.service;
-
-public class UDFRegistrationInformation {
-
- private final String functionName;
- private final String className;
- private final boolean isTemporary;
-
- private Class<?> functionClass;
-
- public UDFRegistrationInformation(String functionName, String className,
boolean isTemporary,
- Class<?> functionClass) {
- this.functionName = functionName;
- this.className = className;
- this.isTemporary = isTemporary;
- this.functionClass = functionClass;
- }
-
- public String getFunctionName() {
- return functionName;
- }
-
- public String getClassName() {
- return className;
- }
-
- public boolean isTemporary() {
- return isTemporary;
- }
-
- public Class<?> getFunctionClass() {
- return functionClass;
- }
-
- public void updateFunctionClass(UDFClassLoader udfClassLoader) throws
ClassNotFoundException {
- functionClass = Class.forName(className, true, udfClassLoader);
- }
-}
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.udf.builtin;
+
+/**
+ * All built-in UDFs need to register function names and full class names here.
+ */
+public enum BuiltinFunction {
+ ;
+
+ private final String functionName;
+ private final String className;
+
+ BuiltinFunction(String functionName, String className) {
+ this.functionName = functionName;
+ this.className = className;
+ }
+
+ public String getFunctionName() {
+ return functionName;
+ }
+
+ public String getClassName() {
+ return className;
+ }
+}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
b/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
index c25bf69..2c072a9 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationInformation.java
@@ -19,19 +19,24 @@
package org.apache.iotdb.db.query.udf.service;
+import java.lang.reflect.InvocationTargetException;
+import org.apache.iotdb.db.query.udf.api.UDTF;
+
public class UDFRegistrationInformation {
private final String functionName;
private final String className;
private final boolean isTemporary;
+ private final boolean isBuiltin;
private Class<?> functionClass;
public UDFRegistrationInformation(String functionName, String className,
boolean isTemporary,
- Class<?> functionClass) {
+ boolean isBuiltin, Class<?> functionClass) {
this.functionName = functionName;
this.className = className;
this.isTemporary = isTemporary;
+ this.isBuiltin = isBuiltin;
this.functionClass = functionClass;
}
@@ -43,8 +48,15 @@ public class UDFRegistrationInformation {
return className;
}
+ /**
+ * For a builtin function, this method always returns false.
+ */
public boolean isTemporary() {
- return isTemporary;
+ return !isBuiltin && isTemporary;
+ }
+
+ public boolean isBuiltin() {
+ return isBuiltin;
}
public Class<?> getFunctionClass() {
@@ -54,4 +66,13 @@ public class UDFRegistrationInformation {
public void updateFunctionClass(UDFClassLoader udfClassLoader) throws
ClassNotFoundException {
functionClass = Class.forName(className, true, udfClassLoader);
}
+
+ public boolean isUDTF()
+ throws NoSuchMethodException, IllegalAccessException,
InvocationTargetException, InstantiationException {
+ return functionClass.getDeclaredConstructor().newInstance() instanceof
UDTF;
+ }
+
+ public boolean isUDAF() {
+ return false;
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationService.java
b/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationService.java
index 1ef46be..62e82ec 100644
---
a/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationService.java
+++
b/server/src/main/java/org/apache/iotdb/db/query/udf/service/UDFRegistrationService.java
@@ -24,11 +24,8 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
-import java.util.Arrays;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map.Entry;
-import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -40,6 +37,7 @@ import org.apache.iotdb.db.exception.UDFRegistrationException;
import org.apache.iotdb.db.exception.query.QueryProcessException;
import org.apache.iotdb.db.qp.constant.SQLConstant;
import org.apache.iotdb.db.query.udf.api.UDF;
+import org.apache.iotdb.db.query.udf.builtin.BuiltinFunction;
import org.apache.iotdb.db.query.udf.core.context.UDFContext;
import org.apache.iotdb.db.service.IService;
import org.apache.iotdb.db.service.ServiceType;
@@ -58,11 +56,6 @@ public class UDFRegistrationService implements IService {
private static final String LOG_FILE_NAME = ULOG_FILE_DIR + "ulog.txt";
private static final String TEMPORARY_LOG_FILE_NAME = LOG_FILE_NAME + ".tmp";
- private static final Set<String> BUILTIN_FUNCTION_NAMES = new
HashSet<>(Arrays.asList(
- SQLConstant.MIN_TIME, SQLConstant.MAX_TIME, SQLConstant.MIN_VALUE,
SQLConstant.MAX_VALUE,
- SQLConstant.FIRST_VALUE, SQLConstant.LAST_VALUE, SQLConstant.COUNT,
SQLConstant.SUM,
- SQLConstant.AVG));
-
private final ReentrantLock registrationLock;
private final ConcurrentHashMap<String, UDFRegistrationInformation>
registrationInformation;
@@ -83,14 +76,41 @@ public class UDFRegistrationService implements IService {
registrationLock.unlock();
}
- @SuppressWarnings("squid:S3776") // Suppress high Cognitive Complexity
warning
public void register(String functionName, String className, boolean
isTemporary,
boolean writeToTemporaryLogFile) throws UDFRegistrationException {
validateFunctionName(functionName, className);
+ checkIfRegistered(functionName, className, isTemporary);
+ doRegister(functionName, className, isTemporary);
+ tryAppendRegistrationLog(functionName, className, isTemporary,
writeToTemporaryLogFile);
+ }
+
+ private static void validateFunctionName(String functionName, String
className)
+ throws UDFRegistrationException {
+ if
(!SQLConstant.getNativeFunctionNames().contains(functionName.toLowerCase())) {
+ return;
+ }
+
+ String errorMessage = String.format(
+ "Failed to register UDF %s(%s), because the given function name
conflicts with the built-in function name",
+ functionName, className);
+ logger.warn(errorMessage);
+ throw new UDFRegistrationException(errorMessage);
+ }
+
+ private void checkIfRegistered(String functionName, String className,
boolean isTemporary)
+ throws UDFRegistrationException {
UDFRegistrationInformation information =
registrationInformation.get(functionName);
- if (information != null) {
- String errorMessage;
+ if (information == null) {
+ return;
+ }
+
+ String errorMessage;
+ if (information.isBuiltin()) {
+ errorMessage = String.format(
+ "Failed to register UDF %s(%s), because the given function name is
the same as a built-in UDF function name.",
+ functionName, className);
+ } else {
if (information.getClassName().equals(className)) {
errorMessage = String.format(
"Failed to register %sTEMPORARY UDF %s(%s), because a %sTEMPORARY
UDF %s(%s) with the same function name and the class name has already been
registered.",
@@ -103,10 +123,14 @@ public class UDFRegistrationService implements IService {
functionName, className,
information.getFunctionName(), information.getClassName());
}
- logger.warn(errorMessage);
- throw new UDFRegistrationException(errorMessage);
}
+ logger.warn(errorMessage);
+ throw new UDFRegistrationException(errorMessage);
+ }
+
+ private void doRegister(String functionName, String className, boolean
isTemporary)
+ throws UDFRegistrationException {
acquireRegistrationLock();
try {
UDFClassLoader currentActiveClassLoader =
UDFClassLoaderManager.getInstance()
@@ -116,7 +140,8 @@ public class UDFRegistrationService implements IService {
Class<?> functionClass = Class.forName(className, true,
currentActiveClassLoader);
functionClass.getDeclaredConstructor().newInstance();
registrationInformation.put(functionName,
- new UDFRegistrationInformation(functionName, className, isTemporary,
functionClass));
+ new UDFRegistrationInformation(functionName, className, isTemporary,
false,
+ functionClass));
} catch (IOException | InstantiationException | InvocationTargetException
| NoSuchMethodException | IllegalAccessException | ClassNotFoundException e) {
String errorMessage = String.format(
"Failed to register UDF %s(%s), because its instance can not be
constructed successfully. Exception: %s",
@@ -126,36 +151,32 @@ public class UDFRegistrationService implements IService {
} finally {
releaseRegistrationLock();
}
+ }
- if (writeToTemporaryLogFile && !isTemporary) {
- try {
- appendRegistrationLog(functionName, className);
- } catch (IOException e) {
- registrationInformation.remove(functionName);
- String errorMessage = String
- .format("Failed to append UDF log when registering UDF %s(%s),
because %s",
- functionName, className, e.toString());
- logger.error(errorMessage);
- throw new UDFRegistrationException(errorMessage, e);
- }
+ private void tryAppendRegistrationLog(String functionName, String className,
boolean isTemporary,
+ boolean writeToTemporaryLogFile) throws UDFRegistrationException {
+ if (!writeToTemporaryLogFile || isTemporary) {
+ return;
}
- }
- private static void validateFunctionName(String functionName, String
className)
- throws UDFRegistrationException {
- if (BUILTIN_FUNCTION_NAMES.contains(functionName.toLowerCase())) {
- String errorMessage = String.format(
- "Failed to register UDF %s(%s), because the given function name
conflicts with the built-in function name",
- functionName, className);
- logger.warn(errorMessage);
- throw new UDFRegistrationException(errorMessage);
+ try {
+ appendRegistrationLog(functionName, className);
+ } catch (IOException e) {
+ registrationInformation.remove(functionName);
+ String errorMessage = String
+ .format("Failed to append UDF log when registering UDF %s(%s),
because %s",
+ functionName, className, e.toString());
+ logger.error(errorMessage);
+ throw new UDFRegistrationException(errorMessage, e);
}
}
private void updateAllRegisteredClasses(UDFClassLoader activeClassLoader)
throws ClassNotFoundException {
for (UDFRegistrationInformation information :
getRegistrationInformation()) {
- information.updateFunctionClass(activeClassLoader);
+ if (!information.isBuiltin()) {
+ information.updateFunctionClass(activeClassLoader);
+ }
}
}
@@ -167,6 +188,13 @@ public class UDFRegistrationService implements IService {
throw new UDFRegistrationException(errorMessage);
}
+ if (information.isBuiltin()) {
+ String errorMessage = String
+ .format("Built-in function %s can not be deregistered.",
functionName);
+ logger.error(errorMessage);
+ throw new UDFRegistrationException(errorMessage);
+ }
+
if (!information.isTemporary()) {
try {
appendDeregistrationLog(functionName);
@@ -209,8 +237,11 @@ public class UDFRegistrationService implements IService {
throw new QueryProcessException(errorMessage);
}
- Thread.currentThread()
-
.setContextClassLoader(UDFClassLoaderManager.getInstance().getActiveClassLoader());
+ if (!information.isBuiltin()) {
+ Thread.currentThread()
+
.setContextClassLoader(UDFClassLoaderManager.getInstance().getActiveClassLoader());
+ }
+
try {
return (UDF)
information.getFunctionClass().getDeclaredConstructor().newInstance();
} catch (InstantiationException | InvocationTargetException |
NoSuchMethodException | IllegalAccessException e) {
@@ -228,6 +259,7 @@ public class UDFRegistrationService implements IService {
@Override
public void start() throws StartupException {
try {
+ registerBuiltinFunctions();
makeDirIfNecessary();
doRecovery();
logWriter = new UDFLogWriter(LOG_FILE_NAME);
@@ -236,6 +268,17 @@ public class UDFRegistrationService implements IService {
}
}
+ private void registerBuiltinFunctions() throws ClassNotFoundException {
+ ClassLoader classLoader = getClass().getClassLoader();
+ for (BuiltinFunction builtinFunction : BuiltinFunction.values()) {
+ String functionName = builtinFunction.getFunctionName();
+ String className = builtinFunction.getClassName();
+ Class<?> functionClass = Class.forName(className, true, classLoader);
+ registrationInformation.put(functionName,
+ new UDFRegistrationInformation(functionName, className, false, true,
functionClass));
+ }
+ }
+
private void makeDirIfNecessary() throws IOException {
File file = SystemFileFactory.INSTANCE.getFile(ULOG_FILE_DIR);
if (file.exists() && file.isDirectory()) {
@@ -305,7 +348,7 @@ public class UDFRegistrationService implements IService {
private void writeTemporaryLogFile() throws IOException {
UDFLogWriter temporaryLogFile = new UDFLogWriter(TEMPORARY_LOG_FILE_NAME);
for (UDFRegistrationInformation information :
registrationInformation.values()) {
- if (information.isTemporary()) {
+ if (information.isBuiltin() || information.isTemporary()) {
continue;
}
temporaryLogFile.register(information.getFunctionName(),
information.getClassName());
@@ -316,10 +359,26 @@ public class UDFRegistrationService implements IService {
@TestOnly
public void deregisterAll() throws UDFRegistrationException {
for (UDFRegistrationInformation information :
getRegistrationInformation()) {
- deregister(information.getFunctionName());
+ if (!information.isBuiltin()) {
+ deregister(information.getFunctionName());
+ }
}
}
+ @TestOnly
+ public void registerBuiltinFunction(String functionName, String className)
+ throws ClassNotFoundException {
+ ClassLoader classLoader = getClass().getClassLoader();
+ Class<?> functionClass = Class.forName(className, true, classLoader);
+ registrationInformation.put(functionName,
+ new UDFRegistrationInformation(functionName, className, false, true,
functionClass));
+ }
+
+ @TestOnly
+ public void deregisterBuiltinFunction(String functionName) {
+ registrationInformation.remove(functionName);
+ }
+
@Override
public ServiceType getID() {
return ServiceType.UDF_REGISTRATION_SERVICE;
diff --git
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDFManagementIT.java
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDFManagementIT.java
index 61cbd7d..14923bb 100644
---
a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDFManagementIT.java
+++
b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBUDFManagementIT.java
@@ -19,6 +19,9 @@
package org.apache.iotdb.db.integration;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDTF;
+import static
org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDTF;
+import static org.apache.iotdb.db.conf.IoTDBConstant.FUNCTION_TYPE_NATIVE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -29,6 +32,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import org.apache.iotdb.db.metadata.PartialPath;
+import org.apache.iotdb.db.qp.constant.SQLConstant;
+import org.apache.iotdb.db.query.udf.service.UDFRegistrationService;
import org.apache.iotdb.db.service.IoTDB;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
@@ -42,6 +47,8 @@ import org.junit.Test;
public class IoTDBUDFManagementIT {
+ private static final int NATIVE_FUNCTIONS_COUNT =
SQLConstant.getNativeFunctionNames().size();
+
@Before
public void setUp() throws Exception {
EnvironmentUtils.envSetUp();
@@ -67,26 +74,34 @@ public class IoTDBUDFManagementIT {
Statement statement = connection.createStatement()) {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
statement.execute("select udf(*, *) from root.vehicle");
+
ResultSet resultSet = statement.executeQuery("show functions");
- assertEquals(2, resultSet.getMetaData().getColumnCount());
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
int count = 0;
while (resultSet.next()) {
- ++count;
StringBuilder stringBuilder = new StringBuilder();
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); ++i) {
stringBuilder.append(resultSet.getString(i)).append(",");
}
- Assert.assertEquals("udf,org.apache.iotdb.db.query.udf.example.Adder,",
- stringBuilder.toString());
+ String result = stringBuilder.toString();
+ if (result.contains(FUNCTION_TYPE_NATIVE)) {
+ continue;
+ }
+
+
Assert.assertEquals(String.format("udf,%s,org.apache.iotdb.db.query.udf.example.Adder,",
+ FUNCTION_TYPE_EXTERNAL_UDTF), result);
+ ++count;
}
Assert.assertEquals(1, count);
+
resultSet = statement.executeQuery("show temporary functions");
count = 0;
while (resultSet.next()) {
++count;
}
Assert.assertEquals(0, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+
statement.execute("drop function udf");
} catch (SQLException throwable) {
throwable.printStackTrace();
@@ -101,57 +116,64 @@ public class IoTDBUDFManagementIT {
Statement statement = connection.createStatement()) {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
statement.execute("select udf(*, *) from root.vehicle");
+
ResultSet resultSet = statement.executeQuery("show functions");
int count = 0;
while (resultSet.next()) {
++count;
}
- Assert.assertEquals(1, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
+ Assert.assertEquals(1 + NATIVE_FUNCTIONS_COUNT, count);
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+
resultSet = statement.executeQuery("show temporary functions");
count = 0;
while (resultSet.next()) {
++count;
}
Assert.assertEquals(0, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
- statement.execute("drop function udf");
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+ statement.execute("drop function udf");
statement.execute(
"create temporary function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
statement.execute("select udf(*, *) from root.vehicle");
+
resultSet = statement.executeQuery("show functions");
count = 0;
while (resultSet.next()) {
++count;
}
- Assert.assertEquals(1, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
+ Assert.assertEquals(1 + NATIVE_FUNCTIONS_COUNT, count);
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+
resultSet = statement.executeQuery("show temporary functions");
count = 0;
while (resultSet.next()) {
++count;
}
Assert.assertEquals(1, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
- statement.execute("drop function udf");
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+ statement.execute("drop function udf");
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
statement.execute("select udf(*, *) from root.vehicle");
+
resultSet = statement.executeQuery("show functions");
count = 0;
while (resultSet.next()) {
++count;
}
- Assert.assertEquals(1, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
+ Assert.assertEquals(1 + NATIVE_FUNCTIONS_COUNT, count);
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+
resultSet = statement.executeQuery("show temporary functions");
count = 0;
while (resultSet.next()) {
++count;
}
Assert.assertEquals(0, count);
- assertEquals(2, resultSet.getMetaData().getColumnCount());
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+
statement.execute("drop function udf");
} catch (SQLException throwable) {
throwable.printStackTrace();
@@ -218,6 +240,7 @@ public class IoTDBUDFManagementIT {
.getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root",
"root");
Statement statement = connection.createStatement()) {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
+
try {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
fail();
@@ -234,6 +257,7 @@ public class IoTDBUDFManagementIT {
Statement statement = connection.createStatement()) {
statement.execute(
"create temporary function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
+
try {
statement.execute(
"create temporary function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
@@ -251,6 +275,7 @@ public class IoTDBUDFManagementIT {
Statement statement = connection.createStatement()) {
statement.execute(
"create temporary function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
+
try {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
fail();
@@ -267,6 +292,7 @@ public class IoTDBUDFManagementIT {
Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
Statement statement = connection.createStatement()) {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
+
try {
statement.execute(
"create temporary function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
@@ -285,6 +311,7 @@ public class IoTDBUDFManagementIT {
Statement statement = connection.createStatement()) {
statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
statement.execute("drop function udf");
+
try {
statement.execute("drop function udf");
fail();
@@ -305,4 +332,107 @@ public class IoTDBUDFManagementIT {
assertTrue(throwable.getMessage().contains("does not exist"));
}
}
+
+ @Test
+ public void testCreateBuiltinFunction() throws ClassNotFoundException {
+ UDFRegistrationService.getInstance().registerBuiltinFunction("adder",
+ "org.apache.iotdb.db.query.udf.example.Adder");
+ try (Connection connection = DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("create function adder as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
+ fail();
+ } catch (SQLException throwable) {
+ throwable.printStackTrace();
+ assertTrue(throwable.getMessage()
+ .contains("the given function name is the same as a built-in UDF
function name"));
+ } finally {
+ UDFRegistrationService.getInstance().deregisterBuiltinFunction("adder");
+ }
+ }
+
+ @Test
+ public void testDropBuiltinFunction() throws ClassNotFoundException {
+ UDFRegistrationService.getInstance().registerBuiltinFunction("adder",
+ "org.apache.iotdb.db.query.udf.example.Adder");
+ try (Connection connection = DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("drop function adder");
+ fail();
+ } catch (SQLException throwable) {
+ throwable.printStackTrace();
+ assertTrue(
+ throwable.getMessage().contains("Built-in function adder can not be
deregistered"));
+ } finally {
+ UDFRegistrationService.getInstance().deregisterBuiltinFunction("adder");
+ }
+ }
+
+ @Test
+ public void testReflectBuiltinFunction() throws ClassNotFoundException {
+ UDFRegistrationService.getInstance().registerBuiltinFunction("adder",
+ "org.apache.iotdb.db.query.udf.example.Adder");
+ try (Connection connection = DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("select adder(*, *) from root.vehicle");
+ } catch (SQLException throwable) {
+ throwable.printStackTrace();
+ fail(throwable.getMessage());
+ } finally {
+ UDFRegistrationService.getInstance().deregisterBuiltinFunction("adder");
+ }
+ }
+
+ @Test
+ public void testShowBuiltinFunction() throws ClassNotFoundException {
+ UDFRegistrationService.getInstance().registerBuiltinFunction("adder",
+ "org.apache.iotdb.db.query.udf.example.Adder");
+ try (Connection connection = DriverManager.getConnection(
+ Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/", "root", "root");
+ Statement statement = connection.createStatement()) {
+ statement.execute("create function udf as
\"org.apache.iotdb.db.query.udf.example.Adder\"");
+
+ ResultSet resultSet = statement.executeQuery("show functions");
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+ int count = 0;
+ while (resultSet.next()) {
+ StringBuilder stringBuilder = new StringBuilder();
+ for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); ++i) {
+ stringBuilder.append(resultSet.getString(i)).append(",");
+ }
+ String result = stringBuilder.toString();
+ if (result.contains(FUNCTION_TYPE_NATIVE)) {
+ continue;
+ }
+
+ if (result.contains(FUNCTION_TYPE_EXTERNAL_UDTF)) {
+
Assert.assertEquals(String.format("udf,%s,org.apache.iotdb.db.query.udf.example.Adder,",
+ FUNCTION_TYPE_EXTERNAL_UDTF), result);
+ ++count;
+ } else if (result.contains(FUNCTION_TYPE_BUILTIN_UDTF)) {
+
Assert.assertEquals(String.format("adder,%s,org.apache.iotdb.db.query.udf.example.Adder,",
+ FUNCTION_TYPE_BUILTIN_UDTF), result);
+ ++count;
+ }
+ }
+ Assert.assertEquals(2, count);
+
+ resultSet = statement.executeQuery("show temporary functions");
+ count = 0;
+ while (resultSet.next()) {
+ ++count;
+ }
+ Assert.assertEquals(0, count);
+ assertEquals(3, resultSet.getMetaData().getColumnCount());
+
+ statement.execute("drop function udf");
+ } catch (SQLException throwable) {
+ throwable.printStackTrace();
+ fail(throwable.getMessage());
+ } finally {
+ UDFRegistrationService.getInstance().deregisterBuiltinFunction("adder");
+ }
+ }
}
