This is an automated email from the ASF dual-hosted git repository. rong pushed a commit to branch iotdb-3229 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 6eec8697f1a95647dbbce91d4401cfe6520463f8 Author: Steve Yurong Su <[email protected]> AuthorDate: Tue May 31 18:03:54 2022 +0800 [IOTDB-3229] Apache IoTDB UDF Management in MPP Cluster: SHOW FUNCTIONS --- .../iotdb/db/mpp/common/header/HeaderConstant.java | 18 +++ .../plan/execution/config/ConfigTaskVisitor.java | 7 + .../plan/execution/config/ShowFunctionsTask.java | 153 +++++++++++++++++++++ .../iotdb/db/mpp/plan/parser/ASTVisitor.java | 8 ++ .../db/mpp/plan/statement/StatementVisitor.java | 5 + .../statement/metadata/ShowFunctionsStatement.java | 46 +++++++ 6 files changed, 237 insertions(+) diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java index 0f2a26469e..e48e71754c 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/common/header/HeaderConstant.java @@ -64,6 +64,11 @@ public class HeaderConstant { public static final String COLUMN_HOST_ADDRESS = "HostAdress"; public static final String COLUMN_PORT = "Port"; + // column names for show functions statement + public static final String COLUMN_FUNCTION_NAME = "function name"; + public static final String COLUMN_FUNCTION_TYPE = "function type"; + public static final String COLUMN_FUNCTION_CLASS = "class name (UDF)"; + // dataset header for schema statement public static final DatasetHeader showTimeSeriesHeader; public static final DatasetHeader showDevicesHeader; @@ -86,6 +91,9 @@ public class HeaderConstant { // dataset header for last query public static final DatasetHeader LAST_QUERY_HEADER; + // dataset header for show functions + public static final DatasetHeader SHOW_FUNCTIONS_HEADER; + static { countStorageGroupHeader = new DatasetHeader( @@ -182,4 +190,14 @@ public class HeaderConstant { new ColumnHeader(COLUMN_PORT, TSDataType.INT32)), true); } + + static { + SHOW_FUNCTIONS_HEADER = + new DatasetHeader( + Arrays.asList( + new ColumnHeader(COLUMN_FUNCTION_NAME, TSDataType.TEXT), + new ColumnHeader(COLUMN_FUNCTION_TYPE, TSDataType.TEXT), + new ColumnHeader(COLUMN_FUNCTION_CLASS, TSDataType.TEXT)), + true); + } } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java index 9ef88cd17b..27311d6a7f 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ConfigTaskVisitor.java @@ -29,6 +29,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.DropFunctionStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.SetStorageGroupStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.SetTTLStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowFunctionsStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStorageGroupStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTTLStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.UnSetTTLStatement; @@ -110,5 +111,11 @@ public class ConfigTaskVisitor return new DropFunctionTask(dropFunctionStatement); } + @Override + public IConfigTask visitShowFunctions( + ShowFunctionsStatement showFunctionsStatement, TaskContext context) { + return new ShowFunctionsTask(); + } + public static class TaskContext {} } diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ShowFunctionsTask.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ShowFunctionsTask.java new file mode 100644 index 0000000000..c51b1d59c3 --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/execution/config/ShowFunctionsTask.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.execution.config; + +import org.apache.iotdb.commons.client.IClientManager; +import org.apache.iotdb.commons.consensus.PartitionRegionId; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.udf.builtin.BuiltinAggregationFunction; +import org.apache.iotdb.commons.udf.service.UDFRegistrationInformation; +import org.apache.iotdb.commons.udf.service.UDFRegistrationService; +import org.apache.iotdb.db.client.ConfigNodeClient; +import org.apache.iotdb.db.mpp.common.header.HeaderConstant; +import org.apache.iotdb.db.query.dataset.ListDataSet; +import org.apache.iotdb.rpc.TSStatusCode; +import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType; +import org.apache.iotdb.tsfile.read.common.RowRecord; +import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder; +import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet; +import org.apache.iotdb.tsfile.utils.Binary; + +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.util.Arrays; + +import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_FUNCTION_CLASS; +import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_FUNCTION_NAME; +import static org.apache.iotdb.commons.conf.IoTDBConstant.COLUMN_FUNCTION_TYPE; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDAF; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_BUILTIN_UDTF; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDAF; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_EXTERNAL_UDTF; +import static org.apache.iotdb.commons.conf.IoTDBConstant.FUNCTION_TYPE_NATIVE; + +public class ShowFunctionsTask implements IConfigTask { + + private static final Logger LOGGER = LoggerFactory.getLogger(ShowFunctionsTask.class); + + @Override + public ListenableFuture<ConfigTaskResult> execute( + IClientManager<PartitionRegionId, ConfigNodeClient> clientManager) + throws InterruptedException { + final SettableFuture<ConfigTaskResult> future = SettableFuture.create(); + + try { + final TsBlockBuilder builder = + new TsBlockBuilder(HeaderConstant.SHOW_FUNCTIONS_HEADER.getRespDataTypes()); + + final QueryDataSet showDataSet = generateShowFunctionsListDataSet(); + while (showDataSet.hasNextWithoutConstraint()) { + final RowRecord record = showDataSet.nextWithoutConstraint(); + + builder.getTimeColumnBuilder().writeLong(0); + for (int i = 0; i < 3; ++i) { + builder.getColumnBuilder(i).writeBinary(record.getFields().get(i).getBinaryV()); + } + builder.declarePosition(); + } + + future.set( + new ConfigTaskResult( + TSStatusCode.SUCCESS_STATUS, builder.build(), HeaderConstant.SHOW_FUNCTIONS_HEADER)); + } catch (Exception e) { + LOGGER.error("Failed to get functions.", e); + future.setException(e); + } + + return future; + } + + private QueryDataSet generateShowFunctionsListDataSet() { + ListDataSet listDataSet = + new ListDataSet( + Arrays.asList( + new PartialPath(COLUMN_FUNCTION_NAME, false), + new PartialPath(COLUMN_FUNCTION_TYPE, false), + new PartialPath(COLUMN_FUNCTION_CLASS, false)), + Arrays.asList(TSDataType.TEXT, TSDataType.TEXT, TSDataType.TEXT)); + + appendUDFs(listDataSet); + appendNativeFunctions(listDataSet); + + listDataSet.sort( + (r1, r2) -> + String.CASE_INSENSITIVE_ORDER.compare( + r1.getFields().get(0).getStringValue(), r2.getFields().get(0).getStringValue())); + return listDataSet; + } + + private void appendUDFs(ListDataSet listDataSet) { + for (UDFRegistrationInformation info : + UDFRegistrationService.getInstance().getRegistrationInformation()) { + RowRecord rowRecord = new RowRecord(0); // ignore timestamp + rowRecord.addField(Binary.valueOf(info.getFunctionName()), TSDataType.TEXT); + String functionType = ""; + try { + if (info.isBuiltin()) { + if (info.isUDTF()) { + functionType = FUNCTION_TYPE_BUILTIN_UDTF; + } else if (info.isUDAF()) { + functionType = FUNCTION_TYPE_BUILTIN_UDAF; + } + } else { + if (info.isUDTF()) { + functionType = FUNCTION_TYPE_EXTERNAL_UDTF; + } else if (info.isUDAF()) { + functionType = FUNCTION_TYPE_EXTERNAL_UDAF; + } + } + } catch (InstantiationException + | InvocationTargetException + | NoSuchMethodException + | IllegalAccessException e) { + throw new RuntimeException(e.toString()); + } + rowRecord.addField(Binary.valueOf(functionType), TSDataType.TEXT); + rowRecord.addField(Binary.valueOf(info.getClassName()), TSDataType.TEXT); + listDataSet.putRecord(rowRecord); + } + } + + private void appendNativeFunctions(ListDataSet listDataSet) { + final Binary functionType = Binary.valueOf(FUNCTION_TYPE_NATIVE); + final Binary className = Binary.valueOf(""); + for (String functionName : BuiltinAggregationFunction.getNativeFunctionNames()) { + RowRecord rowRecord = new RowRecord(0); // ignore timestamp + rowRecord.addField(Binary.valueOf(functionName.toUpperCase()), TSDataType.TEXT); + rowRecord.addField(functionType, TSDataType.TEXT); + rowRecord.addField(className, TSDataType.TEXT); + listDataSet.putRecord(rowRecord); + } + } +} diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java index ced604bfbf..a9ec1eda89 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/parser/ASTVisitor.java @@ -92,6 +92,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildNodesStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildPathsStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowDevicesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowFunctionsStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStorageGroupStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTTLStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTimeSeriesStatement; @@ -108,6 +109,7 @@ import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CountTimeseriesContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.CreateFunctionContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.DropFunctionContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ExpressionContext; +import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.ShowFunctionsContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParser.UriContext; import org.apache.iotdb.db.qp.sql.IoTDBSqlParserBaseVisitor; import org.apache.iotdb.db.qp.utils.DatetimeUtils; @@ -637,6 +639,12 @@ public class ASTVisitor extends IoTDBSqlParserBaseVisitor<Statement> { return new DropFunctionStatement(parseIdentifier(ctx.udfName.getText())); } + // Show Functions + @Override + public Statement visitShowFunctions(ShowFunctionsContext ctx) { + return new ShowFunctionsStatement(); + } + // Show Child Paths ===================================================================== @Override public Statement visitShowChildPaths(IoTDBSqlParser.ShowChildPathsContext ctx) { diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java index adadae7760..b20124cbea 100644 --- a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/StatementVisitor.java @@ -49,6 +49,7 @@ import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildNodesStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowChildPathsStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowClusterStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowDevicesStatement; +import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowFunctionsStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowStorageGroupStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTTLStatement; import org.apache.iotdb.db.mpp.plan.statement.metadata.ShowTimeSeriesStatement; @@ -145,6 +146,10 @@ public abstract class StatementVisitor<R, C> { return visitStatement(dropFunctionStatement, context); } + public R visitShowFunctions(ShowFunctionsStatement showFunctionsStatement, C context) { + return visitStatement(showFunctionsStatement, context); + } + /** Data Manipulation Language (DML) */ // Select Statement diff --git a/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/ShowFunctionsStatement.java b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/ShowFunctionsStatement.java new file mode 100644 index 0000000000..269fd07dbe --- /dev/null +++ b/server/src/main/java/org/apache/iotdb/db/mpp/plan/statement/metadata/ShowFunctionsStatement.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.mpp.plan.statement.metadata; + +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.mpp.plan.analyze.QueryType; +import org.apache.iotdb.db.mpp.plan.statement.IConfigStatement; +import org.apache.iotdb.db.mpp.plan.statement.StatementVisitor; + +import java.util.Collections; +import java.util.List; + +public class ShowFunctionsStatement extends ShowStatement implements IConfigStatement { + + @Override + public <R, C> R accept(StatementVisitor<R, C> visitor, C context) { + return visitor.visitShowFunctions(this, context); + } + + @Override + public QueryType getQueryType() { + return QueryType.READ; + } + + @Override + public List<PartialPath> getPaths() { + return Collections.emptyList(); + } +}
