This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch force_ci/alter_column_datatype in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e0c37ac9567417bb0de6db31df4e6c47272c2c20 Merge: 7fe66e15fa1 82c85483572 Author: Tian Jiang <[email protected]> AuthorDate: Mon Jan 13 17:00:45 2025 +0800 Merge branch 'master' into force_ci/alter_column_datatype # Conflicts: # iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java # pom.xml .github/workflows/pipe-it-2cluster.yml | 342 +++++++++++-- example/udf/pom.xml | 6 +- .../apache/iotdb/udf/AggregateFunctionExample.java | 35 +- .../apache/iotdb/udf/ScalarFunctionExample.java | 57 +-- .../db/query/udf/example/relational/AllSum.java | 49 +- .../query/udf/example/relational/ContainNull.java | 21 +- .../relational/{DatePlusOne.java => DatePlus.java} | 31 +- .../query/udf/example/relational/FirstTwoSum.java | 33 +- .../db/query/udf/example/relational/MyAvg.java | 32 +- .../db/query/udf/example/relational/MyCount.java | 19 +- .../iotdb/it/env/cluster/env/AbstractEnv.java | 5 + .../iotdb/it/env/remote/env/RemoteServerEnv.java | 5 + .../iotdb/it/utils/TsFileTableGenerator.java | 202 ++++++++ .../java/org/apache/iotdb/itbase/env/BaseEnv.java | 3 + .../org/apache/iotdb/db/it/IoTDBLoadTsFileIT.java | 80 ++- .../org/apache/iotdb/db/it/utils/TestUtils.java | 23 + .../iotdb/pipe/it/autocreate/IoTDBPipeAlterIT.java | 3 +- .../it/autocreate/IoTDBPipeSwitchStatusIT.java | 23 +- .../pipe/it/manual/IoTDBPipeTypeConversionIT.java | 2 +- .../iotdb/pipe/it/tablemodel/IoTDBPipeAlterIT.java | 30 +- .../pipe/it/tablemodel/IoTDBPipeClusterIT.java | 290 ++++++----- .../pipe/it/tablemodel/IoTDBPipeDataSinkIT.java | 429 +++++++++++++++- .../it/tablemodel/IoTDBPipeDoubleLivingIT.java | 329 ++++++++++++ .../pipe/it/tablemodel/IoTDBPipeIsolationIT.java | 377 ++++++++++++++ .../pipe/it/tablemodel/IoTDBPipeNullValueIT.java | 4 +- .../it/tablemodel/IoTDBPipeSwitchStatusIT.java | 23 +- .../pipe/it/tablemodel/IoTDBPipeSyntaxIT.java | 33 +- .../IoTDBPipeTypeConversionISessionIT.java | 93 ++-- .../pipe/it/tablemodel/IoTDBPipeWithLoadIT.java | 8 +- .../iotdb/pipe/it/tablemodel/TableModelUtils.java | 563 +++++++++++++++++---- .../relational/it/db/it/IoTDBDeletionTableIT.java | 2 +- .../relational/it/db/it/IoTDBInsertTableIT.java | 2 +- .../it/db/it/udf/IoTDBSQLFunctionManagementIT.java | 19 +- .../it/udf/IoTDBUserDefinedScalarFunctionIT.java | 2 +- .../IoTDBAlignByDeviceWithTemplateTableIT.java | 4 +- .../scalar/IoTDBFormatFunctionTableIT.java | 21 +- ...oTDBNoSelectExpressionAfterAnalyzedTableIT.java | 6 +- .../query/old/query/IoTDBNullValueFillTableIT.java | 327 ------------ .../it/query/old/query/IoTDBResultSetTableIT.java | 2 +- .../query/old/query/IoTDBSelectSchemaTableIT.java | 22 +- .../it/query/recent/IoTDBTableAggregationIT.java | 343 ++++++++++++- .../recent/subquery/SubqueryDataSetUtils.java | 12 +- .../IoTDBUncorrelatedInPredicateSubqueryIT.java | 319 ++++++++++++ .../IoTDBUncorrelatedScalarSubqueryIT.java | 4 +- .../it/schema/IoTDBAlterColumnTypeIT.java | 55 ++ .../relational/it/schema/IoTDBDatabaseIT.java | 41 ++ .../iotdb/relational/it/schema/IoTDBDeviceIT.java | 47 +- .../iotdb/relational/it/schema/IoTDBTableIT.java | 122 +++-- .../it/session/IoTDBSessionRelationalIT.java | 183 ++++--- .../iotdb/session/it/IoTDBSessionSimpleIT.java | 36 +- .../analysis/AggregateFunctionAnalysis.java | 62 +++ .../api/customizer/analysis/FunctionAnalysis.java | 15 +- .../analysis/ScalarFunctionAnalysis.java | 51 ++ ...ctionParameters.java => FunctionArguments.java} | 28 +- .../exception/UDFArgumentNotValidException.java | 10 +- .../udf/api/relational/AggregateFunction.java | 48 +- .../iotdb/udf/api/relational/ScalarFunction.java | 47 +- .../cli/src/assembly/resources/sbin/start-cli.bat | 2 +- .../apache/iotdb/tool/data/AbstractDataTool.java | 32 +- .../org/apache/iotdb/tool/data/ExportData.java | 43 +- .../org/apache/iotdb/tool/data/ImportData.java | 24 +- .../client-py/table_model_session_example.py | 12 +- .../client-py/table_model_session_pool_example.py | 8 +- .../payload/SubscriptionFileHandler.java | 24 +- .../org/apache/iotdb/session/util/RetryUtils.java | 30 +- .../consensus/request/ConfigPhysicalPlanType.java | 6 +- .../table/DescTable4InformationSchemaPlan.java} | 12 +- .../table/ShowTable4InformationSchemaPlan.java} | 11 +- .../response/pipe/task/PipeTableResp.java | 7 + .../table/DescTable4InformationSchemaResp.java | 42 ++ .../table/ShowTable4InformationSchemaResp.java | 44 ++ .../iotdb/confignode/manager/ConfigManager.java | 32 +- .../apache/iotdb/confignode/manager/IManager.java | 31 +- .../iotdb/confignode/manager/load/LoadManager.java | 11 + .../confignode/manager/load/cache/LoadCache.java | 17 + .../iotdb/confignode/manager/node/NodeManager.java | 13 + .../pipe/coordinator/task/PipeTaskCoordinator.java | 74 ++- .../manager/schema/ClusterSchemaManager.java | 129 +++-- .../persistence/executor/ConfigPlanExecutor.java | 4 + .../confignode/persistence/pipe/PipeTaskInfo.java | 21 +- .../persistence/schema/ClusterSchemaInfo.java | 88 ++++ .../confignode/persistence/schema/ConfigMTree.java | 22 +- .../impl/pipe/task/AlterPipeProcedureV2.java | 4 +- .../thrift/ConfigNodeRPCServiceProcessor.java | 32 +- .../apache/iotdb/consensus/pipe/PipeConsensus.java | 57 ++- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +- .../schemaregion/SchemaExecutionVisitor.java | 15 +- .../LoadAnalyzeException.java} | 8 +- .../LoadAnalyzeTableColumnDisorderException.java} | 7 +- .../LoadAnalyzeTypeMismatchException.java} | 6 +- .../batch/PipeTabletEventTsFileBatch.java | 384 +++----------- .../request/PipeTransferTabletRawReq.java | 4 +- .../request/PipeTransferTabletRawReqV2.java | 9 +- .../connector/protocol/opcua/OpcUaNameSpace.java | 8 +- .../request/PipeConsensusDeleteNodeReq.java | 4 +- .../async/IoTDBDataRegionAsyncConnector.java | 10 +- .../PipeTransferTabletBatchEventHandler.java | 2 +- .../async/handler/PipeTransferTsFileHandler.java | 13 +- .../thrift/sync/IoTDBDataRegionSyncConnector.java | 19 +- .../util/builder/PipeTableModeTsFileBuilder.java | 273 ++++++++++ .../util/builder/PipeTreeModelTsFileBuilder.java | 268 ++++++++++ .../connector/util/builder/PipeTsFileBuilder.java | 162 ++++++ .../util/{ => cacher}/LeaderCacheUtils.java | 2 +- .../sorter/PipeTableModelTabletEventSorter.java | 271 ++++++++++ .../util/{ => sorter}/PipeTabletEventSorter.java | 107 +--- .../sorter/PipeTreeModelTabletEventSorter.java | 121 +++++ .../dataregion/IoTDBDataRegionExtractor.java | 20 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 20 +- .../pipeconsensus/PipeConsensusReceiver.java | 31 +- .../protocol/thrift/IoTDBDataNodeReceiver.java | 7 +- .../statement/PipeConvertedInsertRowStatement.java | 30 +- .../PipeConvertedInsertTabletStatement.java | 52 +- ...leStatementDataTypeConvertExecutionVisitor.java | 10 +- ...eeStatementDataTypeConvertExecutionVisitor.java | 3 +- .../resource/tsfile/PipeTsFileResourceManager.java | 5 +- .../iotdb/db/protocol/client/ConfigNodeClient.java | 32 +- .../operator/process/EnforceSingleRowOperator.java | 5 +- .../merge/comparator/JoinKeyComparatorFactory.java | 2 +- .../schema/source/TableDeviceQuerySource.java | 4 +- .../relational/AbstractAggTableScanOperator.java | 241 ++++----- .../AbstractDefaultAggTableScanOperator.java | 109 ++++ ...perator.java => AbstractTableScanOperator.java} | 94 ++-- .../relational/DefaultAggTableScanOperator.java | 58 +-- .../InformationSchemaContentSupplierFactory.java | 372 ++++++++++++-- .../relational/LastQueryAggTableScanOperator.java | 66 +-- .../relational/MergeSortSemiJoinOperator.java | 228 +++++++++ .../source/relational/TableScanOperator.java | 332 +----------- ...eAlignedDeviceViewAggregationScanOperator.java} | 24 +- .../TreeAlignedDeviceViewScanOperator.java} | 23 +- .../relational/aggregation/AccumulatorFactory.java | 23 +- .../relational/aggregation/CountIfAccumulator.java | 101 ++++ .../UserDefinedAggregateFunctionAccumulator.java | 27 +- .../grouped/GroupedCountIfAccumulator.java | 79 +++ .../relational/ColumnTransformerBuilder.java | 14 +- .../iotdb/db/queryengine/plan/Coordinator.java | 4 + .../db/queryengine/plan/analyze/Analysis.java | 10 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 57 ++- .../plan/analyze/load/LoadTsFileAnalyzer.java | 44 +- .../analyze/load/LoadTsFileTableSchemaCache.java | 14 +- .../load/LoadTsFileToTableModelAnalyzer.java | 27 +- .../load/LoadTsFileToTreeModelAnalyzer.java | 17 +- .../load/TreeSchemaAutoCreatorAndVerifier.java | 30 +- .../config/executor/ClusterConfigTaskExecutor.java | 20 +- .../config/metadata/relational/ShowDBTask.java | 4 +- .../execution/config/sys/pipe/AlterPipeTask.java | 8 +- .../execution/config/sys/pipe/DropPipeTask.java | 7 +- .../execution/config/sys/pipe/ShowPipeTask.java | 3 +- .../execution/config/sys/pipe/StartPipeTask.java | 7 +- .../execution/config/sys/pipe/StopPipeTask.java | 7 +- .../db/queryengine/plan/parser/ASTVisitor.java | 9 +- .../plan/planner/LogicalPlanBuilder.java | 2 +- .../plan/planner/TableOperatorGenerator.java | 544 ++++++++++++++------ .../plan/planner/plan/node/PlanGraphPrinter.java | 36 +- .../plan/planner/plan/node/PlanNode.java | 2 + .../plan/planner/plan/node/PlanNodeType.java | 16 + .../plan/planner/plan/node/PlanVisitor.java | 25 + .../plan/planner/plan/node/source/SourceNode.java | 1 + .../relational/analyzer/ExpressionAnalyzer.java | 37 +- .../plan/relational/analyzer/Scope.java | 8 +- .../relational/analyzer/StatementAnalyzer.java | 25 +- .../AlignedDeviceEntry.java} | 47 +- .../plan/relational/metadata/DeviceEntry.java | 42 +- .../plan/relational/metadata/Metadata.java | 4 +- .../NonAlignedAlignedDeviceEntry.java} | 47 +- .../relational/metadata/TableMetadataImpl.java | 56 +- .../relational/metadata/TreeDeviceViewSchema.java} | 21 +- .../metadata/fetcher/TableDeviceSchemaFetcher.java | 19 +- .../fetcher/TableHeaderSchemaValidator.java | 11 +- .../plan/relational/planner/IrTypeAnalyzer.java | 11 + .../plan/relational/planner/QueryPlanner.java | 2 +- .../plan/relational/planner/RelationPlanner.java | 20 + .../plan/relational/planner/SubqueryPlanner.java | 84 +-- .../distribute/TableDistributedPlanGenerator.java | 253 +++++++-- .../planner/iterative/rule/PruneApplyColumns.java | 138 +++++ .../iterative/rule/PruneApplyCorrelation.java | 70 +++ .../iterative/rule/PruneApplySourceColumns.java | 95 ++++ .../iterative/rule/PruneTableScanColumns.java | 52 +- .../rule/RemoveUnreferencedScalarApplyNodes.java | 42 ++ .../rule/RemoveUnreferencedScalarSubqueries.java | 70 +++ .../TransformFilteringSemiJoinToInnerJoin.java | 150 ++++++ ...mUncorrelatedInPredicateSubqueryToSemiJoin.java | 95 ++++ .../planner/node/AggregationTableScanNode.java | 387 +++++++------- .../node/AggregationTreeDeviceViewScanNode.java | 205 ++++++++ .../planner/node/DeviceTableScanNode.java | 197 +++---- .../node/InformationSchemaTableScanNode.java | 105 +--- .../plan/relational/planner/node/Patterns.java | 7 +- .../plan/relational/planner/node/SemiJoinNode.java | 181 +++++++ .../relational/planner/node/TableScanNode.java | 118 ++++- .../node/TreeAlignedDeviceViewScanNode.java | 126 +++++ .../planner/node/TreeDeviceViewScanNode.java | 208 ++++++++ .../node/TreeNonAlignedDeviceViewScanNode.java | 126 +++++ .../DataNodeLocationSupplierFactory.java | 51 +- .../optimizations/LogicalOptimizeFactory.java | 18 +- .../PushAggregationIntoTableScan.java | 4 + .../optimizations/PushPredicateIntoTableScan.java | 218 +++++++- .../optimizations/UnaliasSymbolReferences.java | 69 ++- .../relational/planner/optimizations/Util.java | 73 ++- .../sql/ast/AbstractQueryDeviceWithCache.java | 2 + .../plan/relational/sql/ast/AlterPipe.java | 22 +- .../plan/relational/sql/ast/AstVisitor.java | 4 + .../ast/{DropColumn.java => CountStatement.java} | 61 +-- .../plan/relational/sql/ast/DeleteDevice.java | 2 - .../plan/relational/sql/ast/DropColumn.java | 3 - .../plan/relational/sql/ast/DropPipe.java | 10 +- .../plan/relational/sql/ast/LoadTsFile.java | 14 +- .../plan/relational/sql/ast/ShowPipes.java | 10 +- .../plan/relational/sql/ast/StartPipe.java | 8 +- .../plan/relational/sql/ast/StopPipe.java | 8 +- .../plan/relational/sql/parser/AstBuilder.java | 41 +- .../plan/relational/sql/rewrite/ShowRewrite.java | 51 +- .../plan/scheduler/load/LoadTsFileScheduler.java | 80 ++- .../plan/statement/crud/InsertBaseStatement.java | 6 +- .../plan/statement/crud/InsertRowStatement.java | 18 +- .../plan/statement/crud/LoadTsFileStatement.java | 9 + .../metadata/pipe/AlterPipeStatement.java | 31 +- .../statement/metadata/pipe/DropPipeStatement.java | 23 +- .../metadata/pipe/ShowPipesStatement.java | 18 +- .../metadata/pipe/StartPipeStatement.java | 17 +- .../statement/metadata/pipe/StopPipeStatement.java | 17 +- .../schemaengine/table/InformationSchemaUtils.java | 11 +- .../db/service/metrics/CompactionMetrics.java | 24 + .../db/service/metrics/DataNodeMetricsHelper.java | 4 +- .../metrics/memory/ConsensusMemoryMetrics.java | 73 +++ .../metrics/memory/GlobalMemoryMetrics.java | 132 +++++ .../metrics/memory/QueryEngineMemoryMetrics.java | 179 +++++++ .../metrics/memory/SchemaEngineMemoryMetrics.java | 124 +++++ .../metrics/memory/StorageEngineMemoryMetrics.java | 210 ++++++++ .../metrics/memory/StreamEngineMemoryMetrics.java | 73 +++ .../metrics/memory/ThresholdMemoryMetrics.java | 194 ------- .../db/storageengine/dataregion/DataRegion.java | 135 ++--- .../CompactionSourceFileDeletedException.java} | 13 +- .../execute/task/InnerSpaceCompactionTask.java | 6 +- .../schedule/CompactionScheduleContext.java | 42 +- .../compaction/schedule/CompactionScheduler.java | 60 ++- .../estimator/AbstractCrossSpaceEstimator.java | 7 +- .../estimator/AbstractInnerSpaceEstimator.java | 8 +- .../estimator/CompactionEstimateUtils.java | 35 +- .../FastCompactionInnerCompactionEstimator.java | 16 +- .../FastCrossSpaceCompactionEstimator.java | 17 +- .../selector/estimator/MetadataInfo.java | 42 ++ .../ReadChunkInnerCompactionEstimator.java | 15 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 23 +- .../utils/CrossSpaceCompactionCandidate.java | 4 +- .../selector/utils/TsFileResourceCandidate.java | 76 +-- .../dataregion/tsfile/TsFileResource.java | 34 +- .../db/storageengine/load/LoadTsFileManager.java | 21 +- .../load/active/ActiveLoadDirScanner.java | 27 +- .../load/active/ActiveLoadTsFileLoader.java | 13 +- .../LoadConvertedInsertTabletStatement.java | 18 +- ...leStatementDataTypeConvertExecutionVisitor.java | 10 +- ...eeStatementDataTypeConvertExecutionVisitor.java | 3 +- .../converter/LoadTsFileDataTypeConverter.java | 103 ++-- .../load/metrics/LoadTsFileCostMetricsSet.java | 13 +- .../rescon/memory/TsFileResourceManager.java | 2 +- .../broker/SubscriptionPrefetchingQueue.java | 2 + .../db/subscription/event/SubscriptionEvent.java | 21 +- .../batch/SubscriptionPipeTabletEventBatch.java | 22 + .../batch/SubscriptionPipeTsFileEventBatch.java | 11 +- .../org/apache/iotdb/db/utils/CommonUtils.java | 6 - .../iotdb/db/utils/datastructure/BinaryTVList.java | 6 +- .../db/utils/datastructure/BooleanTVList.java | 6 +- .../iotdb/db/utils/datastructure/DoubleTVList.java | 6 +- .../iotdb/db/utils/datastructure/FloatTVList.java | 6 +- .../iotdb/db/utils/datastructure/IntTVList.java | 6 +- .../iotdb/db/utils/datastructure/LongTVList.java | 6 +- .../pipe/connector/PipeTabletEventSorterTest.java | 210 +++++++- .../analyzer/MockTableModelDataPartition.java | 10 +- .../plan/relational/analyzer/TSBSMetadata.java | 23 +- .../plan/relational/analyzer/TestMatadata.java | 138 +++-- .../plan/relational/analyzer/TestPlanBuilder.java | 6 +- .../plan/relational/analyzer/TreeViewTest.java | 198 ++++++++ .../plan/relational/planner/SubqueryTest.java | 98 ++++ .../relational/planner/assertions/PlanAssert.java | 2 +- .../planner/assertions/PlanMatchPattern.java | 83 +++ .../planner/assertions/SemiJoinMatcher.java | 79 +++ .../storageengine/dataregion/DataRegionTest.java | 24 +- .../cross/CrossSpaceCompactionSelectorTest.java | 102 +--- .../InsertionCrossSpaceCompactionSelectorTest.java | 4 +- .../cross/InsertionCrossSpaceCompactionTest.java | 64 ++- .../utils/CompactionTaskMemCostEstimatorTest.java | 48 ++ .../conf/iotdb-system.properties.template | 6 +- .../apache/iotdb/commons/cluster/NodeStatus.java | 13 + .../exception}/table/ColumnNotExistsException.java | 2 +- .../table/TableAlreadyExistsException.java | 2 +- .../exception}/table/TableNotExistsException.java | 2 +- .../commons/pipe/agent/task/meta/PipeMeta.java | 40 ++ .../pipe/agent/task/meta/PipeMetaKeeper.java | 8 + .../config/constant/PipeExtractorConstant.java | 3 + .../pipe/datastructure/pattern/TablePattern.java | 21 +- .../pipe/datastructure/pattern/TreePattern.java | 22 +- .../commons/pipe/extractor/IoTDBExtractor.java | 65 ++- .../commons/pipe/receiver/IoTDBFileReceiver.java | 17 +- .../commons/pipe/receiver/IoTDBReceiverAgent.java | 7 +- .../snapshot/PipeSnapshotResourceManager.java | 9 +- .../iotdb/commons/service/metric/enums/Metric.java | 4 +- .../TableBuiltinAggregationFunction.java | 2 + .../org/apache/iotdb/commons/utils/FileUtils.java | 49 ++ .../apache/iotdb/commons/utils/RetryUtils.java} | 30 +- .../src/main/thrift/confignode.thrift | 43 +- 299 files changed, 13455 insertions(+), 4206 deletions(-) diff --cc integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java index 029fc6653f4,00000000000..b69069039f6 mode 100644,000000..100644 --- a/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/relational/it/schema/IoTDBAlterColumnTypeIT.java @@@ -1,322 -1,0 +1,377 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.relational.it.schema; + +import org.apache.iotdb.commons.utils.MetadataUtils; +import org.apache.iotdb.isession.ITableSession; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.TableClusterIT; +import org.apache.iotdb.itbase.category.TableLocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.RowRecord; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.record.Tablet.ColumnCategory; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +import static org.apache.iotdb.relational.it.session.IoTDBSessionRelationalIT.genValue; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.fail; + +@RunWith(IoTDBTestRunner.class) +@Category({TableLocalStandaloneIT.class, TableClusterIT.class}) +public class IoTDBAlterColumnTypeIT { + + @BeforeClass + public static void setUp() throws Exception { + EnvFactory.getEnv().initClusterEnvironment(); + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) { + session.executeNonQueryStatement("CREATE DATABASE IF NOT EXISTS test"); + } + } + + @AfterClass + public static void tearDown() throws Exception { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void testWriteAndAlter() throws IoTDBConnectionException, StatementExecutionException { + Set<TSDataType> typesToTest = new HashSet<>(); + Collections.addAll(typesToTest, TSDataType.values()); + typesToTest.remove(TSDataType.VECTOR); + typesToTest.remove(TSDataType.UNKNOWN); + + // doWriteAndAlter(TSDataType.INT32, TSDataType.FLOAT, false); + // doWriteAndAlter(TSDataType.INT32, TSDataType.FLOAT, true); + + for (TSDataType from : typesToTest) { + for (TSDataType to : typesToTest) { + System.out.printf("testing %s to %s%n", from, to); + doWriteAndAlter(from, to, false); + doWriteAndAlter(from, to, true); + } + } + } + + private void doWriteAndAlter(TSDataType from, TSDataType to, boolean flush) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS write_and_alter_column_type (s1 " + from + ")"); + + // write a point of "from" + Tablet tablet = + new Tablet( + "write_and_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(from), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(from, 1)); + session.insert(tablet); + tablet.reset(); + + if (flush) { + session.executeNonQueryStatement("FLUSH"); + } + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE write_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE write_and_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from write_and_alter_column_type order by time"); + RowRecord rec = dataSet.next(); + TSDataType newType = isCompatible ? to : from; + assertEquals(1, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getDateV()); + } else { + assertEquals(genValue(newType, 1).toString(), rec.getFields().get(1).toString()); + } + + // write a point + tablet = + new Tablet( + "write_and_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(newType), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 2); + tablet.addValue("s1", 0, genValue(newType, 2)); + session.insert(tablet); + tablet.reset(); + + if (flush) { + session.executeNonQueryStatement("FLUSH"); + } + + dataSet = + session.executeQueryStatement("select * from write_and_alter_column_type order by time"); + rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getDateV()); + } else { + assertEquals(genValue(newType, 1).toString(), rec.getFields().get(1).toString()); + } + + rec = dataSet.next(); + assertEquals(2, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, 2), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, 2), rec.getFields().get(1).getDateV()); + } else { + assertEquals(genValue(newType, 2).toString(), rec.getFields().get(1).toString()); + } + + dataSet = + session.executeQueryStatement( + "select min(s1),max(s1),first(s1),last(s1) from write_and_alter_column_type"); + rec = dataSet.next(); + for (int i = 0; i < 4; i++) { + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, i % 2 + 1), rec.getFields().get(i).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, i % 2 + 1), rec.getFields().get(i).getDateV()); + } else { + assertEquals(genValue(newType, i % 2 + 1).toString(), rec.getFields().get(i).toString()); + } + } + assertFalse(dataSet.hasNext()); + + if (newType.isNumeric()) { + dataSet = + session.executeQueryStatement( + "select avg(s1),sum(s1) from write_and_alter_column_type"); + rec = dataSet.next(); + assertEquals(1.5, rec.getFields().get(0).getDoubleV(), 0.001); + assertEquals(3.0, rec.getFields().get(1).getDoubleV(), 0.001); + assertFalse(dataSet.hasNext()); + } + + session.executeNonQueryStatement("DROP TABLE write_and_alter_column_type"); + } + } + + @Test + public void testAlterWithoutWrite() throws IoTDBConnectionException, StatementExecutionException { + Set<TSDataType> typesToTest = new HashSet<>(); + Collections.addAll(typesToTest, TSDataType.values()); + typesToTest.remove(TSDataType.VECTOR); + typesToTest.remove(TSDataType.UNKNOWN); + + for (TSDataType from : typesToTest) { + for (TSDataType to : typesToTest) { + System.out.printf("testing %s to %s%n", from, to); + doAlterWithoutWrite(from, to, false); + doAlterWithoutWrite(from, to, true); + } + } + } + + private void doAlterWithoutWrite(TSDataType from, TSDataType to, boolean flush) + throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + // create a table with type of "from" + session.executeNonQueryStatement( + "CREATE TABLE IF NOT EXISTS just_alter_column_type (s1 " + from + ")"); + + // alter the type to "to" + boolean isCompatible = MetadataUtils.canAlter(from, to); + if (isCompatible) { + session.executeNonQueryStatement( + "ALTER TABLE just_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } else { + try { + session.executeNonQueryStatement( + "ALTER TABLE just_alter_column_type ALTER COLUMN s1 SET DATA TYPE " + to); + } catch (StatementExecutionException e) { + assertEquals( + "701: New type " + to + " is not compatible with the existing one " + from, + e.getMessage()); + } + } + + TSDataType newType = isCompatible ? to : from; + + // write a point + Tablet tablet = + new Tablet( + "just_alter_column_type", + Collections.singletonList("s1"), + Collections.singletonList(newType), + Collections.singletonList(ColumnCategory.FIELD)); + tablet.addTimestamp(0, 1); + tablet.addValue("s1", 0, genValue(newType, 1)); + session.insert(tablet); + tablet.reset(); + + if (flush) { + session.executeNonQueryStatement("FLUSH"); + } + + SessionDataSet dataSet = + session.executeQueryStatement("select * from just_alter_column_type order by time"); + RowRecord rec = dataSet.next(); + assertEquals(1, rec.getFields().get(0).getLongV()); + if (newType == TSDataType.BLOB) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getBinaryV()); + } else if (newType == TSDataType.DATE) { + assertEquals(genValue(newType, 1), rec.getFields().get(1).getDateV()); + } else { + assertEquals(genValue(newType, 1).toString(), rec.getFields().get(1).toString()); + } + + assertFalse(dataSet.hasNext()); + + session.executeNonQueryStatement("DROP TABLE just_alter_column_type"); + } + } + + @Test + public void testAlterNonExist() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + try { + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN s1 SET DATA TYPE INT64"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("550: Table 'test.non_exist' does not exist", e.getMessage()); + } + session.executeNonQueryStatement( + "ALTER TABLE IF EXISTS non_exist ALTER COLUMN s1 SET DATA TYPE INT64"); + + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS non_exist (s1 int32)"); + + try { + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN s2 SET DATA TYPE INT64"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("616: Column s2 in table 'test.non_exist' does not exist.", e.getMessage()); + } + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN IF EXISTS s2 SET DATA TYPE INT64"); + } + } + + @Test + public void testAlterWrongType() throws IoTDBConnectionException, StatementExecutionException { + try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { + session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS wrong_type (s1 int32)"); + + try { + session.executeNonQueryStatement( + "ALTER TABLE non_exist ALTER COLUMN s1 SET DATA TYPE VECTOR"); + fail("Should throw exception"); + } catch (StatementExecutionException e) { + assertEquals("701: Unknown type: VECTOR", e.getMessage()); + } + } + } ++ ++ @Test ++ public void testDropAndAlter() throws IoTDBConnectionException, StatementExecutionException { ++ try (ITableSession session = EnvFactory.getEnv().getTableSessionConnectionWithDB("test")) { ++ session.executeNonQueryStatement("CREATE TABLE IF NOT EXISTS drop_and_alter (s1 int32)"); ++ ++ Tablet tablet = ++ new Tablet( ++ "drop_and_alter", ++ Collections.singletonList("s1"), ++ Collections.singletonList(TSDataType.INT32), ++ Collections.singletonList(ColumnCategory.FIELD)); ++ tablet.addTimestamp(0, 1); ++ tablet.addValue("s1", 0, genValue(TSDataType.INT32, 1)); ++ session.insert(tablet); ++ tablet.reset(); ++ ++ session.executeNonQueryStatement("ALTER TABLE drop_and_alter DROP COLUMN s1"); ++ ++ tablet = ++ new Tablet( ++ "drop_and_alter", ++ Collections.singletonList("s1"), ++ Collections.singletonList(TSDataType.STRING), ++ Collections.singletonList(ColumnCategory.FIELD)); ++ tablet.addTimestamp(0, 2); ++ tablet.addValue("s1", 0, genValue(TSDataType.STRING, 2)); ++ session.insert(tablet); ++ tablet.reset(); ++ ++ session.executeNonQueryStatement( ++ "ALTER TABLE drop_and_alter ALTER COLUMN s1 SET DATA TYPE TEXT"); ++ ++ tablet = ++ new Tablet( ++ "drop_and_alter", ++ Collections.singletonList("s1"), ++ Collections.singletonList(TSDataType.TEXT), ++ Collections.singletonList(ColumnCategory.FIELD)); ++ tablet.addTimestamp(0, 3); ++ tablet.addValue("s1", 0, genValue(TSDataType.STRING, 3)); ++ session.insert(tablet); ++ tablet.reset(); ++ ++ SessionDataSet dataSet = ++ session.executeQueryStatement("select * from drop_and_alter order by time"); ++ RowRecord rec = dataSet.next(); ++ assertEquals(2, rec.getFields().get(0).getLongV()); ++ assertEquals(genValue(TSDataType.STRING, 2).toString(), rec.getFields().get(1).toString()); ++ rec = dataSet.next(); ++ assertEquals(3, rec.getFields().get(0).getLongV()); ++ assertEquals(genValue(TSDataType.STRING, 3).toString(), rec.getFields().get(1).toString()); ++ assertFalse(dataSet.hasNext()); ++ } ++ } +} diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java index e3857b1f03c,15236010654..9de4d1886ec --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/ConfigPhysicalPlanType.java @@@ -171,8 -171,8 +171,10 @@@ public enum ConfigPhysicalPlanType PreDeleteColumn((short) 860), CommitDeleteColumn((short) 861), DescTable((short) 862), - AlterColumnDataType((short) 863), - CommitAlterColumnDataType((short) 864), + ShowTable4InformationSchema((short) 863), + DescTable4InformationSchema((short) 864), ++ AlterColumnDataType((short) 865), ++ CommitAlterColumnDataType((short) 866), /** Deprecated types for sync, restored them for upgrade. */ @Deprecated diff --cc iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java index 8c03e76a2d1,debf7510699..ffd4f524e36 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/persistence/schema/ClusterSchemaInfo.java @@@ -78,8 -80,8 +81,9 @@@ import org.apache.iotdb.confignode.cons import org.apache.iotdb.confignode.consensus.response.template.TemplateInfoResp; import org.apache.iotdb.confignode.consensus.response.template.TemplateSetInfoResp; import org.apache.iotdb.confignode.exception.DatabaseNotExistsException; +import org.apache.iotdb.confignode.persistence.schema.ConfigMTree.TableSchemaDetails; import org.apache.iotdb.confignode.rpc.thrift.TDatabaseSchema; + import org.apache.iotdb.confignode.rpc.thrift.TTableColumnInfo; import org.apache.iotdb.confignode.rpc.thrift.TTableInfo; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; import org.apache.iotdb.db.exception.sql.SemanticException; @@@ -106,6 -107,6 +110,7 @@@ import java.util.HashMap import java.util.HashSet; import java.util.List; import java.util.Map; ++import java.util.Map.Entry; import java.util.Optional; import java.util.Set; import java.util.UUID; @@@ -1247,6 -1273,51 +1280,61 @@@ public class ClusterSchemaInfo implemen } } + public DescTable4InformationSchemaResp descTable4InformationSchema() { + databaseReadWriteLock.readLock().lock(); + try { + return new DescTable4InformationSchemaResp( + StatusUtils.OK, + tableModelMTree.getAllDatabasePaths(true).stream() + .collect( + Collectors.toMap( + databasePath -> PathUtils.unQualifyDatabaseName(databasePath.getFullPath()), + databasePath -> { + try { + return tableModelMTree + .getAllTablesUnderSpecificDatabase(databasePath) + .stream() + .map( + pair -> { + try { + return tableModelMTree.getTableSchemaDetails( + databasePath, pair.getLeft().getTableName()); + } catch (final MetadataException ignore) { + // Table path must exist because the "getTableSchemaDetails()" + // is called in databaseReadWriteLock.readLock(). + } - return new Pair<TsTable, Set<String>>(null, null); ++ return new TableSchemaDetails(); + }) + .collect( + Collectors.toMap( - pair -> pair.getLeft().getTableName(), - pair -> ++ tableSchemaDetails -> tableSchemaDetails.table.getTableName(), ++ tableSchemaDetails -> + new TTableColumnInfo() + .setTableInfo( + TsTableInternalRPCUtil.serializeSingleTsTable( - pair.getLeft())) - .setPreDeletedColumns(pair.getRight()))); ++ tableSchemaDetails.table)) ++ .setPreDeletedColumns( ++ tableSchemaDetails.preDeletedColumns) ++ .setPreAlteredColumns( ++ tableSchemaDetails ++ .preAlteredColumns ++ .entrySet() ++ .stream() ++ .collect( ++ Collectors.toMap( ++ Entry::getKey, ++ e -> e.getValue().serialize()))))); + } catch (final MetadataException ignore) { + // Database path must exist because the "getAllDatabasePaths()" is called + // in databaseReadWriteLock.readLock(). + } + return Collections.emptyMap(); + }))); + } finally { + databaseReadWriteLock.readLock().unlock(); + } + } + public Map<String, List<TsTable>> getAllUsingTables() { databaseReadWriteLock.readLock().lock(); try { diff --cc iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift index cbca5db7c4a,ebe67399552..b30660b80aa --- a/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift +++ b/iotdb-protocol/thrift-confignode/src/main/thrift/confignode.thrift @@@ -1075,9 -1093,18 +1093,20 @@@ struct TDescTableResp 1: required common.TSStatus status 2: optional binary tableInfo 3: optional set<string> preDeletedColumns + 4: optional map<string, byte> preAlteredColumns } + struct TDescTable4InformationSchemaResp { + 1: required common.TSStatus status + 2: optional map<string, map<string, TTableColumnInfo>> tableColumnInfoMap + } + + struct TTableColumnInfo { + 1: required binary tableInfo + 2: optional set<string> preDeletedColumns ++ 3: optional map<string, byte> preAlteredColumns + } + struct TFetchTableResp { 1: required common.TSStatus status 2: optional binary tableInfoMap
