This is an automated email from the ASF dual-hosted git repository. jackietien pushed a commit to branch mergeMaster0607 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit dccc4832b9b733f04200cd2db875c0aaa00c3e85 Merge: 411255134b1 9340708c4f8 Author: JackieTien97 <[email protected]> AuthorDate: Fri Jun 7 17:48:05 2024 +0800 resolve conflicts .github/workflows/cluster-it-1c1d.yml | 2 +- .github/workflows/cluster-it-1c3d.yml | 2 +- .github/workflows/pipe-it-2cluster.yml | 2 +- .github/workflows/sonar-codecov.yml | 2 +- .github/workflows/unit-test.yml | 2 +- .gitignore | 1 + .mvn/{gradle-enterprise.xml => develocity.xml} | 17 +- .mvn/extensions.xml | 6 +- code-coverage/pom.xml | 2 +- distribution/pom.xml | 10 +- example/client-cpp-example/pom.xml | 2 +- example/jdbc/pom.xml | 2 +- .../java/org/apache/iotdb/JDBCCharsetExample.java | 2 +- example/mqtt-customize/pom.xml | 2 +- example/mqtt/pom.xml | 2 +- example/pipe-count-point-processor/pom.xml | 2 +- example/pipe-opc-ua-sink/pom.xml | 2 +- example/pom.xml | 2 +- example/rest-java-example/pom.xml | 2 +- example/schema/pom.xml | 2 +- example/session/pom.xml | 2 +- example/trigger/pom.xml | 2 +- example/udf/pom.xml | 2 +- integration-test/pom.xml | 32 +- .../org/apache/iotdb/db/it/IoTDBRepairDataIT.java | 3 +- .../db/it/orderBy/IoTDBOrderByForDebugIT.java | 1406 ++++++++++++++++++++ .../db/it/regionscan/IoTDBActiveRegionScanIT.java | 23 +- .../db/it/schema/IoTDBCreateTimeseriesIT.java | 31 + .../iotdb/db/it/selectinto/IoTDBSelectIntoIT.java | 2 +- .../iotdb/db/it/udaf/IoTDBUDAFOrderByIT.java | 5 +- .../it/autocreate/IoTDBPipeAutoConflictIT.java | 9 +- .../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 62 +- .../pipe/it/autocreate/IoTDBPipeClusterIT.java | 44 +- .../IoTDBPipeConnectorCompressionIT.java | 126 ++ .../pipe/it/autocreate/IoTDBPipeLifeCycleIT.java | 31 +- .../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 2 +- .../pipe/it/manual/IoTDBPipeMetaRestartIT.java | 18 +- .../it/dual/AbstractSubscriptionDualIT.java | 13 +- .../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 9 + .../it/dual/IoTDBSubscriptionTimePrecisionIT.java | 195 +++ .../it/dual/IoTDBSubscriptionTopicIT.java | 2 +- .../it/local/IoTDBSubscriptionBasicIT.java | 118 +- .../it/local/IoTDBSubscriptionRestartIT.java | 31 +- .../apache/iotdb/tools/it/ExportDataTestIT.java | 12 +- .../apache/iotdb/tools/it/ImportDataTestIT.java | 16 +- iotdb-api/external-api/pom.xml | 2 +- iotdb-api/pipe-api/pom.xml | 2 +- iotdb-api/pom.xml | 2 +- iotdb-api/trigger-api/pom.xml | 2 +- iotdb-api/udf-api/pom.xml | 2 +- .../org/apache/iotdb/udf/api/utils/RowImpl.java | 21 + iotdb-client/cli/pom.xml | 18 +- .../org/apache/iotdb/tool/AbstractDataTool.java | 33 +- .../java/org/apache/iotdb/tool/ExportData.java | 14 +- .../java/org/apache/iotdb/tool/ImportData.java | 8 +- .../java/org/apache/iotdb/tool/ImportSchema.java | 62 +- iotdb-client/client-cpp/pom.xml | 4 +- iotdb-client/client-py/pom.xml | 4 +- iotdb-client/client-py/requirements.txt | 4 +- iotdb-client/client-py/resources/setup.py | 7 +- iotdb-client/isession/pom.xml | 8 +- iotdb-client/jdbc/pom.xml | 8 +- .../org/apache/iotdb/jdbc/IoTDBJDBCResultSet.java | 12 +- iotdb-client/pom.xml | 2 +- iotdb-client/service-rpc/pom.xml | 6 +- .../java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 8 +- iotdb-client/session/pom.xml | 10 +- iotdb-core/antlr/pom.xml | 2 +- iotdb-core/confignode/pom.xml | 24 +- .../metric/PipeConfigRegionExtractorMetrics.java | 4 +- .../procedure/env/RegionMaintainHandler.java | 1 + .../subscription/topic/CreateTopicProcedure.java | 6 +- iotdb-core/consensus/pom.xml | 14 +- .../consensus/iot/logdispatcher/LogDispatcher.java | 34 +- .../apache/iotdb/consensus/iot/ReplicateTest.java | 111 +- iotdb-core/datanode/pom.xml | 36 +- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 + .../PipeTransferTabletBatchEventHandler.java | 28 +- .../PipeTransferTabletInsertionEventHandler.java | 22 +- .../PipeTransferTsFileInsertionEventHandler.java | 18 +- .../event/common/heartbeat/PipeHeartbeatEvent.java | 48 +- .../PipeHistoricalDataRegionTsFileExtractor.java | 68 +- .../realtime/PipeRealtimeDataRegionExtractor.java | 11 +- .../PipeRealtimeDataRegionHybridExtractor.java | 9 +- .../PipeDataNodeRemainingEventAndTimeOperator.java | 3 - .../subtask/connector/PipeConnectorSubtask.java | 2 +- .../process/ActiveRegionScanMergeOperator.java | 37 +- .../AbstractRegionScanDataSourceOperator.java | 19 +- .../source/ActiveDeviceRegionScanOperator.java | 40 +- .../source/ActiveTimeSeriesRegionScanOperator.java | 34 +- .../queryengine/plan/analyze/AnalyzeVisitor.java | 23 +- .../config/executor/ClusterConfigTaskExecutor.java | 8 +- .../plan/optimization/LimitOffsetPushDown.java | 3 +- .../db/queryengine/plan/parser/ASTVisitor.java | 8 +- .../plan/planner/OperatorTreeGenerator.java | 10 +- .../plan/planner/distribution/SourceRewriter.java | 1 + .../plan/scheduler/AsyncPlanNodeSender.java | 6 +- .../scheduler/FragmentInstanceDispatcherImpl.java | 52 +- .../execute/recover/CompactionRecoverManager.java | 2 +- .../dataregion/tsfile/TsFileResource.java | 22 +- .../tsfile/timeindex/ArrayDeviceTimeIndex.java | 16 + .../dataregion/tsfile/timeindex/FileTimeIndex.java | 5 + .../receiver/SubscriptionReceiverV1.java | 11 +- .../plan/optimization/LimitOffsetPushDownTest.java | 28 + iotdb-core/metrics/core/pom.xml | 4 +- iotdb-core/metrics/interface/pom.xml | 8 +- iotdb-core/metrics/pom.xml | 2 +- iotdb-core/node-commons/pom.xml | 26 +- .../resources/conf/iotdb-system.properties | 18 + .../apache/iotdb/commons/conf/CommonConfig.java | 20 + .../iotdb/commons/conf/CommonDescriptor.java | 16 + .../config/constant/PipeConnectorConstant.java | 10 + ...TDCompressor.java => PipeCompressorConfig.java} | 28 +- .../compressor/PipeCompressorFactory.java | 62 +- .../connector/compressor/PipeZSTDCompressor.java | 17 +- .../pipe/connector/protocol/IoTDBConnector.java | 25 +- .../service/metric/PerformanceOverviewMetrics.java | 21 + .../commons/subscription/meta/topic/TopicMeta.java | 2 +- .../apache/iotdb/commons/utils/StatusUtils.java | 22 +- iotdb-core/pom.xml | 2 +- iotdb-protocol/openapi/pom.xml | 2 +- iotdb-protocol/pom.xml | 2 +- iotdb-protocol/thrift-commons/pom.xml | 2 +- iotdb-protocol/thrift-confignode/pom.xml | 4 +- iotdb-protocol/thrift-consensus/pom.xml | 4 +- iotdb-protocol/thrift-datanode/pom.xml | 4 +- library-udf/pom.xml | 4 +- pom.xml | 2 +- 128 files changed, 2946 insertions(+), 631 deletions(-) diff --cc iotdb-core/datanode/pom.xml index 61a76f891c7,cabc80a5aa1..62836bd827a --- a/iotdb-core/datanode/pom.xml +++ b/iotdb-core/datanode/pom.xml @@@ -77,13 -77,8 +77,13 @@@ <dependency> <groupId>org.apache.iotdb</groupId> <artifactId>iotdb-antlr</artifactId> - <version>1.3.2-SNAPSHOT</version> + <version>1.3.3-SNAPSHOT</version> </dependency> + <dependency> + <groupId>org.apache.iotdb</groupId> + <artifactId>iotdb-relational-grammar</artifactId> + <version>1.3.2-SNAPSHOT</version> + </dependency> <dependency> <groupId>org.apache.iotdb</groupId> <artifactId>iotdb-thrift-commons</artifactId> diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java index 9868afb0ca0,873352b8b74..1f7478a0070 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveDeviceRegionScanOperator.java @@@ -70,21 -72,26 +72,26 @@@ public class ActiveDeviceRegionScanOper @Override protected void updateActiveData() { - TimeColumnBuilder timeColumnBuilder = resultTsBlockBuilder.getTimeColumnBuilder(); - ColumnBuilder[] columnBuilders = resultTsBlockBuilder.getValueColumnBuilders(); - List<IDeviceID> activeDevices = ((RegionScanForActiveDeviceUtil) regionScanUtil).getActiveDevices(); - for (IDeviceID deviceID : activeDevices) { - timeColumnBuilder.writeLong(-1); - columnBuilders[0].writeBinary(new Binary(deviceID.toString(), TSFileConfig.STRING_CHARSET)); - columnBuilders[1].writeBinary( - new Binary( - String.valueOf(deviceToAlignedMap.get(deviceID)), TSFileConfig.STRING_CHARSET)); - columnBuilders[2].appendNull(); - columnBuilders[3].appendNull(); - resultTsBlockBuilder.declarePosition(); - deviceToAlignedMap.remove(deviceID); + + if (this.outputCount) { + count += activeDevices.size(); + activeDevices.forEach(deviceToAlignedMap.keySet()::remove); + } else { + TimeColumnBuilder timeColumnBuilder = resultTsBlockBuilder.getTimeColumnBuilder(); + ColumnBuilder[] columnBuilders = resultTsBlockBuilder.getValueColumnBuilders(); + for (IDeviceID deviceID : activeDevices) { + timeColumnBuilder.writeLong(-1); - columnBuilders[0].writeBinary(new Binary(deviceID.getBytes())); ++ columnBuilders[0].writeBinary(new Binary(deviceID.toString(), TSFileConfig.STRING_CHARSET)); + columnBuilders[1].writeBinary( + new Binary( + String.valueOf(deviceToAlignedMap.get(deviceID)), TSFileConfig.STRING_CHARSET)); + columnBuilders[2].appendNull(); + columnBuilders[3].appendNull(); + resultTsBlockBuilder.declarePosition(); + deviceToAlignedMap.remove(deviceID); + } } } diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java index 6df57b1140c,e3385f92ecd..c1f4cc951d0 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/ActiveTimeSeriesRegionScanOperator.java @@@ -94,9 -97,19 +96,19 @@@ public class ActiveTimeSeriesRegionScan Map<IDeviceID, List<String>> activeTimeSeries = ((RegionScanForActiveTimeSeriesUtil) regionScanUtil).getActiveTimeSeries(); + + if (outputCount) { + for (Map.Entry<IDeviceID, List<String>> entry : activeTimeSeries.entrySet()) { + List<String> timeSeriesList = entry.getValue(); + count += timeSeriesList.size(); + removeTimeseriesListFromDevice(entry.getKey(), timeSeriesList); + } + return; + } + for (Map.Entry<IDeviceID, List<String>> entry : activeTimeSeries.entrySet()) { IDeviceID deviceID = entry.getKey(); - String deviceStr = ((PlainDeviceID) deviceID).toStringID(); + String deviceStr = deviceID.toString(); List<String> timeSeriesList = entry.getValue(); Map<String, TimeseriesSchemaInfo> timeSeriesInfo = timeSeriesToSchemasInfo.get(deviceID); for (String timeSeries : timeSeriesList) { diff --cc iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index 86a610063d7,f233261d076..d220e9f9e70 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@@ -440,6 -448,22 +440,22 @@@ public class ArrayDeviceTimeIndex imple @Override public byte getTimeIndexType() { - return DEVICE_TIME_INDEX_TYPE; + return ARRAY_DEVICE_TIME_INDEX_TYPE; } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append(" DeviceIndexMapSize = ").append(deviceToIndex.size()); + builder.append(" startTimeLength = ").append(startTimes.length); + builder.append(" endTimeLength = ").append(endTimes.length); + builder.append(" DeviceIndexMap = ["); + deviceToIndex.forEach( + (key, value) -> + builder.append(" device = ").append(key).append(", index = ").append(value)); + builder.append("]"); + builder.append(" StartTimes = ").append(Arrays.toString(startTimes)); + builder.append(" EndTimes = ").append(Arrays.toString(endTimes)); + return builder.toString(); + } }
