This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/agg_template_alignbydevice
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 569d6eddcd5 add ascending and descending aggregation descriptors
support
add cfe19176edb Fix compaction write rate can not hot load
add e3aa4d8954c Pipe: RPC compression for data sync connectors (SNAPPY,
GZIP, LZ4, ZSTD and LZMA2) (#12519)
add 44ed96111ab Add device number metric
add 58331a6c73d bump ratis to 3.1.0-snapshot (#12533)
add b50ec7d64a4 Revert "bump ratis to 3.1.0-snapshot (#12533)"
add e0eea1d14bf JDBC: Fix maxRows cannot be set to 0 (#12540)
add 2f1ca39f391 Fix non-root user query data returns unexpected path
add 29fab685489 Pipe IT: Added flush on
IoTDBPipeExtractorIT#testExtractorPatternMatch() to avoid pipe on previous
follower getting unclosed historical tsfile from the old leader (#12534)
add 8562eead2bc add remote storage block for tiered storage (#12535)
add 95708409a01 [RatisConsensus] Use empty map for create peer (#12463)
add 9f92b1ae829 bump ratis 3.1.0-d09cbac-SNAPSHOT (#12547)
add 81382d71c05 Fix client cpp IT warning (#12548)
add 9f61a4e2aca Feat/jenkins auto redeploy (#12542)
add e1ea758f173 ODBC: Add example to the example module (#12544)
add f9292455bce Fixed deadlock problem caused by concurrent auto create
schema and delete database (#12489)
add 318718dad28 Bump org.jacoco:jacoco-maven-plugin from 0.8.10 to 0.8.12
(#12457)
add d81972227d9 Check duplicated measurements in one row for all insert
APIs
add 337bfe5b891 Pipe IT: Fix
IoTDBPipeClusterIT#testCreatePipesWithSameConnector() fails because the thread
is interrupted or encountered non-pipe exceptions (#12553)
add 70f18ccc4e2 Load Tsfile Tool: load does not delete the source tsfile
by default (#12549)
add 9d564e33618 Pipe: implemented ChangingValueSamplingProcessor (#12466)
add b0306420a10 Pipe: Fix RPC payload compression in
PipeTransferTsFileInsertionEventHandler (#12558)
add e4f4ed03113 Pipe: Added metrics to schema transfer / remaining event /
remaining time (#12478)
add 77d61207f6c Pipe IT: Fix CN connection timeout in IT is not functional
(#12563)
add 3b0dfcd1736 Pipe: Enable realtime first strategy in data transfer
(#12559)
add cabdec75b63 Fixed restart error when DataRegion is RatisConsensus
(#12554)
add 49dc7650e80 Pipe: Support parsing pattern when tranferring schema
plans (#12464)
add bb80f6cea14 Fix Python IT (#12567)
add 8150f7bde1d Pipe: Adjust the reloading logic to restart all the pipes
when deleted tsfiles are linked too many (#12566)
add 7d4dd9e1427 Pipe: Fix insertNode.getDevicePath() is not handled
correctly for insertRowsNode (NPE) (#12569)
add f1d8fce46c2 Pipe: Fix PipeSetTTLPlan is not handled correctly (#12571)
add 9a29af7ce1d [IOTDB-6325] Support RegionScan for active metadata
queries [BE Part One]
add 40934ddc572 Pipe: Add completion signal to historical events & allow
all data regions' completion signal to drop the pipe (#12490)
add 0ebac6b7e11 Modify strategy of wait task time limitation & Fix proc id
(#12552)
add b2c5680f3d3 [RatisConsensus] add ut for force snapshot (#12560)
add 911d7b673a6 Pipe: Fixed the bug that schema region listening queue is
not cleared when the schema region is deleted / migrated (#12575)
add 462c9004a55 Introducing a memory control mechanism during the query
planning stage
add 7ef1eb722e9 Pipe: add parameter realtime.loose-range to support
subscription loose range semantics (#12580)
add e49a07d1129 Pipe: fix PipeEnrichedStatement can't contain redirect
info (#12579)
add f60694b4a46 Add TsFile develop branch for IoTDB in README (#12581)
add fad655319e1 Avoid infinite and unecessary loop when CN RPC processors
are killed/interrupted by OS (#12584)
add 1517e81e878 Fix python ci security warning and update c++ ci commands
(#12583)
add aa2687c1fe7 Pipe: Fix BatchActivateTemplateStatement is not handled
correctly when some of the timeseries already exists (#12587)
add 3d42a2a0292 merge master
No new revisions were added by this update.
Summary of changes:
.github/workflows/client-cpp.yml | 51 +--
.github/workflows/client-go.yml | 5 +-
.github/workflows/client-python.yml | 2 +-
.mvn/wrapper/maven-wrapper.properties | 4 +-
Jenkinsfile | 2 +-
README.md | 2 +
README_ZH.md | 2 +
example/odbc/.gitignore | 344 +++++++++++++++++
example/odbc/ODBC-Example.sln | 35 ++
example/odbc/ODBC-Example/ODBC-Example.csproj | 40 ++
example/odbc/ODBC-Example/ODBC_Example.cs | 84 ++++
example/odbc/README.md | 81 ++++
.../it/env/cluster/config/MppCommonConfig.java | 6 +
.../it/env/cluster/config/MppConfigNodeConfig.java | 6 -
.../env/cluster/config/MppSharedCommonConfig.java | 7 +
.../it/env/remote/config/RemoteCommonConfig.java | 5 +
.../env/remote/config/RemoteConfigNodeConfig.java | 5 -
.../org/apache/iotdb/itbase/env/CommonConfig.java | 2 +
.../apache/iotdb/itbase/env/ConfigNodeConfig.java | 2 -
.../org/apache/iotdb/db/it/auth/IoTDBAuthIT.java | 29 ++
.../pipe/it/autocreate/AbstractPipeDualAutoIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeAutoDropIT.java | 80 ++++
.../pipe/it/autocreate/IoTDBPipeClusterIT.java | 67 ++--
...T.java => IoTDBPipeConnectorCompressionIT.java} | 115 ++++--
.../pipe/it/autocreate/IoTDBPipeDataSinkIT.java | 17 +-
.../pipe/it/autocreate/IoTDBPipeExtractorIT.java | 51 +--
.../pipe/it/autocreate/IoTDBPipeIdempotentIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeProcessorIT.java | 4 +-
.../pipe/it/autocreate/IoTDBPipeProtocolIT.java | 12 +-
.../pipe/it/manual/AbstractPipeDualManualIT.java | 4 +-
.../iotdb/pipe/it/manual/IoTDBPipeInclusionIT.java | 56 ++-
.../pipe/it/manual/IoTDBPipeMetaHistoricalIT.java | 13 +-
.../it/manual/IoTDBPipeMetaLeaderChangeIT.java | 4 +-
.../iotdb/pipe/it/single/AbstractPipeSingleIT.java | 2 +-
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 134 +++++++
.../it/dual/AbstractSubscriptionDualIT.java | 4 +-
.../parameter/PipeParameterValidator.java | 4 +-
iotdb-client/client-py/requirements_dev.txt | 2 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 4 +-
iotdb-core/confignode/pom.xml | 4 +
.../heartbeat/DataNodeHeartbeatHandler.java | 3 +-
.../consensus/request/ConfigPhysicalPlan.java | 4 +
.../consensus/request/ConfigPhysicalPlanType.java | 1 +
.../consensus/request/auth/AuthorPlan.java | 21 +-
.../pipe/payload/PipeDeleteLogicalViewPlan.java | 11 +-
.../pipe/payload/PipeDeleteTimeSeriesPlan.java | 11 +-
.../request/write/pipe/payload/PipeSetTTLPlan.java | 103 +++++
.../iotdb/confignode/manager/ClusterManager.java | 2 +
.../iotdb/confignode/manager/ConfigManager.java | 4 +-
.../manager/consensus/ConsensusManager.java | 8 +
.../payload/PipeTransferConfigSnapshotSealReq.java | 33 +-
.../protocol/IoTDBConfigRegionAirGapConnector.java | 29 +-
.../protocol/IoTDBConfigRegionConnector.java | 22 +-
.../runtime/PipeRuntimeCoordinator.java | 18 +-
.../runtime/heartbeat/PipeHeartbeat.java | 61 +++
.../{ => heartbeat}/PipeHeartbeatParser.java | 51 +--
.../{ => heartbeat}/PipeHeartbeatScheduler.java | 20 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 1 +
.../pipe/execution/PipeConfigNodeSubtask.java | 52 ++-
.../extractor/ConfigRegionListeningFilter.java | 8 +-
.../pipe/extractor/IoTDBConfigRegionExtractor.java | 31 +-
.../PipeConfigPhysicalPlanPatternParseVisitor.java | 287 ++++++++++++++
.../pipe/metric/PipeConfigNodeListenerMetrics.java | 67 ++++
.../manager/pipe/metric/PipeConfigNodeMetrics.java | 14 +-
.../metric/PipeConfigNodeRemainingTimeMetrics.java | 169 ++++++++
.../PipeConfigNodeRemainingTimeOperator.java | 119 ++++++
.../metric/PipeConfigRegionConnectorMetrics.java | 157 ++++++++
.../metric/PipeConfigRegionExtractorMetrics.java | 139 +++++++
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 45 ++-
.../PipeConfigPhysicalPlanTSStatusVisitor.java | 55 +--
.../persistence/executor/ConfigPlanExecutor.java | 8 +-
.../confignode/persistence/pipe/PipeTaskInfo.java | 89 +++--
.../procedure/env/RegionMaintainHandler.java | 40 +-
.../impl/region/AddRegionPeerProcedure.java | 4 +-
.../thrift/ConfigNodeRPCServiceProcessor.java | 65 ++--
.../request/ConfigPhysicalPlanSerDeTest.java | 111 +++---
.../connector/PipeConfigNodeThriftRequestTest.java | 2 +-
.../extractor/IoTDBConfigRegionExtractorTest.java | 7 +-
...eConfigPhysicalPlanPatternParseVisitorTest.java | 408 ++++++++++++++++++++
.../org/apache/iotdb/consensus/IConsensus.java | 3 +-
.../apache/iotdb/consensus/ratis/DiskGuardian.java | 65 ++--
.../iotdb/consensus/ratis/RatisConsensus.java | 113 +++---
.../iotdb/consensus/ratis/DiskGuardianTest.java | 28 +-
.../iotdb/consensus/ratis/RatisConsensusTest.java | 15 +
.../apache/iotdb/consensus/ratis/TestUtils.java | 21 +
iotdb-core/datanode/pom.xml | 4 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 2 +-
.../schemaregion/SchemaExecutionVisitor.java | 25 +-
.../PipeDataRegionProcessorConstructor.java | 4 +
.../runtime/PipeSchemaRegionListenerManager.java | 44 ++-
.../db/pipe/agent/task/PipeDataNodeTaskAgent.java | 203 ++++++++--
.../builder/PipeTransferBatchReqBuilder.java | 7 +-
.../evolvable/request/PipeTransferPlanNodeReq.java | 1 +
.../request/PipeTransferSchemaSnapshotSealReq.java | 32 +-
.../airgap/IoTDBDataNodeAirGapConnector.java | 12 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 24 +-
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 17 +-
.../async/IoTDBDataRegionAsyncConnector.java | 45 ++-
.../PipeTransferTabletBatchEventHandler.java | 7 +-
.../PipeTransferTabletInsertNodeEventHandler.java | 7 +-
.../PipeTransferTsFileInsertionEventHandler.java | 22 +-
.../thrift/sync/IoTDBDataNodeSyncConnector.java | 5 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 42 +-
.../thrift/sync/IoTDBSchemaRegionConnector.java | 17 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 34 +-
.../schema/PipeSchemaRegionWritePlanEvent.java | 4 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 24 +-
.../tablet/TabletInsertionDataContainer.java | 32 +-
.../common/terminate/PipeTerminateEvent.java} | 75 ++--
.../tsfile/TsFileInsertionDataContainer.java | 28 +-
.../common/tsfile/TsFileInsertionPointCounter.java | 10 +-
.../event/realtime/PipeRealtimeEventFactory.java | 1 +
.../dataregion/DataRegionWatermarkInjector.java | 2 +-
.../dataregion/IoTDBDataRegionExtractor.java | 49 +--
.../PipeHistoricalDataRegionTsFileExtractor.java | 56 ++-
.../realtime/PipeRealtimeDataRegionExtractor.java | 50 ++-
.../PipeRealtimeDataRegionHybridExtractor.java | 13 +-
.../dataregion/realtime/epoch/TsFileEpoch.java | 4 +-
.../schemaregion/IoTDBSchemaRegionExtractor.java | 26 +-
.../schemaregion/PipePlanPatternParseVisitor.java | 282 ++++++++++++++
.../iotdb/db/pipe/metric/PipeDataNodeMetrics.java | 20 +-
.../PipeDataNodeRemainingEventAndTimeMetrics.java | 219 +++++++++++
.../PipeDataNodeRemainingEventAndTimeOperator.java | 190 +++++++++
...cs.java => PipeDataRegionConnectorMetrics.java} | 91 ++---
...cs.java => PipeDataRegionExtractorMetrics.java} | 129 ++++---
.../iotdb/db/pipe/metric/PipeProcessorMetrics.java | 39 --
.../metric/PipeSchemaRegionConnectorMetrics.java | 159 ++++++++
.../metric/PipeSchemaRegionExtractorMetrics.java | 144 +++++++
.../metric/PipeSchemaRegionListenerMetrics.java | 135 +++++++
.../pipe/pattern/CachedSchemaPatternMatcher.java | 10 +-
.../downsampling/changing/ChangingValueFilter.java | 95 +++++
.../changing/ChangingValueSamplingProcessor.java | 199 ++++++++++
.../sdt/SwingingDoorTrendingFilter.java | 4 +-
.../protocol/thrift/IoTDBDataNodeReceiver.java | 16 +-
.../visitor/PipePlanToStatementVisitor.java | 2 +-
.../visitor/PipeStatementExceptionVisitor.java | 39 +-
.../visitor/PipeStatementPatternParseVisitor.java | 136 +++++++
.../visitor/PipeStatementTSStatusVisitor.java | 10 +
.../iotdb/db/pipe/task/PipeDataNodeTask.java | 20 +-
.../pipe/task/connection/PipeEventCollector.java | 114 ++----
.../db/pipe/task/stage/PipeTaskConnectorStage.java | 4 +-
.../db/pipe/task/stage/PipeTaskProcessorStage.java | 4 +-
.../subtask/connector/PipeConnectorSubtask.java | 88 +++--
.../connector/PipeConnectorSubtaskLifeCycle.java | 8 +-
.../connector/PipeConnectorSubtaskManager.java | 26 +-
.../PipeRealtimePriorityBlockingQueue.java | 136 +++++++
.../subtask/processor/PipeProcessorSubtask.java | 33 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 19 +-
.../thrift/impl/DataNodeRegionManager.java | 2 +
.../db/queryengine/common/MPPQueryContext.java | 60 +++
.../common/schematree/ClusterSchemaTree.java | 5 +-
.../exception/MemoryNotEnoughException.java | 7 +-
.../execution/MemoryEstimationHelper.java | 6 +-
.../queryengine/execution/driver/DataDriver.java | 25 +-
.../execution/driver/DataDriverContext.java | 4 +-
.../fragment/FragmentInstanceContext.java | 150 ++++++--
.../execution/fragment/QueryContext.java | 76 +++-
.../source/AbstractDataSourceOperator.java | 5 +-
.../operator/source/AlignedSeriesScanOperator.java | 5 +-
.../operator/source/DataSourceOperator.java | 4 +-
.../iotdb/db/queryengine/plan/Coordinator.java | 3 +
.../queryengine/plan/analyze/AnalyzeVisitor.java | 168 +++++---
.../plan/analyze/ConcatPathRewriter.java | 33 +-
.../plan/analyze/ExpressionAnalyzer.java | 42 +-
.../queryengine/plan/analyze/ExpressionUtils.java | 66 +++-
.../queryengine/plan/analyze/TemplatedAnalyze.java | 86 +++--
.../plan/analyze/schema/ClusterSchemaFetcher.java | 4 +-
.../queryengine/plan/execution/QueryExecution.java | 3 +
.../db/queryengine/plan/expression/Expression.java | 3 +-
.../plan/expression/binary/BinaryExpression.java | 12 +
.../plan/expression/leaf/ConstantOperand.java | 9 +
.../plan/expression/leaf/NullOperand.java | 10 +
.../plan/expression/leaf/TimeSeriesOperand.java | 10 +
.../plan/expression/leaf/TimestampOperand.java | 10 +
.../plan/expression/multi/FunctionExpression.java | 26 ++
.../expression/other/CaseWhenThenExpression.java | 16 +
.../expression/other/GroupByTimeExpression.java | 9 +
.../plan/expression/ternary/BetweenExpression.java | 14 +
.../plan/expression/unary/InExpression.java | 12 +-
.../plan/expression/unary/IsNullExpression.java | 10 +
.../plan/expression/unary/LikeExpression.java | 12 +
.../plan/expression/unary/LogicNotExpression.java | 11 +
.../plan/expression/unary/NegationExpression.java | 11 +
.../plan/expression/unary/RegularExpression.java | 12 +
.../cartesian/BindSchemaForExpressionVisitor.java | 50 ++-
.../cartesian/BindSchemaForPredicateVisitor.java | 43 ++-
.../visitor/cartesian/CartesianProductVisitor.java | 23 +-
...catDeviceAndBindSchemaForExpressionVisitor.java | 27 +-
...ncatDeviceAndBindSchemaForPredicateVisitor.java | 30 +-
.../ConcatExpressionWithSuffixPathsVisitor.java | 28 +-
.../visitor/cartesian/QueryContextProvider.java} | 9 +-
.../plan/optimization/AggregationPushDown.java | 77 +++-
.../db/queryengine/plan/parser/ASTVisitor.java | 6 +-
.../plan/planner/LocalExecutionPlanner.java | 22 +-
.../plan/planner/LogicalPlanBuilder.java | 76 ++--
.../queryengine/plan/planner/TreeModelPlanner.java | 15 +-
.../distribution/DistributionPlanContext.java | 6 +
.../plan/planner/distribution/SourceRewriter.java | 9 +
.../plan/node/source/AlignedLastQueryScanNode.java | 13 +
.../source/AlignedSeriesAggregationScanNode.java | 11 +
.../plan/node/source/AlignedSeriesScanNode.java | 12 +
.../plan/node/source/LastQueryScanNode.java | 13 +
.../node/source/SeriesAggregationScanNode.java | 12 +
.../planner/plan/node/source/SeriesScanNode.java | 12 +
.../planner/plan/node/source/SeriesSourceNode.java | 4 +-
.../planner/plan/node/write/DeleteDataNode.java | 21 +
.../plan/statement/crud/InsertBaseStatement.java | 17 +-
.../crud/InsertMultiTabletsStatement.java | 7 +
.../plan/statement/crud/InsertRowStatement.java | 11 +
.../crud/InsertRowsOfOneDeviceStatement.java | 7 +
.../plan/statement/crud/InsertRowsStatement.java | 7 +
.../plan/statement/crud/InsertStatement.java | 35 +-
.../plan/statement/crud/InsertTabletStatement.java | 11 +
.../plan/statement/crud/LoadTsFileStatement.java | 4 +-
.../metadata/CreateAlignedTimeSeriesStatement.java | 55 +++
.../schemaengine/metric/SchemaEngineMemMetric.java | 15 +-
.../schemaengine/metric/SchemaRegionMemMetric.java | 21 +
.../rescon/ISchemaEngineStatistics.java | 2 +
.../rescon/MemSchemaEngineStatistics.java | 14 +
.../rescon/MemSchemaRegionStatistics.java | 3 +
.../schemaregion/utils/ResourceByPathUtils.java | 83 +---
.../db/storageengine/dataregion/DataRegion.java | 208 ++++++++--
.../dataregion/IDataRegionForQuery.java | 20 +-
.../dataregion/VirtualDataRegion.java | 27 ++
.../dataregion/memtable/AbstractMemTable.java | 311 +++++++++++++++
.../memtable/AlignedWritableMemChunk.java | 4 +
.../dataregion/memtable/IMemTable.java | 19 +
.../dataregion/memtable/TsFileProcessor.java | 322 ++++++++++++++++
.../dataregion/read/IQueryDataSource.java} | 11 +-
.../dataregion/read/QueryDataSource.java | 11 +-
.../read/QueryDataSourceForRegionScan.java | 64 +++
.../dataregion/read/QueryDataSourceType.java} | 15 +-
.../filescan/IChunkHandle.java} | 55 ++-
.../dataregion/read/filescan/IFileScanHandle.java | 99 +++++
.../filescan/impl/ClosedFileScanHandleImpl.java | 211 ++++++++++
.../filescan/impl/DiskAlignedChunkHandleImpl.java | 81 ++++
.../read/filescan/impl/DiskChunkHandleImpl.java | 137 +++++++
.../filescan/impl/MemAlignedChunkHandleImpl.java | 72 ++++
.../filescan/impl/MemChunkHandleImpl.java} | 46 ++-
.../filescan/impl/UnclosedFileScanHandleImpl.java | 180 +++++++++
.../filescan/model/AbstractChunkOffset.java} | 41 +-
.../model/AbstractDeviceChunkMetaData.java | 20 +-
.../read/filescan/model/AlignedChunkOffset.java | 52 +++
.../model/AlignedDeviceChunkMetaData.java} | 26 +-
.../read/filescan/model/ChunkOffset.java | 48 +++
.../read/filescan/model/DeviceChunkMetaData.java | 25 +-
.../read/filescan/model/DeviceStartEndTime.java} | 31 +-
.../dataregion/tsfile/TsFileResourceBlockType.java | 1 +
.../dataregion/utils/SharedTimeDataBuffer.java | 91 +++++
.../utils/TsFileDeviceStartEndTimeIterator.java | 52 +++
.../db/subscription/broker/SubscriptionBroker.java | 4 +-
.../broker/SubscriptionPrefetchingQueue.java | 6 +-
.../task/stage/SubscriptionTaskConnectorStage.java | 4 +-
.../task/subtask/SubscriptionConnectorSubtask.java | 6 +-
.../SubscriptionConnectorSubtaskLifeCycle.java | 4 +-
.../SubscriptionConnectorSubtaskManager.java | 20 +-
.../apache/iotdb/db/utils/ErrorHandlingUtils.java | 3 +
.../apache/iotdb/db/utils/ModificationUtils.java | 97 +++++
.../db/utils/datastructure/AlignedTVList.java | 7 +-
.../iotdb/db/utils/datastructure/BinaryTVList.java | 3 +-
.../db/utils/datastructure/BooleanTVList.java | 3 +-
.../iotdb/db/utils/datastructure/DoubleTVList.java | 3 +-
.../iotdb/db/utils/datastructure/FloatTVList.java | 3 +-
.../iotdb/db/utils/datastructure/IntTVList.java | 3 +-
.../iotdb/db/utils/datastructure/LongTVList.java | 3 +-
.../iotdb/db/utils/datastructure/TVList.java | 18 +-
.../iotdb/db/metadata/path/PatternTreeMapTest.java | 55 +++
.../connector/PipeDataNodeThriftRequestTest.java | 2 +-
.../PipeStatementPatternParseVisitorTest.java | 161 ++++++++
.../PipeConnectorSubtaskExecutorTest.java | 4 +-
.../extractor/IoTDBDataRegionExtractorTest.java | 18 +-
.../extractor/PipePlanPatternParseVisitorTest.java | 428 +++++++++++++++++++++
.../db/pipe/pattern/IoTDBPipePatternTest.java | 46 +--
.../db/pipe/pattern/PrefixPipePatternTest.java | 46 +--
.../plan/analyze/ExpressionAnalyzerTest.java | 8 +-
.../metrics/core/reporter/IoTDBJmxReporter.java | 4 +-
.../apache/iotdb/metrics/core/type/IoTDBRate.java | 2 +-
.../core/{uitls => utils}/IoTDBCachedGauge.java | 2 +-
.../{uitls => utils}/IoTDBCachedGaugeMBean.java | 2 +-
.../IoTDBMetricObjNameFactory.java | 2 +-
.../core/{uitls => utils}/IoTDBMovingAverage.java | 2 +-
.../core/{uitls => utils}/ObjectNameFactory.java | 2 +-
.../iotdb/commons/client/ClientPoolFactory.java | 2 +-
.../apache/iotdb/commons/conf/CommonConfig.java | 27 +-
.../iotdb/commons/conf/CommonDescriptor.java | 12 +-
.../apache/iotdb/commons/path/PatternTreeMap.java | 41 ++
.../commons/pipe/agent/task/PipeTaskAgent.java | 75 ++--
.../iotdb/commons/pipe/config/PipeConfig.java | 12 +-
.../config/constant/PipeConnectorConstant.java | 22 ++
.../config/constant/PipeExtractorConstant.java | 8 +
.../config/constant/PipeProcessorConstant.java | 11 +
.../pipe/connector/compressor/PipeCompressor.java} | 51 +--
.../compressor/PipeCompressorFactory.java | 80 ++++
.../connector/compressor/PipeGZIPCompressor.java} | 33 +-
.../connector/compressor/PipeLZ4Compressor.java} | 33 +-
.../connector/compressor/PipeLZMA2Compressor.java} | 33 +-
.../compressor/PipeSnappyCompressor.java} | 33 +-
.../connector/compressor/PipeZSTDCompressor.java} | 33 +-
.../payload/thrift/request/PipeRequestType.java | 3 +
.../thrift/request/PipeTransferCompressedReq.java | 146 +++++++
.../connector/protocol/IoTDBAirGapConnector.java | 7 +-
.../pipe/connector/protocol/IoTDBConnector.java | 62 +++
.../connector/protocol/IoTDBSslSyncConnector.java | 8 +-
.../listening/AbstractPipeListeningQueue.java | 11 +-
.../AbstractSerializableListeningQueue.java | 35 +-
.../commons/pipe/event/PipeWritePlanEvent.java | 9 +-
...ritePlanEvent.java => ProgressReportEvent.java} | 60 ++-
.../commons/pipe/extractor/IoTDBExtractor.java | 23 +-
.../extractor/IoTDBNonDataRegionExtractor.java | 89 +++--
.../pipe/metric/PipeEventCommitMetrics.java | 32 +-
.../commons/pipe/pattern/IoTDBPipePattern.java | 99 ++++-
.../iotdb/commons/pipe/pattern/PipePattern.java | 17 +-
.../commons/pipe/pattern/PrefixPipePattern.java | 20 +-
.../pipe/plugin/builtin/BuiltinPipePlugin.java | 4 +
.../ChangingValueSamplingProcessor.java} | 18 +-
.../pipe/progress/PipeEventCommitManager.java | 58 ++-
.../commons/pipe/progress/PipeEventCommitter.java | 16 +-
.../iotdb/commons/pipe/task/meta/PipeMeta.java | 38 +-
.../iotdb/commons/pipe/task/meta/PipeTaskMeta.java | 2 +-
.../commons/pipe/task/meta/PipeTemporaryMeta.java | 60 +++
.../pipe/task/subtask/PipeReportableSubtask.java | 2 +-
.../iotdb/commons/service/metric/enums/Metric.java | 9 +
.../src/main/thrift/datanode.thrift | 2 +
pom.xml | 12 +-
324 files changed, 11167 insertions(+), 2307 deletions(-)
create mode 100644 example/odbc/.gitignore
create mode 100644 example/odbc/ODBC-Example.sln
create mode 100644 example/odbc/ODBC-Example/ODBC-Example.csproj
create mode 100644 example/odbc/ODBC-Example/ODBC_Example.cs
create mode 100644 example/odbc/README.md
create mode 100644
integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/IoTDBPipeAutoDropIT.java
copy
integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/{IoTDBPipeProcessorIT.java
=> IoTDBPipeConnectorCompressionIT.java} (54%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/consensus/request/write/pipe/payload/PipeSetTTLPlan.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/heartbeat/PipeHeartbeat.java
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/{
=> heartbeat}/PipeHeartbeatParser.java (88%)
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/coordinator/runtime/{
=> heartbeat}/PipeHeartbeatScheduler.java (90%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanPatternParseVisitor.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeListenerMetrics.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeMetrics.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigNodeRemainingTimeOperator.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigRegionConnectorMetrics.java
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/pipe/metric/PipeConfigRegionExtractorMetrics.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/manager/pipe/extractor/PipeConfigPhysicalPlanPatternParseVisitorTest.java
copy
iotdb-core/{node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/PipeWritePlanEvent.java
=>
datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/terminate/PipeTerminateEvent.java}
(51%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/extractor/schemaregion/PipePlanPatternParseVisitor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeDataNodeRemainingEventAndTimeOperator.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/{PipeConnectorMetrics.java
=> PipeDataRegionConnectorMetrics.java} (77%)
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/{PipeExtractorMetrics.java
=> PipeDataRegionExtractorMetrics.java} (72%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeSchemaRegionConnectorMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeSchemaRegionExtractorMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/metric/PipeSchemaRegionListenerMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueFilter.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/processor/downsampling/changing/ChangingValueSamplingProcessor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/receiver/visitor/PipeStatementPatternParseVisitor.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/task/subtask/connector/PipeRealtimePriorityBlockingQueue.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/{execution/operator/source/DataSourceOperator.java
=> plan/expression/visitor/cartesian/QueryContextProvider.java} (75%)
copy
iotdb-core/{metrics/core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBCachedGaugeMBean.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/IQueryDataSource.java}
(77%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSourceForRegionScan.java
copy
iotdb-core/{metrics/core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBCachedGaugeMBean.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/QueryDataSourceType.java}
(77%)
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{tsfile/TsFileResourceBlockType.java
=> read/filescan/IChunkHandle.java} (53%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/IFileScanHandle.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/ClosedFileScanHandleImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskAlignedChunkHandleImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/DiskChunkHandleImpl.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/MemAlignedChunkHandleImpl.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{tsfile/TsFileResourceBlockType.java
=> read/filescan/impl/MemChunkHandleImpl.java} (50%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/impl/UnclosedFileScanHandleImpl.java
copy
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/{tsfile/TsFileResourceBlockType.java
=> read/filescan/model/AbstractChunkOffset.java} (52%)
copy
integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/AbstractDeviceChunkMetaData.java
(66%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/AlignedChunkOffset.java
copy
iotdb-core/{metrics/core/src/main/java/org/apache/iotdb/metrics/core/uitls/IoTDBCachedGaugeMBean.java
=>
datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/AlignedDeviceChunkMetaData.java}
(52%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/ChunkOffset.java
copy
integration-test/src/main/java/org/apache/iotdb/it/env/remote/config/RemoteConfigNodeConfig.java
=>
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/DeviceChunkMetaData.java
(56%)
rename
iotdb-core/datanode/src/{test/java/org/apache/iotdb/db/queryengine/exception/MemoryNotEnoughExceptionTest.java
=>
main/java/org/apache/iotdb/db/storageengine/dataregion/read/filescan/model/DeviceStartEndTime.java}
(57%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/SharedTimeDataBuffer.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TsFileDeviceStartEndTimeIterator.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/connector/PipeStatementPatternParseVisitorTest.java
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/extractor/PipePlanPatternParseVisitorTest.java
rename
iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/{uitls =>
utils}/IoTDBCachedGauge.java (98%)
rename
iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/{uitls =>
utils}/IoTDBCachedGaugeMBean.java (95%)
rename
iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/{uitls =>
utils}/IoTDBMetricObjNameFactory.java (98%)
rename
iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/{uitls =>
utils}/IoTDBMovingAverage.java (98%)
rename
iotdb-core/metrics/core/src/main/java/org/apache/iotdb/metrics/core/{uitls =>
utils}/ObjectNameFactory.java (96%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResourceBlockType.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeCompressor.java}
(50%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeCompressorFactory.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractDataSourceOperator.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeGZIPCompressor.java}
(51%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractDataSourceOperator.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeLZ4Compressor.java}
(51%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractDataSourceOperator.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeLZMA2Compressor.java}
(51%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractDataSourceOperator.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeSnappyCompressor.java}
(51%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/AbstractDataSourceOperator.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/compressor/PipeZSTDCompressor.java}
(51%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/payload/thrift/request/PipeTransferCompressedReq.java
copy
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/event/{PipeWritePlanEvent.java
=> ProgressReportEvent.java} (59%)
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/exception/MemoryNotEnoughException.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/plugin/builtin/processor/downsampling/ChangingValueSamplingProcessor.java}
(57%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/task/meta/PipeTemporaryMeta.java
