This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch beyyes/agg_template_alignbydevice
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 29caf67ab5a fix deviceToMeasurementIdx error in SingleDeviceViewNode
add 21a23967d7c Fix error compare method in IT framework
add 28bb9b8cd6c Pipe: Support pipe's endpoint & global connector transfer
rate limit & Fix RPC compression not enabled in stream batch mode (#12543)
add a9c27bb249d Fix potential NPE for empty InsertRowsOfOneDevice #12593
add a8f3ca3fcc0 Fix the bug that iot consensus doesn't adapt
InsertRowsNode (#12594)
add 88cadb2d800 more compaction config items allow hot load (#12596)
add 2ed61401d3c Subscription: support TsFile message format & improve
exception handling logic (#12326)
add 4403bb890b8 Fix agg push down bug (#12598)
add baf37c3711d add --help with remove-xx script (#12585)
add 8f0fbe350da Merge branch 'master' into
beyyes/agg_template_alignbydevice
add 30b041e6402 remove useless code
add d5dbbafa9eb add more having case
No new revisions were added by this update.
Summary of changes:
.../apache/iotdb/SubscriptionSessionExample.java | 188 +++--
.../iotdb/itbase/runtime/RequestDelegate.java | 26 +-
...oTDBAlignByDeviceWithTemplateAggregationIT.java | 57 ++
.../it/IoTDBSubscriptionITConstant.java | 19 +-
.../it/dual/AbstractSubscriptionDualIT.java | 3 +
.../it/dual/IoTDBSubscriptionConsumerGroupIT.java | 327 +++++++-
.../it/dual/IoTDBSubscriptionTopicIT.java | 228 ++++--
.../it/local/IoTDBSubscriptionBasicIT.java | 227 ++++--
.../it/local/IoTDBSubscriptionRestartIT.java | 103 ++-
iotdb-client/service-rpc/pom.xml | 4 +
.../rpc/subscription/config/ConsumerConfig.java | 11 +-
.../rpc/subscription/config/ConsumerConstant.java | 22 +-
.../iotdb/rpc/subscription/config/TopicConfig.java | 37 +-
.../rpc/subscription/config/TopicConstant.java | 5 +-
...n.java => SubscriptionConnectionException.java} | 21 +-
.../exception/SubscriptionException.java | 8 +-
... SubscriptionIncompatibleHandlerException.java} | 25 +-
.../SubscriptionParameterNotValidException.java | 13 +-
...a => SubscriptionRuntimeCriticalException.java} | 24 +-
...tion.java => SubscriptionRuntimeException.java} | 23 +-
...> SubscriptionRuntimeNonCriticalException.java} | 25 +-
.../rpc/subscription/payload/EnrichedTablets.java | 104 ---
.../subscription/payload/poll/ErrorPayload.java | 85 ++
.../subscription/payload/poll/FileInitPayload.java | 75 ++
.../payload/poll/FilePiecePayload.java | 123 +++
.../subscription/payload/poll/FileSealPayload.java | 85 ++
.../subscription/payload/poll/PollFilePayload.java | 103 +++
.../rpc/subscription/payload/poll/PollPayload.java | 79 ++
.../payload/poll/SubscriptionCommitContext.java | 152 ++++
.../SubscriptionPollPayload.java} | 18 +-
.../payload/poll/SubscriptionPollRequest.java | 111 +++
.../SubscriptionPollRequestType.java} | 20 +-
.../payload/poll/SubscriptionPollResponse.java | 122 +++
.../SubscriptionPollResponseType.java} | 25 +-
.../subscription/payload/poll/TabletsPayload.java | 92 +++
.../payload/request/PipeSubscribeCloseReq.java | 6 +-
.../payload/request/PipeSubscribeCommitReq.java | 48 +-
.../payload/request/PipeSubscribeHandshakeReq.java | 19 +-
.../payload/request/PipeSubscribeHeartbeatReq.java | 7 +-
.../payload/request/PipeSubscribePollReq.java | 46 +-
.../payload/request/PipeSubscribeRequestType.java | 8 +-
.../request/PipeSubscribeRequestVersion.java | 2 +-
.../payload/request/PipeSubscribeSubscribeReq.java | 9 +-
.../request/PipeSubscribeUnsubscribeReq.java | 8 +-
.../payload/response/PipeSubscribeCloseResp.java | 8 +-
.../payload/response/PipeSubscribeCommitResp.java | 9 +-
.../response/PipeSubscribeHandshakeResp.java | 27 +-
.../response/PipeSubscribeHeartbeatResp.java | 8 +-
.../payload/response/PipeSubscribePollResp.java | 88 +-
.../response/PipeSubscribeResponseType.java | 9 +-
.../response/PipeSubscribeResponseVersion.java | 2 +-
.../response/PipeSubscribeSubscribeResp.java | 8 +-
.../response/PipeSubscribeUnsubscribeResp.java | 8 +-
.../session/subscription/AsyncCommitCallback.java | 2 +-
.../session/subscription/ConsumeListener.java | 2 +
.../subscription/ConsumerHeartbeatWorker.java | 63 --
.../subscription/PullConsumerAutoCommitWorker.java | 60 --
.../session/subscription/SubscriptionConsumer.java | 884 ++++++++++++++-------
.../subscription/SubscriptionEndpointsSyncer.java | 132 ---
.../session/subscription/SubscriptionMessage.java | 85 --
.../session/subscription/SubscriptionProvider.java | 295 ++++++-
.../subscription/SubscriptionProviders.java | 339 ++++++++
.../subscription/SubscriptionPullConsumer.java | 147 ++--
.../subscription/SubscriptionPushConsumer.java | 112 ++-
.../session/subscription/SubscriptionSession.java | 48 +-
.../SubscriptionSessionConnection.java | 73 +-
.../payload/SubscriptionFileHandler.java | 105 +++
.../subscription/payload/SubscriptionMessage.java | 106 +++
.../SubscriptionMessageHandler.java} | 9 +-
.../payload/SubscriptionMessageType.java} | 20 +-
.../{ => payload}/SubscriptionSessionDataSet.java | 35 +-
.../SubscriptionSessionDataSetsHandler.java} | 35 +-
.../payload/SubscriptionTsFileHandler.java} | 24 +-
.../subscription/util/RandomStringGenerator.java} | 22 +-
.../subscription/util}/SubscriptionPollTimer.java | 2 +-
.../assembly/resources/sbin/remove-confignode.bat | 12 +
.../assembly/resources/sbin/remove-confignode.sh | 11 +
.../protocol/IoTDBConfigRegionAirGapConnector.java | 51 +-
.../protocol/IoTDBConfigRegionConnector.java | 52 +-
.../assembly/resources/sbin/remove-datanode.bat | 12 +
.../src/assembly/resources/sbin/remove-datanode.sh | 11 +
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 48 +-
.../dataregion/DataRegionStateMachine.java | 65 +-
.../payload/evolvable/builder/PipeEventBatch.java | 18 +
.../airgap/IoTDBDataNodeAirGapConnector.java | 20 +-
.../airgap/IoTDBDataRegionAirGapConnector.java | 55 +-
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 32 +-
.../async/IoTDBDataRegionAsyncConnector.java | 11 +-
.../PipeTransferTabletBatchEventHandler.java | 19 +-
.../PipeTransferTabletInsertionEventHandler.java | 5 +
.../PipeTransferTsFileInsertionEventHandler.java | 35 +-
.../thrift/sync/IoTDBDataNodeSyncConnector.java | 17 +-
.../thrift/sync/IoTDBDataRegionSyncConnector.java | 127 +--
.../thrift/sync/IoTDBSchemaRegionConnector.java | 36 +-
.../execution/load/LoadTsFileRateLimiter.java | 32 +-
.../plan/analyze/TemplatedAggregationAnalyze.java | 33 +-
.../config/executor/ClusterConfigTaskExecutor.java | 10 +-
.../plan/optimization/AggregationPushDown.java | 3 -
.../plan/parser/StatementGenerator.java | 6 +-
.../queryengine/plan/planner/TreeModelPlanner.java | 4 -
.../db/service/metrics/DataNodeMetricsHelper.java | 4 +
.../impl/RewriteCrossSpaceCompactionSelector.java | 5 +-
.../impl/SizeTieredCompactionSelector.java | 5 +-
.../agent/SubscriptionBrokerAgent.java | 94 ++-
.../agent/SubscriptionConsumerAgent.java | 7 +-
.../subscription/agent/SubscriptionTopicAgent.java | 20 +-
.../broker/EnrichedTabletsBinaryCache.java | 62 --
.../db/subscription/broker/SubscriptionBroker.java | 101 ++-
.../broker/SubscriptionPrefetchingQueue.java | 269 ++-----
.../SubscriptionPrefetchingTabletsQueue.java | 244 ++++++
.../broker/SubscriptionPrefetchingTsFileQueue.java | 377 +++++++++
.../SubscriptionEvent.java} | 132 +--
.../event/SubscriptionEventBinaryCache.java | 140 ++++
.../event/SubscriptionTsFileEvent.java | 247 ++++++
.../subscription/metric/SubscriptionMetrics.java | 57 ++
.../SubscriptionPrefetchingQueueMetrics.java | 197 +++++
.../receiver/SubscriptionReceiverV1.java | 323 +++++---
.../dataregion/DataRegionStateMachineTest.java | 46 +-
.../resources/conf/iotdb-common.properties | 10 +
.../async/AsyncPipeDataTransferServiceClient.java | 4 +
.../apache/iotdb/commons/conf/CommonConfig.java | 55 +-
.../iotdb/commons/conf/CommonDescriptor.java | 28 +-
.../iotdb/commons/pipe/config/PipeConfig.java | 17 +
.../config/constant/PipeConnectorConstant.java | 4 +
.../pipe/connector/client/IoTDBSyncClient.java | 7 +
.../pipe/connector/limiter/GlobalRateLimiter.java} | 55 +-
.../connector/limiter/PipeEndPointRateLimiter.java | 59 ++
.../connector/protocol/IoTDBAirGapConnector.java | 48 +-
.../pipe/connector/protocol/IoTDBConnector.java | 45 +-
.../connector/protocol/IoTDBSslSyncConnector.java | 24 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 1 +
.../iotdb/commons/service/metric/enums/Metric.java | 4 +
.../subscription/config/SubscriptionConfig.java | 18 +-
.../commons/subscription/meta/topic/TopicMeta.java | 37 +-
135 files changed, 6485 insertions(+), 2498 deletions(-)
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestVersion.java
=>
integration-test/src/test/java/org/apache/iotdb/subscription/it/IoTDBSubscriptionITConstant.java
(68%)
rename
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/{SubscriptionPollTimeOutException.java
=> SubscriptionConnectionException.java} (59%)
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/{SubscriptionException.java
=> SubscriptionIncompatibleHandlerException.java} (59%)
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/{SubscriptionException.java
=> SubscriptionRuntimeCriticalException.java} (60%)
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/{SubscriptionException.java
=> SubscriptionRuntimeException.java} (60%)
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/exception/{SubscriptionException.java
=> SubscriptionRuntimeNonCriticalException.java} (59%)
delete mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/EnrichedTablets.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/ErrorPayload.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileInitPayload.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FilePiecePayload.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/FileSealPayload.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollFilePayload.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/PollPayload.java
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionCommitContext.java
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/{request/PipeSubscribeRequestVersion.java
=> poll/SubscriptionPollPayload.java} (71%)
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollRequest.java
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/{response/PipeSubscribeResponseType.java
=> poll/SubscriptionPollRequestType.java} (67%)
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/SubscriptionPollResponse.java
copy
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/{response/PipeSubscribeResponseType.java
=> poll/SubscriptionPollResponseType.java} (64%)
create mode 100644
iotdb-client/service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/poll/TabletsPayload.java
delete mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/ConsumerHeartbeatWorker.java
delete mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/PullConsumerAutoCommitWorker.java
delete mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionEndpointsSyncer.java
delete mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionMessage.java
create mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/SubscriptionProviders.java
create mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionFileHandler.java
create mode 100644
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessage.java
rename
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/{SubscriptionMessagePayload.java
=> payload/SubscriptionMessageHandler.java} (78%)
copy
iotdb-client/{service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/response/PipeSubscribeResponseType.java
=>
session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionMessageType.java}
(67%)
rename
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/{ =>
payload}/SubscriptionSessionDataSet.java (80%)
rename
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/{SubscriptionSessionDataSets.java
=> payload/SubscriptionSessionDataSetsHandler.java} (55%)
copy
iotdb-client/{service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestVersion.java
=>
session/src/main/java/org/apache/iotdb/session/subscription/payload/SubscriptionTsFileHandler.java}
(59%)
copy
iotdb-client/{service-rpc/src/main/java/org/apache/iotdb/rpc/subscription/payload/request/PipeSubscribeRequestVersion.java
=>
session/src/main/java/org/apache/iotdb/session/subscription/util/RandomStringGenerator.java}
(59%)
rename
{iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/timer =>
iotdb-client/session/src/main/java/org/apache/iotdb/session/subscription/util}/SubscriptionPollTimer.java
(97%)
mode change 100644 => 100755
iotdb-core/datanode/src/assembly/resources/sbin/remove-datanode.sh
delete mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/EnrichedTabletsBinaryCache.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTabletsQueue.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/broker/SubscriptionPrefetchingTsFileQueue.java
rename
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/{broker/SerializedEnrichedEvent.java
=> event/SubscriptionEvent.java} (53%)
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionEventBinaryCache.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/event/SubscriptionTsFileEvent.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/SubscriptionMetrics.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/subscription/metric/SubscriptionPrefetchingQueueMetrics.java
copy
iotdb-core/{datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/load/LoadTsFileRateLimiter.java
=>
node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/GlobalRateLimiter.java}
(51%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/connector/limiter/PipeEndPointRateLimiter.java
