This is an automated email from the ASF dual-hosted git repository.
caogaofei pushed a change to branch agg_table_scan
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 321a4f932ca perfect aggregator and accumulator
add d6598d634e4 Feature/move SBOM check into separate profile (#13649)
add 08bc12c67bd Adjust procedure's default clean interval #13648
add a1ca84a78b3 [Config-properties] Change
iot_consensus_throttle_threshold_in_byte = wal_throttle_threshold_in_byte for
compatibility (#13652)
add c7a31ffa81f Invalidate schema cache when data region leader change
(#13585)
add 9587da224bb Pipe: fix npe when discard events under some cases (#13661)
add 9015ffbc4fc Pipe: Fixed the bug that timeChunk
TsFileInsertionScanDataContainer can be used only once (#13656)
add 8fca51a3c23 Fix unexpected column order in ExportTsFile when wildcard
is not used (#13662)
add b488a79a89c Pipe: fix NPE in SessionUtils Binary/Date value
serialization called by TsFileInsertionScanDataContainer (#13654)
add cc7394642de Remove datanode optimization (#13559)
add 8efd9ef0ac4 Replenish annotations to load balancers (#13675)
add 195949d6439 Feature/make load active listening pipe dir settable
(#13674)
add 220c953f0ba AINode: depend torch of cpu version (#13670)
add 2ec48d28a6c Active Load: Fix load tsfile failed when system is read
only (#13646)
add 5cd607bb5e8 Change database in SessionConnection while using db
add fc965db8734 Add serialize and deserialize method for Agg-related Node
add 5bae76344c0 Bump commons-io:commons-io from 2.13.0 to 2.14.0 (#13676)
add e09a98b4a0b Fix the order of stop schema region and clear schema
engine #13678
add 3544970fe6f Fix the failure of update cluster name in metric framework
of ConfigNode (#13681)
add ff1f1781dee Try to fix error msg like: 301: queue has been destroyed
add 1c2451af9b5 Table ttl & Fix ttl bugs (#13668)
add 4a5c90293da ConfigManager confirms leader before create schema
partition #13693
add 9cc8ffddc0e Catch Exception to improve the robustness of loading
procedure #13696
add 360ca21fe3c Pipe: Introduce TablePattern for table model (#13695)
new 232a29500ff Merge branch 'master' into agg_table_scan
new d4defa7e0a8 add basic agg table scan impl
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/dependency-check.yml | 2 +-
.../IoTDBRegionMigrateReliabilityITFramework.java | 4 +-
.../IoTDBRemoveDataNodeITFramework.java | 288 +++++++++++++
.../IoTDBRemoveDataNodeNormalIT.java} | 14 +-
...FormatIT.java => IoTDBTreePatternFormatIT.java} | 2 +-
.../relational/it/db/it/IoTDBInsertTableIT.java | 79 ++++
.../iotdb/session/it/IoTDBSessionSimpleIT.java | 37 ++
.../org/apache/iotdb/tool/tsfile/ExportTsFile.java | 203 +++++----
.../apache/iotdb/session/SessionConnection.java | 6 +-
iotdb-core/ainode/resources/conf/ainode-env.bat | 4 +-
iotdb-core/ainode/resources/conf/ainode-env.sh | 4 +-
.../iotdb/confignode/client/CnToDnRequestType.java | 2 +-
.../CnToDnInternalServiceAsyncRequestManager.java | 15 +
.../rpc/DataNodeAsyncRequestRPCHandler.java | 3 +
.../client/sync/SyncDataNodeClientPool.java | 6 +-
.../iotdb/confignode/conf/ConfigNodeConfig.java | 4 +-
.../iotdb/confignode/manager/ConfigManager.java | 8 +
.../iotdb/confignode/manager/ProcedureManager.java | 298 ++++++++++---
.../manager/load/balancer/RegionBalancer.java | 4 +-
.../manager/load/balancer/RouteBalancer.java | 39 +-
.../region/GreedyRegionGroupAllocator.java | 57 ++-
...PartiteGraphPlacementRegionGroupAllocator.java} | 162 ++++---
...r.java => CostFlowSelectionLeaderBalancer.java} | 263 ++++++------
.../iotdb/confignode/manager/node/NodeManager.java | 85 ++--
.../manager/partition/PartitionManager.java | 2 +-
.../protocol/IoTDBConfigRegionAirGapConnector.java | 2 +-
.../protocol/IoTDBConfigRegionConnector.java | 2 +-
.../pipe/event/PipeConfigRegionSnapshotEvent.java | 28 +-
.../pipe/event/PipeConfigRegionWritePlanEvent.java | 15 +-
.../PipeConfigPhysicalPlanPatternParseVisitor.java | 52 +--
.../receiver/protocol/IoTDBConfigNodeReceiver.java | 6 +-
.../confignode/persistence/ProcedureInfo.java | 2 +-
.../procedure/CompletedProcedureRecycler.java | 2 +-
.../procedure/env/ConfigNodeProcedureEnv.java | 77 +---
.../procedure/env/RegionMaintainHandler.java | 243 +----------
.../procedure/env/RemoveDataNodeHandler.java | 469 +++++++++++++++++++++
...rocedure.java => RemoveDataNodesProcedure.java} | 162 ++++---
.../pipe/plugin/CreatePipePluginProcedure.java | 4 +-
.../impl/pipe/plugin/DropPipePluginProcedure.java | 4 +-
.../impl/region/RegionMigrateProcedure.java | 4 +
.../procedure/impl/region/RegionMigrationPlan.java | 97 +++++
.../procedure/store/ProcedureFactory.java | 6 +-
.../iotdb/confignode/service/ConfigNode.java | 21 +-
.../router/leader/CFDLeaderBalancerTest.java | 6 +-
.../leader/LeaderBalancerComparisonTest.java | 4 +-
...eConfigPhysicalPlanPatternParseVisitorTest.java | 6 +-
.../UpgradeFromWALToConsensusLayerTest.java | 31 +-
.../impl/node/RemoveDataNodeProcedureTest.java | 55 ---
.../impl/node/RemoveDataNodesProcedureTest.java | 90 ++++
.../assembly/resources/sbin/remove-datanode.bat | 38 +-
.../src/assembly/resources/sbin/remove-datanode.sh | 25 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 25 +-
.../agent/task/connection/PipeEventCollector.java | 7 +-
.../subtask/connector/PipeConnectorSubtask.java | 2 +-
.../airgap/IoTDBSchemaRegionAirGapConnector.java | 2 +-
.../thrift/sync/IoTDBSchemaRegionConnector.java | 2 +-
.../db/pipe/event/UserDefinedEnrichedEvent.java | 11 +-
.../event/common/heartbeat/PipeHeartbeatEvent.java | 10 +-
.../schema/PipeSchemaRegionSnapshotEvent.java | 22 +-
.../schema/PipeSchemaRegionWritePlanEvent.java | 21 +-
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 23 +-
.../common/tablet/PipeRawTabletInsertionEvent.java | 42 +-
.../tablet/TabletInsertionDataContainer.java | 16 +-
.../event/common/terminate/PipeTerminateEvent.java | 8 +-
.../common/tsfile/PipeTsFileInsertionEvent.java | 21 +-
.../common/tsfile/TsFileInsertionPointCounter.java | 6 +-
.../container/TsFileInsertionDataContainer.java | 6 +-
.../TsFileInsertionDataContainerProvider.java | 14 +-
.../query/TsFileInsertionQueryDataContainer.java | 8 +-
.../scan/TsFileInsertionScanDataContainer.java | 19 +-
.../db/pipe/event/realtime/PipeRealtimeEvent.java | 30 +-
.../event/realtime/PipeRealtimeEventFactory.java | 6 +-
.../dataregion/IoTDBDataRegionExtractor.java | 38 +-
.../PipeHistoricalDataRegionTsFileExtractor.java | 20 +-
.../realtime/PipeRealtimeDataRegionExtractor.java | 57 ++-
.../realtime/assigner/PipeDataRegionAssigner.java | 6 +-
.../realtime/epoch/TsFileEpochManager.java | 6 +-
.../matcher/CachedSchemaPatternMatcher.java | 9 +-
.../schemaregion/PipePlanPatternParseVisitor.java | 52 +--
.../protocol/thrift/IoTDBDataNodeReceiver.java | 6 +-
...peStatementDataTypeConvertExecutionVisitor.java | 4 +-
.../visitor/PipeStatementPatternParseVisitor.java | 26 +-
.../impl/DataNodeInternalRPCServiceImpl.java | 5 +-
.../execution/exchange/SharedTsBlockQueue.java | 10 +
.../exchange/source/LocalSourceHandle.java | 12 +
.../operator/schema/source/DeviceSchemaSource.java | 2 +-
.../source/ActiveDeviceRegionScanOperator.java | 2 +-
.../execution/operator/source/SeriesScanUtil.java | 16 +-
.../TableAggregationTableScanOperator.java | 33 +-
.../db/queryengine/plan/analyze/TypeProvider.java | 10 +-
.../analyze/cache/schema/DataNodeTTLCache.java | 62 +--
.../plan/planner/OperatorTreeGenerator.java | 8 +-
.../plan/planner/TableOperatorGenerator.java | 67 +--
.../plan/planner/plan/node/PlanNodeType.java | 8 +
.../plan/relational/function/BoundSignature.java | 36 ++
.../plan/relational/function/FunctionKind.java | 20 +-
.../plan/relational/metadata/ColumnSchema.java | 47 +--
.../relational/metadata/FunctionNullability.java | 31 ++
.../plan/relational/metadata/ResolvedFunction.java | 32 +-
.../TableModelTypeProviderExtractor.java | 16 +-
.../relational/planner/node/AggregationNode.java | 219 +++++++++-
.../planner/node/AggregationTableScanNode.java | 275 ++++++++++++
.../plan/relational/utils/TypeUtil.java | 105 +++++
.../java/org/apache/iotdb/db/service/DataNode.java | 37 +-
.../iotdb/db/service/RegionMigrateService.java | 4 +-
.../iotdb/db/storageengine/StorageEngine.java | 8 +-
.../db/storageengine/dataregion/DataRegion.java | 47 +--
.../execute/performer/ICompactionPerformer.java | 2 -
.../performer/impl/FastCompactionPerformer.java | 49 ++-
.../impl/ReadChunkCompactionPerformer.java | 16 +-
.../impl/ReadPointCompactionPerformer.java | 10 +-
.../RepairUnsortedFileCompactionPerformer.java | 3 +-
.../task/RepairUnsortedFileCompactionTask.java | 10 +-
.../execute/utils/MultiTsFileDeviceIterator.java | 42 +-
.../ReadChunkAlignedSeriesCompactionExecutor.java | 8 +-
.../repair/RepairTimePartitionScanTask.java | 6 +-
.../schedule/CompactionScheduleContext.java | 25 +-
.../selector/impl/SettleSelectorImpl.java | 16 +-
.../impl/SizeTieredCompactionSelector.java | 3 +-
.../dataregion/memtable/TsFileProcessor.java | 13 +-
.../dataregion/read/QueryDataSource.java | 5 +
.../storageengine/dataregion/wal/WALManager.java | 2 +-
.../load/active/ActiveLoadDirScanner.java | 13 +
.../load/active/ActiveLoadTsFileLoader.java | 15 +
.../PipeStatementPatternParseVisitorTest.java | 6 +-
.../pipe/event/PipeTabletInsertionEventTest.java | 26 +-
.../event/TsFileInsertionDataContainerTest.java | 48 +--
.../extractor/PipePlanPatternParseVisitorTest.java | 6 +-
.../pattern/CachedSchemaPatternMatcherTest.java | 16 +-
...ePatternTest.java => IoTDBTreePatternTest.java} | 24 +-
...PatternTest.java => PrefixTreePatternTest.java} | 24 +-
.../db/storageengine/dataregion/TTLQueryTest.java | 14 +-
.../iotdb/db/storageengine/dataregion/TTLTest.java | 46 +-
.../compaction/AbstractCompactionTest.java | 3 +-
.../repair/RepairUnsortedFileCompactionTest.java | 32 +-
.../settle/SettleCompactionRecoverTest.java | 2 +-
.../settle/SettleCompactionSelectorTest.java | 14 +-
.../settle/SettleCompactionTaskTest.java | 22 +-
.../tablemodel/CompactionWithAllNullRowsTest.java | 9 +-
.../TableModelCompactionWithTTLTest.java | 191 +++++++++
.../compaction/utils/CompactionCheckerUtils.java | 3 +-
.../utils/MultiTsFileDeviceIteratorTest.java | 38 +-
.../conf/iotdb-system.properties.template | 4 +-
.../apache/iotdb/commons/ServerCommandLine.java | 24 +-
.../client/request/AsyncRequestManager.java | 2 +-
.../config/constant/PipeExtractorConstant.java | 4 +
...IoTDBPipePattern.java => IoTDBTreePattern.java} | 12 +-
...efixPipePattern.java => PrefixTreePattern.java} | 4 +-
.../pipe/datastructure/pattern/TablePattern.java | 94 +++++
.../pattern/{PipePattern.java => TreePattern.java} | 28 +-
.../iotdb/commons/pipe/event/EnrichedEvent.java | 48 ++-
.../commons/pipe/event/PipeSnapshotEvent.java | 15 +-
.../commons/pipe/event/PipeWritePlanEvent.java | 15 +-
.../commons/pipe/event/ProgressReportEvent.java | 13 +-
.../extractor/IoTDBNonDataRegionExtractor.java | 40 +-
.../apache/iotdb/commons/schema/table/TsTable.java | 15 +
.../iotdb/commons/ServerCommandLineTest.java | 35 +-
.../src/main/thrift/datanode.thrift | 8 +-
pom.xml | 181 ++++----
160 files changed, 4217 insertions(+), 1770 deletions(-)
create mode 100644
integration-test/src/test/java/org/apache/iotdb/confignode/it/removedatanode/IoTDBRemoveDataNodeITFramework.java
copy
integration-test/src/test/java/org/apache/iotdb/confignode/it/{regionmigration/pass/IoTDBRegionMigrateNormalIT.java
=> removedatanode/IoTDBRemoveDataNodeNormalIT.java} (66%)
rename
integration-test/src/test/java/org/apache/iotdb/pipe/it/autocreate/{IoTDBPipePatternFormatIT.java
=> IoTDBTreePatternFormatIT.java} (99%)
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/region/{PartiteGraphReplicationRegionGroupAllocator.java
=> PartiteGraphPlacementRegionGroupAllocator.java} (59%)
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/manager/load/balancer/router/leader/{MinCostFlowLeaderBalancer.java
=> CostFlowSelectionLeaderBalancer.java} (52%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/env/RemoveDataNodeHandler.java
rename
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/node/{RemoveDataNodeProcedure.java
=> RemoveDataNodesProcedure.java} (55%)
create mode 100644
iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/procedure/impl/region/RegionMigrationPlan.java
delete mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodeProcedureTest.java
create mode 100644
iotdb-core/confignode/src/test/java/org/apache/iotdb/confignode/procedure/impl/node/RemoveDataNodesProcedureTest.java
create mode 100644
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/utils/TypeUtil.java
rename
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/{IoTDBPipePatternTest.java
=> IoTDBTreePatternTest.java} (83%)
rename
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/pattern/{PrefixPipePatternTest.java
=> PrefixTreePatternTest.java} (82%)
create mode 100644
iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/tablemodel/TableModelCompactionWithTTLTest.java
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/{IoTDBPipePattern.java
=> IoTDBTreePattern.java} (95%)
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/{PrefixPipePattern.java
=> PrefixTreePattern.java} (97%)
create mode 100644
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/TablePattern.java
rename
iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/pipe/datastructure/pattern/{PipePattern.java
=> TreePattern.java} (85%)
